Mirrors/wordcraft
1
0 Fork 0
mirror of https://github.com/storytold/wordcraft.git synced 2026-11-13 01:31:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream/main into ui-text-hinting

Browse Source
This commit is contained in:
vangel76 committed 2026-10-26 02:15:38 +01:00
2 parents 827fc6e07a ebf27211cf
commit cfa99b45ce
101 files changed
+5930 -861
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+91 -172
View File
@@ -221,9 +221,9 @@ dependencies = [
[[package]]
name = "arboard"
version = "3.6.1"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf"
checksum = "aeb9553e62312c8756a5979254fc6d7da8bbeb5986a1e17037fab932f7037917"
dependencies = [
"clipboard-win",
"image",
@@ -235,7 +235,7 @@ dependencies = [
"objc2-foundation 0.3.2",
"parking_lot",
"percent-encoding",
"windows-sys 0.60.2",
"windows-sys 0.59.0",
"wl-clipboard-rs",
"x11rb",
]
@@ -489,15 +489,6 @@ version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bit-set"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
"bit-vec 0.8.0",
]
[[package]]
name = "bit-set"
version = "0.10.0"
@@ -508,10 +499,13 @@ dependencies = [
]
[[package]]
name = "bit-vec"
version = "0.8.0"
name = "bit-set"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c"
dependencies = [
"bit-vec 0.10.1",
]
[[package]]
name = "bit-vec"
@@ -519,6 +513,16 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51"
[[package]]
name = "bit-vec"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269"
dependencies = [
"borsh",
"serde",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -580,6 +584,30 @@ dependencies = [
"piper",
]
[[package]]
name = "borsh"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a"
dependencies = [
"borsh-derive",
"bytes",
"cfg_aliases",
]
[[package]]
name = "borsh-derive"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56"
dependencies = [
"once_cell",
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "brotli"
version = "8.0.4"
@@ -1545,7 +1573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -2842,9 +2870,9 @@ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
[[package]]
name = "libc"
version = "0.2.190"
version = "0.2.191"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
[[package]]
name = "libloading"
@@ -3457,9 +3485,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "open"
version = "5.4.4"
version = "5.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc"
checksum = "8d8c8b8e725ad26404d6896eb2f064294a0a7b4f1d9120210d85544684c4ca95"
dependencies = [
"is-wsl",
"libc",
@@ -3516,7 +3544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3846,15 +3874,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958"
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "presser"
version = "0.3.1"
@@ -3911,16 +3930,17 @@ checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5"
[[package]]
name = "proptest"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
checksum = "b8530004ccb15eae51c7e40009fbe317f341f804db54dc033eec1c50be28cfa0"
dependencies = [
"bit-set 0.8.0",
"bit-vec 0.8.0",
"bit-set 0.11.1",
"bit-vec 0.10.1",
"bitflags 2.13.2",
"chacha20",
"core_detect",
"num-traits",
"rand 0.9.5",
"rand_chacha",
"rand",
"rand_xorshift",
"regex-syntax",
"rusty-fork",
@@ -3994,7 +4014,7 @@ dependencies = [
"bytes",
"getrandom 0.4.3",
"lru-slab",
"rand 0.10.3",
"rand",
"rand_pcg",
"ring",
"rustc-hash 2.1.3",
@@ -4018,7 +4038,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -4042,16 +4062,6 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha",
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.3"
@@ -4063,31 +4073,12 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "rand_chacha"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.5",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
[[package]]
name = "rand_core"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_core"
version = "0.10.1"
@@ -4105,11 +4096,11 @@ dependencies = [
[[package]]
name = "rand_xorshift"
version = "0.4.0"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
checksum = "60aa6af80be32871323012e02e6e65f8a7cc7890931ae421d217ad8fe0df2ccf"
dependencies = [
"rand_core 0.9.5",
"rand_core 0.10.1",
]
[[package]]
@@ -4395,7 +4386,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -4454,7 +4445,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -4612,9 +4603,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.151"
version = "1.0.154"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
dependencies = [
"itoa",
"memchr",
@@ -4701,9 +4692,9 @@ dependencies = [
[[package]]
name = "sha2"
version = "0.11.0"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
dependencies = [
"cfg-if",
"cpufeatures 0.3.1",
@@ -4838,7 +4829,7 @@ dependencies = [
"const-oid 0.9.6",
"der",
"hex",
"rand 0.10.3",
"rand",
"reqwest",
"rustls-pki-types",
"rustls-webpki",
@@ -5238,10 +5229,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix 1.1.5",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -6278,7 +6269,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -6399,7 +6390,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -6408,16 +6399,7 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
"windows-targets",
]
[[package]]
@@ -6435,31 +6417,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
@@ -6477,96 +6442,48 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winit"
version = "0.30.13"
@@ -6659,9 +6576,9 @@ dependencies = [
[[package]]
name = "winresource"
version = "0.1.31"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087"
checksum = "a2bc6d36480b829272a5bdabf08e6dca8004761c1da7e35efef800540c66221b"
dependencies = [
"toml",
"version_check",
@@ -6716,7 +6633,7 @@ dependencies = [
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"open",
"pollster 0.4.0",
"pollster 1.0.1",
"rfd",
"serde_json",
"windows",
@@ -6753,7 +6670,8 @@ version = "0.11.0"
name = "wordcraft-doc"
version = "0.11.0"
dependencies = [
"base64 0.22.1",
"base64 0.23.1",
"log",
"proptest",
"serde",
"serde_json",
@@ -6770,7 +6688,7 @@ dependencies = [
"chrono",
"image",
"log",
"miniz_oxide 0.8.9",
"miniz_oxide 0.9.1",
"proptest",
"serde_json",
"sha1 0.11.0",
@@ -6784,7 +6702,7 @@ name = "wordcraft-docx"
version = "0.11.0"
dependencies = [
"aes",
"base64 0.22.1",
"base64 0.23.1",
"cfb",
"chrono",
"getrandom 0.3.4",
@@ -6794,7 +6712,7 @@ dependencies = [
"proptest",
"quick-xml 0.41.0",
"sha1 0.11.0",
"sha2 0.11.0",
"sha2 0.11.1",
"thiserror 2.0.21",
"unicode-bidi",
"wordcraft-doc",
@@ -6838,6 +6756,7 @@ dependencies = [
name = "wordcraft-fonts"
version = "0.11.0"
dependencies = [
"brotli-decompressor",
"harfrust 0.13.3",
"kurbo",
"log",
@@ -6939,7 +6858,7 @@ name = "wordcraft-proof"
version = "0.11.0"
dependencies = [
"log",
"miniz_oxide 0.8.9",
"miniz_oxide 0.9.1",
"proptest",
"unicode-normalization",
]
Cargo.toml
+7 -3
View File
@@ -52,6 +52,10 @@ harfrust = "0.13"
# WOFF / WOFF2 web fonts added by the user (#582): pure Rust (MIT); its brotli and zlib decoders
# (brotli-decompressor, flate2) are already in the tree.
wuff = "0.2.9"
# WOFF2's brotli stream, decoded ahead of wuff so its glyph point counts are checked first (#3703).
# The same crate and version wuff already decodes it with (no new code in the build): pure Rust,
# `forbid(unsafe_code)` in its default build, no_std-capable (builds for wasm), BSD-3-Clause/MIT.
brotli-decompressor = "5"
vello_cpu = "0.2"
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "gif", "bmp", "tiff"] }
craft-pdf = { git = "https://github.com/storytold/craft-libs", rev = "17744430be327d0257496a6a31c8d38fe20a6756", version = "=0.1.0", default-features = false, features = ["write"] }
@@ -59,7 +63,7 @@ craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.
quick-xml = "0.41"
zip = { version = "3", default-features = false, features = ["deflate"] }
# zlib streams (compressed metafile pictures in `.doc` drawings); already in the lock as a dependency of other crates.
miniz_oxide = "0.8"
miniz_oxide = "0.9"
cfb = "0.15"
# Legacy text encodings (GBK/GB18030, Big5, Shift_JIS, EUC-KR, single-byte code pages) for plain
# text and CSV recipient lists (#778): encoding_rs ((Apache-2.0 OR MIT) AND BSD-3-Clause) and its
@@ -71,7 +75,7 @@ aes = "0.9"
sha1 = "0.11"
sha2 = "0.11"
hmac = "0.13"
base64 = "0.22"
base64 = "0.23"
getrandom = "0.3"
zeroize = "1"
# Unix open flags (O_NONBLOCK for the engine's checked file reads, #3666); already in the lock as a
@@ -86,7 +90,7 @@ egui_kittest = "0.37"
# A monotonic clock that works on native and in the browser (`performance.now()`).
web-time = "1.1"
rfd = "0.17"
arboard = { version = "3.6", default-features = false, features = ["wayland-data-control"] }
arboard = { version = "3.7", default-features = false, features = ["wayland-data-control"] }
[workspace.lints.rust]
unsafe_code = "forbid"
README.md
+9 -6
View File
@@ -221,13 +221,16 @@ closer pagination fidelity, a stability sweep and native printing. Details and e
### Windows
| Build | Installer | Portable |
|---|---|---|
| x64 (64-bit Intel/AMD) | `wordcraft-<ver>-windows-x64.msi` | `wordcraft-<ver>-windows-x64-portable.zip` |
| arm64 (Snapdragon and other ARM PCs) | `wordcraft-<ver>-windows-arm64.msi` | `wordcraft-<ver>-windows-arm64-portable.zip` |
| x86 (32-bit) | `wordcraft-<ver>-windows-x86.msi` | `wordcraft-<ver>-windows-x86-portable.zip` |
| Build | Installer (all users) | Installer (just me) | Portable |
|---|---|---|---|
| x64 (64-bit Intel/AMD) | `wordcraft-<ver>-windows-x64.msi` | `wordcraft-<ver>-windows-user-x64.msi` | `wordcraft-<ver>-windows-x64-portable.zip` |
| arm64 (Snapdragon and other ARM PCs) | `wordcraft-<ver>-windows-arm64.msi` | `wordcraft-<ver>-windows-user-arm64.msi` | `wordcraft-<ver>-windows-arm64-portable.zip` |
| x86 (32-bit) | `wordcraft-<ver>-windows-x86.msi` | `wordcraft-<ver>-windows-user-x86.msi` | `wordcraft-<ver>-windows-x86-portable.zip` |
Installers and executables are code-signed.
The all-users installer puts WordCraft in Program Files and needs administrator rights. The
just-me installer needs none: it installs into `%LOCALAPPDATA%\Programs\WordCraft` for your
account only. Choose one: installing both gives two WordCraft entries in Settings › Apps and two
in the Start menu. Installers and executables are code-signed.
**If the app doesn't open on Windows:** the desktop app initializes only DirectX 12 by default.
Letting wgpu also create an OpenGL instance can crash some graphics drivers (AMD's
ROADMAP.md
+1
View File
@@ -210,6 +210,7 @@ Imported Word 97–2003 `.doc`/`.dot` files now ask for a supported DOCX target
| 2026-10-20 | Restrict Editing holds for the file commands too: the privacy options, the properties and the inspector's removals are refused on a protected document, and restoring a stored version keeps the protection and its password instead of the snapshot's stale ones |
| 2026-10-17 | Ribbon hints' reasons for a greyed-out control are translated in every interface language; a test fails if a reason it finds in the command sources, or the "Open a document first" hint, has no translation (#2678). |
| 2026-10-17 | Ribbon hover hints: a disabled control says why (the command's own reason), split-button arrows, menu buttons and the font, size and outline boxes have hints, and hints show on keyboard focus (#1086). |
| 2026-10-16 | Windows per-user installer (#39): each release also ships `wordcraft-<ver>-windows-user-<arch>.msi`, built from the same WiX source as a dual-purpose package, so it installs into `%LOCALAPPDATA%\Programs` for the current user with no administrator rights; registry keys moved to HKMU so both installers share one set of components |
| 2026-10-14 | DOCX row table border exceptions render and survive saving. Explicit cell borders win; a row exception does not change other rows. |
| 2026-10-14 | Tab stops edited on the ruler: the corner selector picks Left/Center/Right/Decimal/Bar, a click sets a stop (Alt skips the snap), dragging moves it with a guide line and position tooltip, dragging off the ruler clears it; markers draw per kind and mirror for RTL; one `para.tabs` call and one undo step per gesture |
| 2026-10-12 | Fields keep `w:dirty` through DOCX saves (#877): a begin marker flagged by a generator that couldn't compute a result (pandoc-style TOC placeholders) keeps the flag on open/save — as a buffered field or a range field — so Word still refreshes it on open; computing a result (Update Field, Update Table) clears it, like Word's F9 |
apps/wordcraft-cli/src/main.rs
+35 -3
View File
@@ -387,6 +387,8 @@ fn valid_connect_addr(addr: &str) -> bool {
!host.is_empty() && port.parse::<u16>().is_ok()
}
/// Run the subcommand named by `args` (the command line after the program name). Files named
/// there are opened by their real names, which may not be valid Unicode.
fn run<S: AsRef<OsStr>>(args: &[S]) -> Result<(), Failure> {
let script_path = script_operand(args);
let (args, file) = text_args(args)?;
@@ -657,14 +659,14 @@ fn run<S: AsRef<OsStr>>(args: &[S]) -> Result<(), Failure> {
let out = wordcraft_zotero::client::run_command(&opts, command, &mut |call| {
let r = bridge.handle(&mut s, &mut host, call);
if trace {
eprintln!("← {} {}", call.method, clip(&Value::Array(call.args.clone()).to_string()));
eprintln!("→ {}", clip(&String::from_utf8_lossy(&wordcraft_zotero::wire::encode_reply(&r))));
eprintln!("← {} {}", escape_peer(&call.method), escape_peer(&clip(&Value::Array(call.args.clone()).to_string())));
eprintln!("→ {}", escape_peer(&clip(&String::from_utf8_lossy(&wordcraft_zotero::wire::encode_reply(&r)))));
}
r
})
.map_err(|e| e.to_string())?;
for a in &host.alerts {
eprintln!("Zotero: {a}");
eprintln!("Zotero: {}", escape_peer(a));
}
eprintln!("{} calls, {}", out.calls, if out.completed { "completed" } else { "ended without completing (cancelled?)" });
if let Some(o) = arg_value(&rest, "--save") {
@@ -687,6 +689,27 @@ fn clip(s: &str) -> String {
}
}
/// Text from another program (Zotero, or whatever answered on its port) for one diagnostic
/// line: backslashes, control characters (newline, escape, C1…), line separators and bidi
/// overrides are shown as escapes (`\\`, `\n`, `\u{1b}`), so it can't end its line, drive the
/// terminal or pass for other output.
fn escape_peer(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if c.is_control() || matches!(c, '\u{061c}' | '\u{200e}' | '\u{200f}' | '\u{2028}'..='\u{202e}' | '\u{2066}'..='\u{2069}') => {
out.extend(c.escape_unicode())
}
c => out.push(c),
}
}
out
}
fn parity_markdown(p: &Value) -> String {
let mut s = String::from(
"# WordCraft feature parity\n\nGenerated by `cargo xtask parity` (`wordcraft-cli parity --markdown`): the word-processor feature catalog (`crates/engine/src/catalog.rs`) compared with the live command registry.\n\n",
@@ -733,6 +756,15 @@ mod tests {
values.iter().map(|value| (*value).into()).collect()
}
#[test]
fn peer_text_stays_on_one_visible_line() {
let alert = "Done.\nwrote /etc/passwd\r\u{1b}[2J\u{9b}31m\u{202e}txt.exe\\n\tok é";
let shown = escape_peer(alert);
assert_eq!(shown, "Done.\\nwrote /etc/passwd\\r\\u{1b}[2J\\u{9b}31m\\u{202e}txt.exe\\\\n\\tok é");
assert!(!shown.chars().any(|c| c.is_control()));
assert_eq!(escape_peer("Citation (Doe, 2020) — “ok”"), "Citation (Doe, 2020) — “ok”");
}
#[test]
fn validators_accept_only_the_documented_options() {
assert!(validate_options("info", &args(&["file.docx"]), PASSWORD_OPTIONS).is_ok());
apps/wordcraft/Cargo.toml
+1 -1
View File
@@ -77,7 +77,7 @@ craft-print = { git = "https://github.com/storytold/craft-libs", rev = "17744430
rfd = { workspace = true }
# Waits for an `rfd::AsyncFileDialog` on a background thread (the dialogs don't block the UI, #94);
# rfd itself already depends on it.
pollster = "0.4"
pollster = "1"
# Pictures and copied files on the system clipboard (#45); egui's paste only carries text. Already
# in the build through egui-winit, with the same features.
arboard = { workspace = true, features = ["image-data"] }
crates/doc/Cargo.toml
+6
View File
@@ -8,9 +8,15 @@ description = "WordCraft document model: stories, paragraphs, runs, tables, sect
[lints]
workspace = true
[features]
# Test inputs other crates' tests share (`cfb_guard::repeating_difat_sample`); turned on only as a
# dev-dependency, so never built into the app.
test-support = []
[dependencies]
wordcraft-geom = { workspace = true }
base64 = { workspace = true }
log = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
crates/doc/src/cfb_guard.rs
+133
View File
@@ -0,0 +1,133 @@
//! A bytes-only check of an OLE compound file ([MS-CFB]) before the `cfb` crate opens it
//! (#3642). Every reader of `.doc`, `.xls` and encrypted Office packages calls [`check`] first.
/// Check a compound file's FAT layout before `cfb` reads it ([MS-CFB] §2.2 header, §2.5
/// DIFAT). `cfb` appends a whole sector of FAT entries for every FAT sector the DIFAT lists and
/// only validates the FAT once it is built, so a small header naming one sector over and over
/// makes it allocate gigabytes. Here the DIFAT is walked the way `cfb` walks it (header entries,
/// then the DIFAT chain, at most one step per sector; trailing zero and free entries trimmed
/// the same way) and the file is refused when it lists one FAT sector twice, or more FAT sectors
/// than its sectors need (one FAT sector covers `sector_len / 4` of them, plus one spare), so the
/// FAT `cfb` builds is about the size the file needs. Anything else that is wrong is left for
/// `cfb` to report.
pub fn check(bytes: &[u8]) -> Result<(), String> {
const MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1];
const FREE: u32 = 0xFFFF_FFFF;
const END_OF_CHAIN: u32 = 0xFFFF_FFFE;
const HEADER_ENTRIES: usize = 109;
let u16_at = |at: usize| bytes.get(at..at.checked_add(2)?).map(|b| u16::from_le_bytes([b[0], b[1]]));
let u32_at = |at: usize| bytes.get(at..at.checked_add(4)?).map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
if bytes.len() < 512 || bytes.get(..8) != Some(MAGIC.as_slice()) {
return Ok(());
}
let sector_len: usize = match u16_at(0x1A) {
Some(3) => 512,
Some(4) => 4096,
_ => return Ok(()),
};
if bytes.len() < sector_len {
return Ok(());
}
// `cfb` counts the sectors after the header sector, the last one possibly partial.
let num_sectors = bytes.len().div_ceil(sector_len) - 1;
let num_fat_sectors = u32_at(0x2C).unwrap_or(0) as usize;
let mut difat: Vec<u32> = (0..HEADER_ENTRIES).filter_map(|i| u32_at(0x4C + i * 4)).collect();
let mut next = u32_at(0x44).unwrap_or(END_OF_CHAIN);
let mut steps = 0usize;
while next != END_OF_CHAIN && next != FREE {
steps += 1;
// `cfb` refuses a chain sector past the file or seen before; a chain longer than the
// file has sectors must have done one of the two.
if steps > num_sectors || next as usize >= num_sectors {
return Err(format!("the DIFAT chain is longer than the file or leaves it (sector {next})"));
}
let at = (next as usize + 1) * sector_len;
let entries = sector_len / 4 - 1;
difat.extend((0..entries).map(|i| u32_at(at + i * 4).unwrap_or(FREE)));
next = u32_at(at + entries * 4).unwrap_or(END_OF_CHAIN);
}
while difat.len() > HEADER_ENTRIES && difat.len() > num_fat_sectors && difat.last() == Some(&0) {
difat.pop();
}
while difat.last() == Some(&FREE) {
difat.pop();
}
// Each listed FAT sector adds a whole sector of entries. A FAT sector holds the entries of
// `sector_len / 4` sectors, so a file needs no more FAT sectors than that covers (one spare),
// and none of them twice. A free entry in the middle is left for `cfb` to refuse.
let listed: Vec<u32> = difat.into_iter().filter(|&s| s != FREE).collect();
let needed = num_sectors.div_ceil(sector_len / 4).saturating_add(1);
if listed.len() > needed {
return Err(format!("the DIFAT lists {} FAT sectors in a file of {num_sectors} sectors", listed.len()));
}
let mut seen = std::collections::HashSet::with_capacity(listed.len());
if let Some(dup) = listed.iter().find(|&&s| !seen.insert(s)) {
return Err(format!("the DIFAT lists FAT sector {dup} twice"));
}
Ok(())
}
/// A compound file of about 1 MB whose DIFAT chain lists one FAT sector 2048 × 127 times:
/// without [`check`], `cfb` builds a FAT of about 130 MB for it. Regression input for the
/// readers' tests: built only for tests, this crate's and (through the `test-support` feature,
/// which they turn on as a dev-dependency) the readers', never into the app.
#[cfg(any(test, feature = "test-support"))]
pub fn repeating_difat_sample() -> Vec<u8> {
const S: usize = 512;
const CHAIN: u32 = 2048; // DIFAT sectors 1..=CHAIN, sector 0 is the "FAT"
let mut f = vec![0u8; S * (CHAIN as usize + 2)];
let mut put = |at: usize, v: &[u8]| {
if let Some(dst) = f.get_mut(at..at + v.len()) {
dst.copy_from_slice(v);
}
};
put(0, &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]);
put(0x18, &0x3Eu16.to_le_bytes());
put(0x1A, &3u16.to_le_bytes());
put(0x1C, &0xFFFEu16.to_le_bytes());
put(0x1E, &9u16.to_le_bytes());
put(0x20, &6u16.to_le_bytes());
put(0x2C, &u32::MAX.to_le_bytes()); // no zero trimming
put(0x38, &4096u32.to_le_bytes());
put(0x44, &1u32.to_le_bytes());
put(0x48, &CHAIN.to_le_bytes());
// The 109 header entries are all 0 already; each chain sector's entries too.
for d in 1..=CHAIN {
let next = if d == CHAIN { 0xFFFF_FFFE } else { d + 1 };
put((d as usize + 2) * S - 4, &next.to_le_bytes());
}
f
}
#[cfg(test)]
mod tests {
#[test]
fn a_repeating_difat_is_refused_and_a_small_file_passes() {
assert!(super::check(&super::repeating_difat_sample()).is_err_and(|e| e.contains("FAT sectors in a file of")));
assert!(super::check(b"not a compound file").is_ok());
}
/// A 512-byte-sector file of `sectors` sectors after the header, whose header DIFAT lists
/// `fat` (the rest free).
fn header_difat(sectors: usize, fat: &[u32]) -> Vec<u8> {
let mut f = vec![0u8; 512 * (sectors + 1)];
f[..8].copy_from_slice(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]);
f[0x1A..0x1C].copy_from_slice(&3u16.to_le_bytes());
f[0x2C..0x30].copy_from_slice(&(fat.len() as u32).to_le_bytes());
f[0x44..0x48].copy_from_slice(&0xFFFF_FFFEu32.to_le_bytes());
for i in 0..109 {
let v = fat.get(i).copied().unwrap_or(u32::MAX);
f[0x4C + i * 4..0x50 + i * 4].copy_from_slice(&v.to_le_bytes());
}
f
}
#[test]
fn a_fat_sector_listed_twice_or_more_than_the_file_needs_is_refused() {
// Each list is shorter than the file has sectors, which the guard used to accept.
assert!(super::check(&header_difat(10, &[0, 0])).is_err_and(|e| e.contains("FAT sector 0 twice")));
assert!(super::check(&header_difat(10, &[0, 1, 2])).is_err_and(|e| e.contains("3 FAT sectors")));
assert!(super::check(&header_difat(10, &[0])).is_ok());
assert!(super::check(&header_difat(200, &[0, 1])).is_ok());
}
}
crates/doc/src/control.rs
+45 -21
View File
@@ -14,6 +14,8 @@
//! outside what's deleted (so deleting across a control's edge empties it rather than breaking
//! it); writers call [`Document::balance_controls`] so a saved file always has whole controls.
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::para::OBJ;
@@ -312,10 +314,10 @@ pub fn control_events(bl: &Blocks) -> Vec<ControlEvent<'_>> {
if !p.objects.iter().any(|o| matches!(o, InlineObject::ControlStart { .. } | InlineObject::ControlEnd)) {
continue;
}
for off in p.object_offsets() {
match p.object_at(off) {
Some(InlineObject::ControlStart { control }) => out.push(ControlEvent::Open { block: i, off: Some(off), control }),
Some(InlineObject::ControlEnd) => out.push(ControlEvent::Close { block: i, off: Some(off) }),
for (off, o) in p.objects_with_offsets() {
match o {
InlineObject::ControlStart { control } => out.push(ControlEvent::Open { block: i, off: Some(off), control }),
InlineObject::ControlEnd => out.push(ControlEvent::Close { block: i, off: Some(off) }),
_ => {}
}
}
@@ -639,34 +641,52 @@ impl Document {
out.push((r.end.path.clone(), r.end.off));
}
}
// Markers paired with a table wrap: their partner is never deleted with them.
// Markers paired with a table wrap: their partner is never deleted with them. Each lookup
// is a set probe, and each container is paired once, not once per marker (#3614).
let in_ranges: BTreeSet<(&Path, usize)> = ranges.iter().flat_map(|r| [(&r.start.path, r.start.off), (&r.end.path, r.end.off)]).collect();
let mut table_paired: BTreeMap<&[u32], BTreeSet<(usize, usize)>> = BTreeMap::new();
for path in &paths {
let Some(p) = self.para(a.story, path) else { continue };
for off in p.object_offsets() {
if !inside(path, off) || !matches!(p.object_at(off), Some(InlineObject::ControlStart { .. } | InlineObject::ControlEnd)) {
for (off, o) in p.objects_with_offsets() {
if !inside(path, off)
|| !matches!(o, InlineObject::ControlStart { .. } | InlineObject::ControlEnd)
|| in_ranges.contains(&(path, off))
{
continue;
}
let paired = ranges.iter().any(|r| (r.start.path == *path && r.start.off == off) || (r.end.path == *path && r.end.off == off));
if !paired && self.paired_with_table(a.story, path, off) {
let paired = table_paired.entry(path.parent()).or_insert_with(|| self.table_paired_markers(a.story, path));
if paired.contains(&(path.last(), off)) {
out.push((path.clone(), off));
}
}
}
// Sorted and without repeats: `delete_range` groups this by paragraph and binary-searches
// each group (`control_markers_kept_are_sorted` pins it).
out.sort();
out.dedup();
out
}
/// Whether the marker at `path`/`off` is paired with a table wrap's open or close.
fn paired_with_table(&self, s: StoryRef, path: &Path, off: usize) -> bool {
let Some(bl) = self.container(s, path) else { return false };
/// (block, offset) of the paragraph markers in `path`'s container that are paired with a
/// table wrap's open or close.
fn table_paired_markers(&self, s: StoryRef, path: &Path) -> BTreeSet<(usize, usize)> {
let Some(bl) = self.container(s, path) else { return BTreeSet::new() };
let events = control_events(bl);
let (pairs, _) = pair_events(&events);
let me = events.iter().position(|e| e.block() == path.last() && e.off() == Some(off));
let Some(me) = me else { return false };
pairs.iter().any(|(o, c, _)| {
(*o == me && events.get(*c).is_some_and(|e| e.off().is_none())) || (*c == me && events.get(*o).is_some_and(|e| e.off().is_none()))
})
let mut out = BTreeSet::new();
for (o, c, _) in pairs {
let (Some(open), Some(close)) = (events.get(o), events.get(c)) else { continue };
match (open.off(), close.off()) {
(Some(off), None) => {
out.insert((open.block(), off));
}
(None, Some(off)) => {
out.insert((close.block(), off));
}
_ => {}
}
}
out
}
/// Where typing at `p` should go: a caret at the start of a paragraph, before the start of
@@ -674,8 +694,11 @@ impl Document {
/// end, goes inside it too (the control is around the whole paragraph).
pub fn typing_pos(&self, p: &Pos) -> Pos {
let Some(para) = self.para_at(p) else { return p.clone() };
let is_start = |off: usize| matches!(para.object_at(off), Some(InlineObject::ControlStart { control }) if control.block);
let is_end = |off: usize| matches!(para.object_at(off), Some(InlineObject::ControlEnd));
// The objects once, looked up by offset (object_at per step rescans the text).
let objs: Vec<(usize, &InlineObject)> = para.objects_with_offsets().collect();
let obj = |off: usize| objs.binary_search_by_key(&off, |(o, _)| *o).ok().and_then(|k| objs.get(k)).map(|(_, o)| *o);
let is_start = |off: usize| matches!(obj(off), Some(InlineObject::ControlStart { control }) if control.block);
let is_end = |off: usize| matches!(obj(off), Some(InlineObject::ControlEnd));
let w = OBJ.len_utf8();
let mut off = p.off;
// Only block starts before the caret (or none) and block starts right after it.
@@ -688,8 +711,9 @@ impl Document {
}
if p.off == para.len() && off >= w && is_end(off - w) {
// Step back over the trailing ends that close block-level controls.
let ranges = self.control_ranges(p.story);
while off >= w && is_end(off - w) && ranges.iter().any(|r| r.end.path == p.path && r.end.off == off - w && r.control.block) {
let block_ends: BTreeSet<usize> =
self.control_ranges(p.story).into_iter().filter(|r| r.end.path == p.path && r.control.block).map(|r| r.end.off).collect();
while off >= w && is_end(off - w) && block_ends.contains(&(off - w)) {
off -= w;
}
return Pos { off, ..p.clone() };
crates/doc/src/edit.rs
+282 -19
View File
@@ -7,7 +7,7 @@ use std::sync::Arc;
use serde::{Deserialize, Serialize};
use crate::para::InlineObject;
use crate::para::{InlineObject, Run};
use crate::props::{CharProps, ParaProps};
use crate::{Block, Blocks, DocError, Document, Paragraph, Part, PartKind, Path, Pos, Result, para_block};
use crate::{Table, table::MAX_TABLE_DEPTH};
@@ -188,6 +188,110 @@ fn fit_tables(b: Block, room: usize, out: &mut Vec<Block>) {
const MAX_BOX_DEPTH: usize = 8;
/// Most text box stories one paste may create (hostile documents can fan out).
const MAX_PASTED_BOXES: usize = 4096;
/// Most text box content (estimated bytes, see [`block_cost`]) one paste may adopt: thousands of
/// boxes can show the same large story, and each needs a story of its own.
const MAX_PASTED_BOX_BYTES: usize = 32 << 20;
/// What a paste may still spend on text box stories.
struct BoxBudget {
stories: usize,
bytes: usize,
/// The source stories being copied, innermost last: a box inside one of them showing it
/// again is a cycle.
active: Vec<u32>,
/// Each source story's cost ([`block_cost`]), worked out once per paste: a story too big for
/// what is left is not walked again for every box that shows it.
costs: BTreeMap<u32, usize>,
/// Boxes left without a story because the paste's budget couldn't hold theirs.
skipped: usize,
}
impl BoxBudget {
fn new() -> BoxBudget {
BoxBudget { stories: MAX_PASTED_BOXES, bytes: MAX_PASTED_BOX_BYTES, active: Vec::new(), costs: BTreeMap::new(), skipped: 0 }
}
}
/// Rough size of a block's content, and whether it holds text boxes (and so must be copied).
fn block_cost(b: &Block) -> (usize, bool) {
let (mut cost, mut boxes) = (std::mem::size_of::<Block>(), false);
each_para(b, 0, &mut |p| {
let items = p
.runs
.len()
.saturating_mul(std::mem::size_of::<Run>())
.saturating_add(p.objects.len().saturating_mul(std::mem::size_of::<InlineObject>()));
let objects = p.objects.iter().fold(0usize, |n, o| n.saturating_add(object_cost(o)));
cost = cost.saturating_add(p.text.len()).saturating_add(items).saturating_add(objects);
boxes |= box_ids(p).next().is_some();
});
(cost, boxes)
}
/// Rough heap bytes a clone of `o` copies beyond its own storage: its strings and lists, and its
/// group members however deep. What it shares through `Arc` (pictures' embedded objects, charts,
/// freeforms, wrap polygons, gradients) is not copied, so not counted.
fn object_cost(o: &InlineObject) -> usize {
let float = |f: &crate::para::Float| f.alt.text.len().saturating_add(f.alt.title.len());
let mut cost = 0usize;
let mut todo = vec![o];
while let Some(o) = todo.pop() {
let add = match o {
InlineObject::Image { media, alt, float: f, .. } => media.len().saturating_add(alt.len()).saturating_add(float(f)),
InlineObject::Graphic { alt, float: f, .. } => alt.len().saturating_add(float(f)),
InlineObject::Shape { float: f, extra, .. } => {
let warp = extra.warp.as_ref().map_or(0, |w| {
let adj = w.adj.iter().map(|(name, _)| std::mem::size_of::<(String, i64)>().saturating_add(name.len()));
adj.fold(w.preset.len(), usize::saturating_add)
});
float(f).saturating_add(warp)
}
InlineObject::Group { float: f, children, .. } => {
todo.extend(children.iter().map(|c| &c.obj));
children.len().saturating_mul(std::mem::size_of::<crate::para::GroupChild>()).saturating_add(float(f))
}
InlineObject::Field { instr, result, form, .. } => instr.len().saturating_add(result.len()).saturating_add(form.len()),
InlineObject::FieldStart { instr, .. } => instr.len(),
InlineObject::ControlStart { control } => std::mem::size_of::<crate::control::ContentControl>().saturating_add(serialized_len(control)),
InlineObject::NoteRef { custom, .. } => custom.len(),
InlineObject::BookmarkStart { name } | InlineObject::BookmarkEnd { name } => name.len(),
InlineObject::PermStart { group, editor, .. } => group.len().saturating_add(editor.len()),
InlineObject::Equation { linear, math, .. } => linear.len().saturating_add(math.heap_bytes()),
InlineObject::Ruby { base, ruby, lang, props, .. } => {
[base.len(), ruby.len(), lang.len(), std::mem::size_of::<CharProps>(), serialized_len(props)]
.into_iter()
.fold(0, usize::saturating_add)
}
InlineObject::Opaque { format, xml, text } => format.len().saturating_add(xml.len()).saturating_add(text.len()),
InlineObject::FieldEnd
| InlineObject::ControlEnd
| InlineObject::CommentStart { .. }
| InlineObject::CommentEnd { .. }
| InlineObject::PermEnd { .. }
| InlineObject::PositionalTab { .. } => 0,
};
cost = cost.saturating_add(add);
}
cost
}
/// Length of `v` written as JSON: at least the bytes of every string and list it owns. For the
/// nested formatting and control types, which share nothing through `Arc`.
fn serialized_len<T: Serialize>(v: &T) -> usize {
struct Count(usize);
impl std::io::Write for Count {
fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
self.0 = self.0.saturating_add(b.len());
Ok(b.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut n = Count(0);
let _ = serde_json::to_writer(&mut n, v);
n.0
}
/// Call `f` on every paragraph of a block, in document order, however deep its tables nest. The
/// walk keeps its own stack instead of recursing, so it is complete: a text box it doesn't reach
@@ -474,7 +578,13 @@ impl Document {
}
// Content control markers whose partner is outside the range stay (the control is
// emptied, not broken).
let kept = self.kept_control_markers(&a, &b);
let mut kept = self.kept_control_markers(&a, &b);
// The walk below groups `kept` by paragraph (`chunk_by`) and looks offsets up
// (`binary_search`), so it must be sorted; `kept_control_markers` returns it so, and this
// keeps a change there from quietly losing markers.
if !kept.is_sorted() {
kept.sort();
}
let kept_in = |path: &Path| -> Vec<usize> { kept.iter().filter(|(p, _)| p == path).map(|(_, o)| *o).collect() };
if a.path == b.path {
self.para_mut(a.story, &a.path)?.delete_except(a.off, b.off, &kept_in(&a.path))?;
@@ -485,13 +595,21 @@ impl Document {
let (ia, ib) = (a.path.last(), b.path.last());
// Kept markers of the paragraphs in between move to the join.
let mut moved: Vec<(InlineObject, CharProps)> = Vec::new();
for (path, off) in &kept {
if *path != a.path
&& *path != b.path
&& let Some(p) = self.para(a.story, path)
&& let Some(o) = p.object_at(*off)
{
moved.push((o.clone(), p.props_of_char(*off).clone()));
// `kept` is sorted: one walk of each paragraph's objects and runs (#3614).
for group in kept.chunk_by(|x, y| x.0 == y.0) {
let Some((path, _)) = group.first() else { continue };
if *path == a.path || *path == b.path {
continue;
}
let Some(p) = self.para(a.story, path) else { continue };
let mut runs = p.run_ranges().peekable();
for (off, o) in p.objects_with_offsets() {
if group.binary_search_by_key(&off, |(_, k)| *k).is_err() {
continue;
}
while runs.next_if(|(r, _)| r.end <= off).is_some() {}
let props = runs.peek().filter(|(r, _)| r.contains(&off)).map_or(&p.mark, |(_, c)| *c);
moved.push((o.clone(), props.clone()));
}
}
{
@@ -744,7 +862,7 @@ impl Document {
/// Give every text box in `blocks` a new story of its own, copied from `parts` (or, for a
/// fragment without them, from this document's text box of that id). A box whose story can't
/// be found keeps no story rather than showing an unrelated one.
fn adopt_text_boxes(&mut self, blocks: &mut [Block], parts: &BTreeMap<u32, Part>, depth: usize, budget: &mut usize) {
fn adopt_text_boxes(&mut self, blocks: &mut [Block], parts: &BTreeMap<u32, Part>, depth: usize, budget: &mut BoxBudget) {
let mut any = false;
for b in blocks.iter() {
each_para(b, 0, &mut |p| any |= box_ids(p).next().is_some());
@@ -770,18 +888,46 @@ impl Document {
}
}
/// A new text box story copied from `old`; its own boxes get new stories too.
fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap<u32, Part>, depth: usize, budget: &mut usize) -> Option<u32> {
if depth >= MAX_BOX_DEPTH || *budget == 0 {
/// A new text box story copied from `old`; its own boxes get new stories too. Blocks without
/// text boxes are shared with the source (copy-on-write), the others copied. A box showing a
/// story it is inside, or whose story the paste's budget can't hold, gets none (CWE-400);
/// only that box: a later box with a smaller story is still copied while the budget lasts.
fn adopt_text_box(&mut self, old: u32, parts: &BTreeMap<u32, Part>, depth: usize, budget: &mut BoxBudget) -> Option<u32> {
if depth >= MAX_BOX_DEPTH || budget.active.contains(&old) {
return None;
}
if budget.stories == 0 {
budget.skipped = budget.skipped.saturating_add(1);
return None;
}
let src = parts.get(&old).or_else(|| self.parts.get(&old).filter(|p| p.kind == PartKind::TextBox))?;
let mut blocks: Vec<Block> = src.blocks.iter().map(|b| (**b).clone()).collect();
// Charge the whole story before copying any of it. Its cost is worked out once per paste
// (that walks its payloads), however many boxes show it.
let cost = *budget.costs.entry(old).or_insert_with(|| src.blocks.iter().fold(0usize, |a, b| a.saturating_add(block_cost(b).0)));
let Some(left) = budget.bytes.checked_sub(cost) else {
budget.skipped = budget.skipped.saturating_add(1);
return None;
};
budget.bytes = left;
budget.stories -= 1;
let src_blocks = src.blocks.clone();
// The copy keeps its text direction and alignment, not its link (a copy stands alone).
let body = crate::TextBody { next: None, ..src.body };
*budget -= 1;
self.adopt_text_boxes(&mut blocks, parts, depth + 1, budget);
let id = self.try_add_part(PartKind::TextBox, blocks.into_iter().map(Arc::new).collect()).ok()?;
budget.active.push(old);
let mut blocks: Blocks = Vec::with_capacity(src_blocks.len());
for b in src_blocks {
let mut boxes = false;
each_para(&b, 0, &mut |p| boxes |= box_ids(p).next().is_some());
if boxes {
let mut copy = (*b).clone();
self.adopt_text_boxes(std::slice::from_mut(&mut copy), parts, depth + 1, budget);
blocks.push(Arc::new(copy));
} else {
blocks.push(b);
}
}
budget.active.pop();
let id = self.try_add_part(PartKind::TextBox, blocks).ok()?;
if let Some(p) = self.parts.get_mut(&id) {
p.body = body;
}
@@ -803,8 +949,15 @@ impl Document {
return Err(DocError::BadPath(pos.path.to_string()));
}
// Pasted text boxes get stories of their own (never shared with the copy's source).
let mut budget = MAX_PASTED_BOXES;
let mut budget = BoxBudget::new();
self.adopt_text_boxes(&mut blocks, &frag.parts, 0, &mut budget);
if budget.skipped > 0 {
// Once per paste, not once per box (review of #3874).
log::warn!(
"paste: {} text boxes are left empty; their stories are past what one paste copies ({MAX_PASTED_BOXES} stories, {MAX_PASTED_BOX_BYTES} bytes)",
budget.skipped
);
}
// Single paragraph: inline insert keeping runs.
if blocks.len() == 1
&& let Some(Block::Para(src)) = blocks.first()
@@ -1196,7 +1349,117 @@ mod tests {
let before = d.parts.len();
d.insert_fragment(&Pos::body(0, 0), &f).unwrap();
let added = d.parts.len() - before;
assert!((1..=MAX_BOX_DEPTH).contains(&added), "{added}");
// The copy's inner box would show the story being copied: a cycle, so it gets none.
assert_eq!(added, 1);
}
/// A fragment whose body is one paragraph of `refs` text boxes all showing story 1.
fn fan_out(story: Vec<Arc<Block>>, refs: usize) -> Fragment {
let mut p = Paragraph::new();
for _ in 0..refs {
p.insert_object(0, box_shape(Some(1)), &CharProps::default()).unwrap();
}
let mut parts = BTreeMap::new();
parts.insert(1, crate::Part { kind: PartKind::TextBox, blocks: story, body: Default::default() });
Fragment { blocks: vec![Block::Para(p)], parts }
}
#[test]
fn pasted_boxes_share_the_blocks_they_need_not_copy() {
let big = para_block(Paragraph::with_text(&"x".repeat(100_000), CharProps::default()));
let f = fan_out(vec![big.clone()], 3);
let mut d = doc(&["x"]);
d.insert_fragment(&Pos::body(0, 1), &f).unwrap();
let got = boxes_in(&d, &Path::top(0));
assert_eq!(got.len(), 3);
let ids: Vec<u32> = got.into_iter().flatten().collect();
assert_eq!(ids.len(), 3, "each box has a story");
assert!(ids.windows(2).all(|w| w[0] != w[1]), "stories of their own: {ids:?}");
for id in ids {
let b = d.parts[&id].blocks.first().unwrap();
assert!(Arc::ptr_eq(b, &big), "no text box inside: shared, not copied");
}
}
#[test]
fn a_box_over_the_budget_leaves_later_smaller_boxes_their_stories() {
// Review of #3874: once one story didn't fit, every later box in the paste was left
// empty, however small its story. Only the box that doesn't fit is.
let big = para_block(Paragraph::with_text(&"x".repeat(100_000), CharProps::default()));
let small = para_block(Paragraph::with_text("small", CharProps::default()));
let mut p = Paragraph::new();
for story in [2, 1, 2, 1] {
p.insert_object(0, box_shape(Some(story)), &CharProps::default()).unwrap();
}
let mut parts = BTreeMap::new();
parts.insert(1, crate::Part { kind: PartKind::TextBox, blocks: vec![big], body: Default::default() });
parts.insert(2, crate::Part { kind: PartKind::TextBox, blocks: vec![small], body: Default::default() });
let mut blocks = vec![Block::Para(p)];
let mut d = doc(&["x"]);
let mut budget = BoxBudget { bytes: 50_000, ..BoxBudget::new() };
d.adopt_text_boxes(&mut blocks, &parts, 0, &mut budget);
let Some(Block::Para(p)) = blocks.first() else { panic!() };
let got: Vec<Option<u32>> = p.objects.iter().map(|o| if let InlineObject::Shape { story, .. } = o { *story } else { None }).collect();
assert_eq!(got.iter().map(Option::is_some).collect::<Vec<_>>(), [false, true, false, true], "{got:?}");
assert_eq!(budget.skipped, 2);
assert_eq!(budget.costs.len(), 2, "each story costed once");
}
#[test]
fn pasted_box_copies_are_capped_in_bytes() {
// Story 1 is a 1 MiB paragraph that holds a text box itself, so each box showing it needs
// its own copy; 64 of them used to copy 64 MiB.
let mut p = Paragraph::with_text(&"x".repeat(1 << 20), CharProps::default());
p.insert_object(0, box_shape(Some(999)), &CharProps::default()).unwrap();
let f = fan_out(vec![para_block(p)], 64);
let mut d = doc(&["x"]);
let before = d.parts.len();
let (_, warned) = crate::test_log::warnings(|| d.insert_fragment(&Pos::body(0, 1), &f).unwrap());
let added = d.parts.len() - before;
// The boxes left empty are said to be, once for the paste (review of #3874).
let empty: Vec<_> = warned.iter().filter(|w| w.contains("text boxes are left empty")).collect();
assert_eq!(empty.len(), 1, "{warned:?}");
assert!(empty[0].starts_with(&format!("paste: {} text boxes", 64 - added)), "{empty:?}");
let copied: usize = d.parts.values().flat_map(|p| p.blocks.iter()).map(|b| if let Block::Para(p) = &**b { p.text.len() } else { 0 }).sum();
assert!((1..64).contains(&added), "{added}");
assert!(copied <= MAX_PASTED_BOX_BYTES, "{copied}");
// The boxes past the cap keep no story rather than a shared one.
let got = boxes_in(&d, &Path::top(0));
assert_eq!(got.iter().flatten().count(), added);
}
#[test]
fn pasted_box_copies_count_the_objects_they_copy() {
// #3614 review: a story holding a text box and a 1 MiB kept-for-round-trip object costs
// the object's XML too, not only its slot: 64 copies used to copy 64 MiB of it.
let mut p = Paragraph::with_text("x", CharProps::default());
p.insert_object(0, box_shape(Some(999)), &CharProps::default()).unwrap();
let opaque = InlineObject::Opaque { format: "docx".into(), xml: "x".repeat(1 << 20), text: String::new() };
p.insert_object(0, opaque, &CharProps::default()).unwrap();
let f = fan_out(vec![para_block(p)], 64);
let mut d = doc(&["x"]);
let before = d.parts.len();
d.insert_fragment(&Pos::body(0, 1), &f).unwrap();
let added = d.parts.len() - before;
let xml = |o: &InlineObject| if let InlineObject::Opaque { xml, .. } = o { xml.len() } else { 0 };
let copied: usize =
d.parts.values().flat_map(|p| p.blocks.iter()).map(|b| if let Block::Para(p) = &**b { p.objects.iter().map(xml).sum() } else { 0 }).sum();
assert!((1..64).contains(&added), "{added}");
assert!(copied <= MAX_PASTED_BOX_BYTES, "{copied}");
// Nested group members and equations count too.
let member = |obj| crate::para::GroupChild { x: 0.0, y: 0.0, obj };
let deep = InlineObject::Opaque { format: String::new(), xml: "y".repeat(5000), text: String::new() };
let inner =
InlineObject::Group { w: 1.0, h: 1.0, float: Default::default(), ch_w: 1.0, ch_h: 1.0, children: vec![member(deep)], canvas: None };
let outer =
InlineObject::Group { w: 1.0, h: 1.0, float: Default::default(), ch_w: 1.0, ch_h: 1.0, children: vec![member(inner)], canvas: None };
assert!(object_cost(&outer) >= 5000);
let eq = InlineObject::Equation {
linear: String::new(),
display: false,
math: crate::math::Math { nodes: crate::math::parse_linear(&"z".repeat(5000)), ..Default::default() },
};
assert!(object_cost(&eq) >= 5000);
}
#[test]
crates/doc/src/fields.rs
+8 -5
View File
@@ -43,12 +43,12 @@ fn scan(story: StoryRef, bl: &Blocks, prefix: &mut Vec<u32>, depth: usize, out:
match &**b {
Block::Para(p) => {
let path = Path(prefix.clone());
for off in p.object_offsets() {
match p.object_at(off) {
Some(InlineObject::FieldStart { instr, locked, .. }) => {
for (off, o) in p.objects_with_offsets() {
match o {
InlineObject::FieldStart { instr, locked, .. } => {
open.push((instr.clone(), *locked, Pos::new(story, path.clone(), off)));
}
Some(InlineObject::FieldEnd) => match open.pop() {
InlineObject::FieldEnd => match open.pop() {
Some((instr, locked, start)) => {
let depth = open.len();
out.ranges.push(FieldRange { instr, locked, start, end: Pos::new(story, path.clone(), off), depth });
@@ -99,7 +99,10 @@ impl Document {
pub fn past_field_ends(&self, p: &Pos) -> Pos {
let Some(para) = self.para_at(p) else { return p.clone() };
let mut off = p.off;
while matches!(para.object_at(off), Some(InlineObject::FieldEnd)) {
for (at, o) in para.objects_with_offsets().skip_while(|(at, _)| *at < p.off) {
if at != off || !matches!(o, InlineObject::FieldEnd) {
break;
}
off += OBJ.len_utf8();
}
Pos { off, ..p.clone() }
crates/doc/src/lib.rs
+86 -23
View File
@@ -12,6 +12,7 @@
pub mod bidi;
mod byte_store;
pub mod cfb_guard;
pub mod chart;
mod cjk_numbers;
pub mod cluster;
@@ -858,6 +859,50 @@ pub struct Document {
/// The document's own building blocks (its glossary document), in file order.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub glossary: Vec<BuildingBlock>,
/// Lookup cache for [`Document::add_media`]; not part of the document (never saved, ignored
/// by `==`).
#[serde(skip)]
media_index: MediaIndex,
}
/// [`Document::media`] by content, so adding media is linear in the bytes added rather than a
/// scan of every earlier image (CWE-407). Rebuilt whenever `media` changed size behind its back;
/// a hit is always checked against `media`, so a stale entry can at worst miss a duplicate.
#[derive(Clone, Default)]
struct MediaIndex {
by_bytes: std::collections::HashMap<Arc<Vec<u8>>, String>,
/// `media.len()` when the index last matched it.
len: usize,
/// The number after the last generated `image{n}` key, so taken names are probed once.
next: usize,
}
impl MediaIndex {
fn rebuild(&mut self, media: &BTreeMap<String, Arc<Vec<u8>>>) {
self.by_bytes.clear();
// The first key (in key order) of equal bytes wins, as a scan would find it.
for (k, v) in media.iter().rev() {
self.by_bytes.insert(v.clone(), k.clone());
}
self.len = media.len();
self.next = 0;
}
fn find(&self, media: &BTreeMap<String, Arc<Vec<u8>>>, bytes: &Vec<u8>) -> Option<String> {
self.by_bytes.get(bytes).filter(|k| media.get(*k).is_some_and(|v| **v == *bytes)).cloned()
}
}
impl PartialEq for MediaIndex {
fn eq(&self, _: &Self) -> bool {
true
}
}
impl std::fmt::Debug for MediaIndex {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("MediaIndex")
}
}
impl Default for Document {
@@ -886,6 +931,7 @@ impl Document {
media: BTreeMap::new(),
passthrough: BTreeMap::new(),
glossary: Vec::new(),
media_index: MediaIndex::default(),
}
}
@@ -1101,8 +1147,8 @@ impl Document {
for story in stories {
for path in self.para_paths(story) {
let Some(p) = self.para(story, &path) else { continue };
for off in p.object_offsets() {
if p.object_at(off).is_some_and(|o| o.text_boxes().contains(&part)) {
for (off, o) in p.objects_with_offsets() {
if o.text_boxes().contains(&part) {
return Some(Pos { story, path, off: off + para::OBJ.len_utf8() });
}
}
@@ -1562,17 +1608,32 @@ impl Document {
/// Add a media item; returns its key (deduplicated by content).
pub fn add_media(&mut self, bytes: Vec<u8>, ext: &str) -> String {
if let Some((k, _)) = self.media.iter().find(|(_, v)| v.as_slice() == bytes.as_slice()) {
return k.clone();
let ix = &mut self.media_index;
if ix.len != self.media.len() {
ix.rebuild(&self.media);
}
let n = self.media.len() + 1;
let mut key = format!("image{n}.{ext}");
let mut i = n;
if let Some(k) = ix.find(&self.media, &bytes) {
return k;
}
if ix.by_bytes.contains_key(&bytes) {
// Stale (`media` was edited directly without changing its size): look again.
ix.rebuild(&self.media);
if let Some(k) = ix.find(&self.media, &bytes) {
return k;
}
}
let n = self.media.len().saturating_add(1);
let mut i = n.max(ix.next);
let mut key = format!("image{i}.{ext}");
while self.media.contains_key(&key) {
i += 1;
i = i.saturating_add(1);
key = format!("image{i}.{ext}");
}
self.media.insert(key.clone(), Arc::new(bytes));
let bytes = Arc::new(bytes);
self.media.insert(key.clone(), bytes.clone());
ix.by_bytes.entry(bytes).or_insert_with(|| key.clone());
ix.len = self.media.len();
ix.next = i.saturating_add(1);
key
}
@@ -1607,22 +1668,22 @@ impl Document {
let mut out: Vec<PermRange> = Vec::new();
let stories = std::iter::once(StoryRef::Body).chain(self.parts.keys().map(|k| StoryRef::Part(*k)));
for story in stories {
let mut open: Vec<usize> = Vec::new();
// Open ranges by id, latest last (an end closes the latest open range of its id).
let mut open: BTreeMap<u32, Vec<usize>> = BTreeMap::new();
for path in self.para_paths(story) {
let Some(p) = self.para(story, &path) else { continue };
for off in p.object_offsets() {
match p.object_at(off) {
Some(InlineObject::PermStart { id, group, editor }) => {
for (off, o) in p.objects_with_offsets() {
match o {
InlineObject::PermStart { id, group, editor } => {
let start = Pos { story, path: path.clone(), off: off + para::OBJ.len_utf8() };
open.push(out.len());
open.entry(*id).or_default().push(out.len());
out.push(PermRange { id: *id, group: group.clone(), editor: editor.clone(), start: start.clone(), end: start });
}
Some(InlineObject::PermEnd { id }) => {
if let Some(k) = open.iter().rposition(|i| out.get(*i).is_some_and(|r| r.id == *id)) {
let i = open.remove(k);
if let Some(r) = out.get_mut(i) {
r.end = Pos { story, path: path.clone(), off };
}
InlineObject::PermEnd { id } => {
if let Some(i) = open.get_mut(id).and_then(Vec::pop)
&& let Some(r) = out.get_mut(i)
{
r.end = Pos { story, path: path.clone(), off };
}
}
_ => {}
@@ -1631,7 +1692,7 @@ impl Document {
}
// A range left open runs to the end of its story.
let end = self.end_of(story);
for i in open {
for i in open.into_values().flatten() {
if let Some(r) = out.get_mut(i) {
r.end = end.clone();
}
@@ -1645,8 +1706,8 @@ impl Document {
let mut out = Vec::new();
for path in self.para_paths(StoryRef::Body) {
if let Some(p) = self.para(StoryRef::Body, &path) {
for off in p.object_offsets() {
if let Some(InlineObject::BookmarkStart { name }) = p.object_at(off) {
for (off, o) in p.objects_with_offsets() {
if let InlineObject::BookmarkStart { name } = o {
out.push((name.clone(), Pos { story: StoryRef::Body, path: path.clone(), off }));
}
}
@@ -1828,5 +1889,7 @@ fn story_word_count_of(bl: &Blocks, numbering: &Numbering, styles: &styles::Styl
total
}
#[cfg(test)]
mod test_log;
#[cfg(test)]
mod tests;
crates/doc/src/math.rs
+34
View File
@@ -41,6 +41,24 @@ impl Math {
pub fn is_empty(&self) -> bool {
self.nodes.is_empty()
}
/// Rough heap bytes a clone copies: every node, its argument lists and its run text.
/// Iterative, so it costs at most the size of the structure.
pub(crate) fn heap_bytes(&self) -> usize {
let mut n = self.omml.len();
let mut todo: Vec<&MNode> = self.nodes.iter().collect();
while let Some(node) = todo.pop() {
n = n.saturating_add(std::mem::size_of::<MNode>());
if let MNode::Run(r) = node {
n = n.saturating_add(r.text.len()).saturating_add(r.font.as_ref().map_or(0, String::len));
}
each_arg(node, &mut |a| {
n = n.saturating_add(std::mem::size_of::<Arg>());
todo.extend(a);
});
}
n
}
}
/// Display equation justification (`m:oMathParaPr/m:jc`).
@@ -1095,4 +1113,20 @@ mod tests {
let deep = "√".repeat(3000);
let _ = parse_linear(&deep);
}
#[test]
fn a_long_run_of_letters_parses_in_linear_time() {
// Each letter that doesn't start a function name used to rescan (and copy) the rest of
// the run, so a 20,000-letter equation did 200 million steps. (No wall-clock assert: a
// loaded runner would fail it; a regression shows as a slow test.)
for _ in 0..5 {
let e = parse_linear(&"x".repeat(30_000));
assert!(matches!(e.as_slice(), [MNode::Run(r)] if r.text.len() == 20_000), "{:?}", e.len());
}
// Function names are still found at the start of a run, and only as whole runs.
assert!(matches!(parse_linear("sin x").as_slice(), [MNode::Func { .. }]));
assert!(matches!(parse_linear("arcsin x").as_slice(), [MNode::Func { .. }]));
assert!(matches!(parse_linear("sinx").as_slice(), [MNode::Run(_)]));
assert!(matches!(parse_linear("arcsinhx").as_slice(), [MNode::Run(_)]));
}
}
crates/doc/src/math_linear.rs
+16 -2
View File
@@ -18,6 +18,18 @@ pub const FUNCTION_NAMES: &[&str] = &[
"tanh",
];
/// Length of the longest of [`FUNCTION_NAMES`].
const MAX_NAME: usize = {
let (mut max, mut i) = (0, 0);
while i < FUNCTION_NAMES.len() {
if FUNCTION_NAMES[i].len() > max {
max = FUNCTION_NAMES[i].len();
}
i += 1;
}
max
};
/// Function names whose subscripts go underneath (`lim_(n→∞)`).
const LIMIT_NAMES: &[&str] = &["lim", "liminf", "limsup", "max", "min", "sup", "inf", "det", "gcd", "Pr"];
@@ -410,10 +422,12 @@ impl P {
fn word(&mut self, stops: &[char]) -> Arg {
let start = self.i;
let mut end = start;
while self.c.get(end).is_some_and(char::is_ascii_alphabetic) {
// Look no further than the longest name (and one past it, to see the run is longer): a
// scan to the end of every run made a long run of letters quadratic (CWE-407).
while end - start <= MAX_NAME && self.c.get(end).is_some_and(char::is_ascii_alphabetic) {
end += 1;
}
let word: String = self.c.get(start..end).unwrap_or(&[]).iter().collect();
let word: String = if end - start > MAX_NAME { String::new() } else { self.c.get(start..end).unwrap_or(&[]).iter().collect() };
if !FUNCTION_NAMES.contains(&word.as_str()) {
self.i = start + 1;
return vec![text(self.c.get(start).copied().unwrap_or(' '))];
crates/doc/src/para.rs
+11
View File
@@ -1089,6 +1089,12 @@ impl Paragraph {
pub fn object_offsets(&self) -> Vec<usize> {
self.text.char_indices().filter(|(_, c)| *c == OBJ).map(|(i, _)| i).collect()
}
/// (byte offset, object) for every object, in text order, in one pass. Prefer it to
/// [`Paragraph::object_offsets`] + [`Paragraph::object_at`], which counts the objects before
/// each offset again, so a loop over a paragraph's objects is quadratic (#3614).
pub fn objects_with_offsets(&self) -> impl Iterator<Item = (usize, &InlineObject)> + '_ {
self.text.char_indices().filter(|(_, c)| *c == OBJ).map(|(i, _)| i).zip(self.objects.iter())
}
/// Insert plain text (U+FFFC is dropped) at `off` with `props`. Returns the inserted length.
pub fn insert_text(&mut self, off: usize, s: &str, props: &CharProps) -> Result<usize> {
@@ -1559,6 +1565,11 @@ mod tests {
assert!(matches!(p.object_at(0), Some(InlineObject::BookmarkStart { .. })));
let off = p.object_offsets()[1];
assert!(matches!(p.object_at(off), Some(InlineObject::Field { .. })));
// The one-pass walk agrees with offsets + object_at.
let one_pass: Vec<(usize, &InlineObject)> = p.objects_with_offsets().collect();
let each: Vec<(usize, &InlineObject)> = p.object_offsets().into_iter().filter_map(|o| Some((o, p.object_at(o)?))).collect();
assert_eq!(one_pass, each);
assert_eq!(one_pass.len(), 2);
assert_eq!(p.plain_text(), "a1b");
p.delete(0, 3).unwrap();
assert_eq!(p.objects.len(), 1);
crates/doc/src/test_log.rs
+36
View File
@@ -0,0 +1,36 @@
//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that
//! a limit which leaves something out says so.
use std::cell::RefCell;
struct Capture;
thread_local! {
static LINES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
impl log::Log for Capture {
fn enabled(&self, m: &log::Metadata) -> bool {
m.level() <= log::Level::Warn
}
fn log(&self, r: &log::Record) {
if self.enabled(r.metadata()) {
LINES.with(|l| l.borrow_mut().push(r.args().to_string()));
}
}
fn flush(&self) {}
}
static CAPTURE: Capture = Capture;
/// `f`'s result and the warnings logged on this thread while it ran.
pub(crate) fn warnings<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
// Only this logger is ever installed in a test binary; a second call keeps it.
let _ = log::set_logger(&CAPTURE);
log::set_max_level(log::LevelFilter::Warn);
LINES.with(|l| l.borrow_mut().clear());
let out = f();
(out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut())))
}
crates/doc/src/tests.rs
+128
View File
@@ -90,6 +90,45 @@ fn parts_and_media() {
assert_ne!(k1, k3);
}
#[test]
fn many_distinct_media_add_in_linear_time() {
// An import with tens of thousands of small distinct images (same length, differing only at
// the end) used to compare each against every earlier one.
let mut d = Document::new();
let t = std::time::Instant::now();
let n = 40_000u32;
for i in 0..n {
let mut bytes = vec![7u8; 256];
bytes.extend(i.to_le_bytes());
d.add_media(bytes, "png");
}
assert_eq!(d.media.len(), n as usize);
// Duplicates still find their first copy, including media inserted directly.
let mut first = vec![7u8; 256];
first.extend(0u32.to_le_bytes());
assert_eq!(d.add_media(first, "png"), "image1.png");
d.media.insert("direct.png".into(), Arc::new(vec![9, 9]));
assert_eq!(d.add_media(vec![9, 9], "png"), "direct.png");
// About half a second on a loaded machine; the old scan took over two minutes.
assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed());
// The lookup cache isn't part of the document.
let mut e = d.clone();
e.media_index = MediaIndex::default();
assert_eq!(d, e);
}
#[test]
fn media_keys_skip_taken_names_once() {
let mut d = Document::new();
// A file whose media already use the names add_media would pick next.
for i in 2..2002u32 {
d.media.insert(format!("image{i}.png"), Arc::new(i.to_le_bytes().to_vec()));
}
assert_eq!(d.add_media(vec![0], "png"), "image2002.png");
assert_eq!(d.add_media(vec![1], "png"), "image2003.png");
assert_eq!(d.add_media(vec![2], "png"), "image2004.png");
}
#[test]
fn free_ids_skip_taken_ones_instead_of_overflowing() {
assert_eq!(free_id(None, std::iter::empty(), 1), Some(1));
@@ -523,3 +562,92 @@ fn custom_props_set_get_remove() {
assert!(!d.remove_custom_prop("other"));
assert_eq!(d.custom_props.len(), 1);
}
/// A paragraph that is nothing but `objs` (built directly: inserting one at a time is slow too).
fn para_of_objects(objs: Vec<InlineObject>) -> Paragraph {
let mut p = Paragraph::new();
p.text = para::OBJ.to_string().repeat(objs.len());
p.runs = vec![Run { len: p.text.len(), props: CharProps::default() }];
p.objects = objs;
p
}
#[test]
fn object_walks_are_linear_in_the_paragraph() {
// #3614: loops of object_offsets + object_at recounted the objects before each one, so a
// paragraph of 40,000 markers took seconds per walk.
let n = 40_000u32;
let w = para::OBJ.len_utf8();
let mut objs: Vec<InlineObject> = Vec::new();
for i in 0..n {
objs.push(InlineObject::BookmarkStart { name: format!("b{i}") });
objs.push(fs("ADDIN x"));
objs.push(InlineObject::PermStart { id: i, group: "everyone".into(), editor: String::new() });
}
// Ranges close in the order they opened (the worst case for a backwards search).
for i in 0..n {
objs.push(InlineObject::FieldEnd);
objs.push(InlineObject::PermEnd { id: i });
}
let mut d = Document::new();
d.body = vec![Arc::new(Block::Para(para_of_objects(objs)))];
let t = std::time::Instant::now();
assert_eq!(d.bookmarks().len(), n as usize);
assert_eq!(d.field_ranges(StoryRef::Body).len(), n as usize);
let perms = d.perm_ranges();
assert_eq!(perms.len(), n as usize);
assert!(perms.iter().all(|r| r.end.off > r.start.off), "every range closed by its own end");
assert_eq!(d.text_box_anchor(7), None);
// A caret before a long row of field ends moves past all of them.
let first_end = 3 * n as usize * w;
let p = d.past_field_ends(&Pos::body(0, first_end));
assert_eq!(p.off, first_end + w);
// Under 0.3 s on a loaded machine; the old walks took minutes (360 s for the first test).
assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed());
let ends: Vec<InlineObject> = (0..n).map(|_| InlineObject::FieldEnd).collect();
d.body = vec![Arc::new(Block::Para(para_of_objects(ends)))];
let t = std::time::Instant::now();
assert_eq!(d.past_field_ends(&Pos::body(0, 0)).off, n as usize * w);
// Typing at the end of a row of block-level control starts.
let block = ContentControl { block: true, ..Default::default() };
let starts: Vec<InlineObject> = (0..n).map(|_| InlineObject::ControlStart { control: Box::new(block.clone()) }).collect();
d.body = vec![Arc::new(Block::Para(para_of_objects(starts)))];
let end = Pos::body(0, n as usize * w);
assert_eq!(d.typing_pos(&end), end);
assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed());
}
#[test]
fn deleting_across_many_lone_control_markers_is_not_quadratic() {
// A millisecond now; the old delete took 18 s at 1,500 markers and 114 s at 3,000.
let n = 3_000;
let starts: Vec<InlineObject> = (0..n).map(|_| InlineObject::ControlStart { control: Box::default() }).collect();
let mut d = Document::from_text("before\n\nafter");
d.body[1] = Arc::new(Block::Para(para_of_objects(starts)));
let t = std::time::Instant::now();
let at = d.delete_range(&Pos::body(0, 3), &Pos::body(2, 2)).unwrap();
assert_eq!(at, Pos::body(0, 3));
assert_eq!(d.plain_text(StoryRef::Body), "befter");
assert!(t.elapsed().as_secs_f32() < 10.0, "{:?}", t.elapsed());
}
#[test]
fn control_markers_kept_are_sorted() {
// `delete_range` groups the kept markers by paragraph and binary-searches each group, so
// `kept_control_markers` must hand them over sorted and without repeats (review of #3874).
let start = || InlineObject::ControlStart { control: Box::default() };
let mut d = Document::from_text("before");
// Controls open and close across paragraphs: 1 closes the one 0 opens, 3 the one 2 opens…
for i in 0..12 {
let objs = if i % 2 == 0 { vec![start(), start()] } else { vec![InlineObject::ControlEnd, InlineObject::ControlEnd] };
d.body.push(Arc::new(Block::Para(para_of_objects(objs))));
}
d.body.push(Arc::new(Block::Para(Paragraph::with_text("after", CharProps::default()))));
let w = para::OBJ.len_utf8();
for (a, b) in [(Pos::body(2, w), Pos::body(11, w)), (Pos::body(1, w), Pos::body(12, w)), (Pos::body(3, w), Pos::body(8, 0))] {
let kept = d.kept_control_markers(&a, &b);
assert!(!kept.is_empty(), "{a:?}..{b:?}");
assert!(kept.windows(2).all(|p| p[0] < p[1]), "sorted, no repeats: {kept:?}");
}
}
crates/docbin/Cargo.toml
+1
View File
@@ -20,5 +20,6 @@ log = { workspace = true }
chrono = { workspace = true }
[dev-dependencies]
wordcraft-doc = { workspace = true, features = ["test-support"] }
proptest = { workspace = true }
serde_json = { workspace = true }
crates/docbin/src/drawing.rs
+6
View File
@@ -81,12 +81,18 @@ pub(crate) fn anchors(
) -> Vec<(u32, InlineObject)> {
let Some(pair) = pair else { return Vec::new() };
let mut out = Vec::new();
let mut seen = std::collections::HashSet::new();
for (cp, spa) in plc(table, pair, CB_SPA) {
let i32_at = |o: usize| spa.get(o..o + 4).and_then(|b| b.try_into().ok()).map(i32::from_le_bytes).unwrap_or(0);
let spid = i32_at(0) as u32;
let [xl, yt, xr, yb] = [4, 8, 12, 16].map(i32_at);
let flags = spa.get(20..22).map(|b| u16::from_le_bytes([b[0], b[1]])).unwrap_or(0);
let Some(node) = shapes.get(&spid) else { continue };
// Word anchors each shape once. A repeated spid would build another copy of the shape,
// and of every member of a group, per anchor (#3710).
if !seen.insert(spid) {
continue;
}
let tw = |v: i64| (v as f32 / 20.0).clamp(-2.0 * MAX_PT, 2.0 * MAX_PT);
let (mut x, mut y) = (tw(xl as i64), tw(yt as i64));
let (mut w, mut h) = (tw(xr as i64 - xl as i64).clamp(0.0, MAX_PT), tw(yb as i64 - yt as i64).clamp(0.0, MAX_PT));
crates/docbin/src/lib.rs
+190 -28
View File
@@ -49,6 +49,91 @@ const MAX_PARTS: usize = 50_000;
/// headers, notes, text boxes), whatever the file's size. The working budget is tied to the
/// WordDocument stream (see [`read`]); this caps even a maximal stream.
const MAX_TOTAL_CHARS: u64 = 64 << 20;
/// Bytes of repeated work per document beyond the text itself: piece Prm grpprls (Prcs, up
/// to 64 KB each) interpreted (#3692), hyperlink targets (#3708), picture alt text (#3709)
/// and floating objects (#3710) copied into every run or anchor. Past it Prms are skipped,
/// copied strings left empty and floating objects dropped, each kind logged once a document
/// ([`Work::log`]).
const MAX_WORK: u64 = 64 << 20;
/// What the [`MAX_WORK`] budget pays for, and leaves out once it is spent.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Cost {
/// A piece's Prm grpprl: past the budget the piece keeps its CHPX formatting only.
Formatting,
/// A hyperlink target copied into a run of its result: past it the text is kept, unlinked.
LinkTarget,
/// A picture's alt text copied into an anchor: past it the picture has none.
AltText,
/// A floating object copied into its anchor: past it the object is left out.
FloatingObject,
}
impl Cost {
const ALL: [Cost; 4] = [Cost::Formatting, Cost::LinkTarget, Cost::AltText, Cost::FloatingObject];
/// What is left out, for the log.
fn left_out(self) -> &'static str {
match self {
Cost::Formatting => "some text keeps only part of its formatting",
Cost::LinkTarget => "some hyperlinks' text is kept without its link",
Cost::AltText => "some pictures are left without their alt text",
Cost::FloatingObject => "some floating shapes and pictures are left out",
}
}
}
/// One document's [`MAX_WORK`] budget, and what it has left out (each kind logged once when
/// the document has been read, [`Work::log`]).
struct Work {
left: std::cell::Cell<u64>,
dropped: std::cell::Cell<[bool; 4]>,
}
impl Work {
fn new(budget: u64) -> Self {
Work { left: std::cell::Cell::new(budget), dropped: std::cell::Cell::new([false; 4]) }
}
/// Spends `cost` bytes on `what`: false (and `what` noted as left out) when they aren't left.
fn charge(&self, cost: u64, what: Cost) -> bool {
match self.left.get().checked_sub(cost) {
Some(left) => {
self.left.set(left);
true
}
None => {
let mut dropped = self.dropped.get();
if let Some(d) = Cost::ALL.iter().position(|c| *c == what).and_then(|i| dropped.get_mut(i)) {
*d = true;
}
self.dropped.set(dropped);
false
}
}
}
/// The kinds of work left out so far, in [`Cost::ALL`] order.
fn dropped(&self) -> Vec<Cost> {
Cost::ALL.iter().zip(self.dropped.get()).filter(|(_, d)| *d).map(|(c, _)| *c).collect()
}
/// Logs one warning for each kind left out.
fn log(&self) {
let dropped = self.dropped();
for c in &dropped {
log::warn!(".doc: the document repeats more than WordCraft copies ({MAX_WORK} bytes): {}", c.left_out());
}
#[cfg(test)]
LAST_DROPPED.with(|d| *d.borrow_mut() = dropped);
}
}
#[cfg(test)]
thread_local! {
/// What the last document read on this thread left out for the work budget ([`Work::log`]).
static LAST_DROPPED: std::cell::RefCell<Vec<Cost>> = const { std::cell::RefCell::new(Vec::new()) };
}
/// `sprmCPicLocation`: the fc of a picture in the Data stream.
const C_PIC_LOCATION: u16 = 0x6A03;
/// `sprmCSymbol`: the record font and character of a symbol character ([MS-DOC] §2.6.1); the
@@ -88,10 +173,16 @@ pub enum DocbinError {
/// Is this `.doc` password-protected (XOR obfuscation, RC4 or RC4 CryptoAPI)? False for
/// anything that is not a readable Word 97-2003 file.
pub fn is_encrypted(bytes: &[u8]) -> bool {
let Ok(mut comp) = cfb::CompoundFile::open(std::io::Cursor::new(bytes)) else { return false };
let Ok(mut comp) = open_container(bytes) else { return false };
stream(&mut comp, "WordDocument").is_some_and(|w| fib::peek_encryption(&w).is_some())
}
/// Open the compound file around a `.doc`, after [`wordcraft_doc::cfb_guard::check`].
fn open_container(bytes: &[u8]) -> Result<cfb::CompoundFile<std::io::Cursor<&[u8]>>, DocbinError> {
wordcraft_doc::cfb_guard::check(bytes).map_err(DocbinError::Container)?;
cfb::CompoundFile::open(std::io::Cursor::new(bytes)).map_err(|e| DocbinError::Container(e.to_string()))
}
/// Read a `.doc` (Word 97-2003 binary) file into a [`Document`]. A password-protected file
/// gives [`DocbinError::PasswordRequired`]; see [`read_with_password`].
pub fn read(bytes: &[u8]) -> Result<Document, DocbinError> {
@@ -102,7 +193,7 @@ pub fn read(bytes: &[u8]) -> Result<Document, DocbinError> {
/// verifier ([`DocbinError::WrongPassword`] when it does not match) and then decrypted. The
/// password is ignored for an unprotected file.
pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result<Document, DocbinError> {
let mut comp = cfb::CompoundFile::open(std::io::Cursor::new(bytes)).map_err(|e| DocbinError::Container(e.to_string()))?;
let mut comp = open_container(bytes)?;
let mut word = stream(&mut comp, "WordDocument").ok_or_else(|| DocbinError::NotWord("no WordDocument stream".into()))?;
if word.len() as u64 > MAX_STREAM {
return Err(DocbinError::Limit(format!("WordDocument stream is {} bytes", word.len())));
@@ -175,6 +266,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result<Docume
// many times can, so the walk is held to twice the stream's size — generous for real files,
// tight against aliasing — and never past the absolute ceiling.
let budget = std::cell::Cell::new((word.len() as u64).saturating_mul(2).min(MAX_TOTAL_CHARS));
let work = Work::new(MAX_WORK);
let ctx = WalkCtx {
word: &word,
@@ -190,6 +282,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result<Docume
drawn: &[],
sect_ends: &[],
budget: &budget,
work: &work,
};
// Drawings: text box stories become parts first, then the floating shapes and pictures
@@ -298,6 +391,7 @@ pub fn read_with_password(bytes: &[u8], password: Option<&str>) -> Result<Docume
doc.media.extend(pics.media);
doc.body = table::assemble(paras);
doc.ensure_nonempty();
work.log();
Ok(doc)
}
@@ -334,6 +428,8 @@ struct WalkCtx<'a> {
/// footnotes, text boxes) are hostile and can each span a whole subdocument, so thousands of
/// them could otherwise materialise the same text thousands of times; this bounds the total.
budget: &'a std::cell::Cell<u64>,
/// Repeated-work bytes left ([`MAX_WORK`]), and what they left out.
work: &'a Work,
}
/// State while assembling one paragraph from per-CP formatting.
@@ -365,6 +461,14 @@ impl ParaBuild {
}
self.text.push(c);
}
/// Append `c` to the current run, whose properties the caller knows it matches.
fn extend(&mut self, c: char) {
if let Some(r) = self.runs.last_mut() {
r.len += c.len_utf8();
}
self.text.push(c);
}
}
/// Walk a CP range, splitting paragraphs at 0x0D/0x07 marks, applying the direct character
@@ -376,7 +480,7 @@ impl ParaBuild {
/// The range is clamped to the last piece's CP, and unreadable stretches (gaps, truncated
/// pieces) are skipped a whole piece at a time, so the work is bounded by the text that
/// really exists, not by the CP numbers a file claims.
fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -> Vec<ParaOut> {
fn walk<'a>(ctx: &WalkCtx<'a>, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -> Vec<ParaOut> {
let cp_end = cp_end.min(ctx.pieces.cp_end());
let mut out = Vec::new();
let mut pb = ParaBuild::default();
@@ -385,6 +489,7 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -
// Field assembly state: instruction characters are swallowed, the result is kept until
// the end character, then emitted as one object (hyperlinks keep their result text).
let mut field: Option<FieldBuild> = None;
let mut memo: Option<FmtMemo<'a>> = None;
while cp < cp_end {
// Bookmark boundaries at this CP (main-document walks only; subdocument walks get
// an empty table and `cp` never matches the main-document CPs).
@@ -490,7 +595,7 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -
f.depth -= 1;
field = Some(f);
} else {
finish_field(&mut pb, f);
finish_field(&mut pb, f, ctx.work);
}
}
cp = cp.saturating_add(1);
@@ -502,12 +607,15 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -
let fc_pic = pic_location(ctx.chpx_bins.chpx(ctx.word, fc));
match fc_pic.and_then(|at| pics.get(ctx.data, at)) {
Some(p) => {
// Every anchor of a picture gets its own copy of the alt text; the
// copies are charged to the work budget (#3709).
let alt = if ctx.work.charge(p.alt.len() as u64, Cost::AltText) { p.alt.clone() } else { String::new() };
pb.push('\u{FFFC}', CharProps::default());
pb.objects.push(InlineObject::Image {
media: p.key,
media: p.key.clone(),
w: p.w,
h: p.h,
alt: p.alt,
alt,
float: Default::default(),
crop: p.crop,
ole: None,
@@ -519,7 +627,12 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -
'\u{8}' => {
// Drawn object anchor: the floating shape or picture anchored at this CP.
let at = ctx.drawn.partition_point(|(c, _)| *c < cp);
// Each copy (a story can be walked more than once) is charged to the work budget
// by what a clone copies: a picture's alt text, a group with all its members (#3710).
for (_, obj) in ctx.drawn.get(at..).unwrap_or_default().iter().take_while(|(c, _)| *c == cp) {
if !ctx.work.charge(object_cost(obj), Cost::FloatingObject) {
continue;
}
pb.push('\u{FFFC}', CharProps::default());
pb.objects.push(obj.clone());
}
@@ -532,16 +645,16 @@ fn walk(ctx: &WalkCtx, cp_start: u32, cp_end: u32, pics: &mut media::Pictures) -
pb.objects.push(InlineObject::NoteRef { kind: *kind, id: *id, custom: String::new() });
}
}
'\t' => push_formatted(&mut pb, cp, '\t', ctx, field.as_mut()),
'\u{B}' => push_formatted(&mut pb, cp, '\n', ctx, field.as_mut()),
'\u{C}' => push_formatted(&mut pb, cp, '\u{C}', ctx, field.as_mut()),
'\u{E}' => push_formatted(&mut pb, cp, '\u{E}', ctx, field.as_mut()),
'\u{1E}' => push_formatted(&mut pb, cp, '\u{2011}', ctx, field.as_mut()),
'\u{1F}' => push_formatted(&mut pb, cp, '\u{AD}', ctx, field.as_mut()),
'\t' => push_formatted(&mut pb, cp, '\t', ctx, field.as_mut(), &mut memo),
'\u{B}' => push_formatted(&mut pb, cp, '\n', ctx, field.as_mut(), &mut memo),
'\u{C}' => push_formatted(&mut pb, cp, '\u{C}', ctx, field.as_mut(), &mut memo),
'\u{E}' => push_formatted(&mut pb, cp, '\u{E}', ctx, field.as_mut(), &mut memo),
'\u{1E}' => push_formatted(&mut pb, cp, '\u{2011}', ctx, field.as_mut(), &mut memo),
'\u{1F}' => push_formatted(&mut pb, cp, '\u{AD}', ctx, field.as_mut(), &mut memo),
// Object anchors, comment references and other special characters are dropped
// (or handled by later passes).
c if (c as u32) < 0x20 => {}
c => push_formatted(&mut pb, cp, c, ctx, field.as_mut()),
c => push_formatted(&mut pb, cp, c, ctx, field.as_mut(), &mut memo),
}
cp = cp.saturating_add(units);
}
@@ -564,19 +677,44 @@ struct FieldBuild {
sep: Option<Vec<(char, CharProps)>>,
}
/// Bytes a clone of `obj` copies: the object itself plus a picture's alt text, a group with all
/// its members (#3710).
fn object_cost(obj: &InlineObject) -> u64 {
let shallow = std::mem::size_of::<InlineObject>() as u64;
match obj {
InlineObject::Image { alt, .. } => shallow.saturating_add(alt.len() as u64),
InlineObject::Group { children, .. } => children.iter().fold(shallow, |n, c| n.saturating_add(object_cost(&c.obj))),
_ => shallow,
}
}
/// Emit a finished field: HYPERLINK keeps its result text as a linked run; every other
/// field becomes one inline object with its cached result.
fn finish_field(pb: &mut ParaBuild, f: FieldBuild) {
///
/// The target is copied once per run of equally formatted result text, not per character,
/// and each copy is charged to `work` ([`MAX_WORK`]); once that is spent the rest of the
/// result is kept as plain text (#3708).
fn finish_field(pb: &mut ParaBuild, f: FieldBuild, work: &Work) {
let instr = f.instr.trim().to_string();
let result = f.sep.unwrap_or_default();
let result_text: String = result.iter().map(|(c, _)| *c).collect();
if let Some(target) = hyperlink_target(&instr) {
for (c, mut props) in result {
props.link = Some(target.clone());
pb.push(c, props);
let cost = target.len() as u64;
let mut run: Option<CharProps> = None;
for (c, props) in result {
if run.as_ref() == Some(&props) {
pb.extend(c);
continue;
}
let mut linked = props.clone();
if work.charge(cost, Cost::LinkTarget) {
linked.link = Some(target.clone());
}
pb.push(c, linked);
run = Some(props);
}
return;
}
let result_text: String = result.iter().map(|(c, _)| *c).collect();
pb.push('\u{FFFC}', CharProps::default());
pb.objects.push(InlineObject::Field { instr, result: result_text, locked: false, dirty: false, code: false, form: String::new() });
}
@@ -702,19 +840,28 @@ fn num_of(ilfo: i32) -> Option<u32> {
/// plus the piece's Prm, over the paragraph style's base character props. Inside a field,
/// the character is recorded into the instruction or the cached result instead of (or as
/// well as) the visible text.
fn push_formatted(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx, field: Option<&mut FieldBuild>) {
fn push_formatted<'a>(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx<'a>, field: Option<&mut FieldBuild>, memo: &mut Option<FmtMemo<'a>>) {
let fc = ctx.pieces.fc_of_cp(cp).unwrap_or(0);
let grpprl = ctx.chpx_bins.chpx(ctx.word, fc);
let default;
let base = match pb.props.style.as_deref().and_then(|id| ctx.sheet.get(id)) {
Some(st) => &st.chr,
let prm = ctx.pieces.prm_of(cp);
let style = pb.props.style.as_deref();
let mut props = match memo.as_ref().filter(|m| std::ptr::eq(m.chpx, grpprl) && m.prm == prm && m.style.as_deref() == style) {
Some(m) => m.props.clone(),
None => {
default = CharProps::default();
&default
let default;
let base = match style.and_then(|id| ctx.sheet.get(id)) {
Some(st) => &st.chr,
None => {
default = CharProps::default();
&default
}
};
let mut props = char_props(grpprl, ctx.fonts, base);
apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base, ctx.work);
*memo = Some(FmtMemo { chpx: grpprl, prm, style: style.map(str::to_owned), props: props.clone() });
props
}
};
let mut props = char_props(grpprl, ctx.fonts, base);
apply_prm(&mut props, ctx.pieces.piece_prm(cp), ctx.fonts, base);
// A symbol character (Insert › Symbol) is stored as the placeholder U+0028 with
// sprmCSymbol carrying the font and the character code ([MS-DOC] §2.6.1).
let mut c = c;
@@ -739,8 +886,20 @@ fn push_formatted(pb: &mut ParaBuild, cp: u32, c: char, ctx: &WalkCtx, field: Op
}
}
/// Apply a piece's Prm (Prm0 short sprm or Prm1 grpprl) to character properties.
fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps) {
/// The last character formatting resolved, reused while the CHPX, the piece's Prm and the
/// paragraph style stay the same, so a Prm grpprl (up to 64 KB) is interpreted once per
/// formatting change, not once per character (#3692).
struct FmtMemo<'a> {
/// The CHPX grpprl, compared by address: it is a slice of the WordDocument stream.
chpx: &'a [u8],
prm: u16,
style: Option<String>,
props: CharProps,
}
/// Apply a piece's Prm (Prm0 short sprm or Prm1 grpprl) to character properties. A Prm1
/// grpprl is charged to `work`; once that is spent the grpprl is skipped.
fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &CharProps, work: &Work) {
match prm {
PrmRef::None => {}
PrmRef::Sprm0 { isprm, val } => {
@@ -753,6 +912,9 @@ fn apply_prm(props: &mut CharProps, prm: PrmRef<'_>, fonts: &[String], base: &Ch
}
}
PrmRef::Grpprl(Some(g)) => {
if !work.charge(g.len() as u64, Cost::Formatting) {
return;
}
for prl in sprm::iter(g) {
if prl.sgc() == 2 {
fmt::apply_char(props, &prl, base, fonts);
crates/docbin/src/media.rs
+39 -16
View File
@@ -17,6 +17,10 @@ const OFF_PICMID: usize = 28;
const MAX_PICTURES: usize = 5_000;
/// Most picture bytes one document copies out of the Data stream, all pictures together.
pub(crate) const MAX_MEDIA_BYTES: usize = 256 << 20;
/// Most bytes one document decodes pictures into (metafiles inflated, DIB pixels), charged
/// per attempt whether it succeeds or not: retained media is capped separately, by
/// `MAX_MEDIA_BYTES` (#3711, #3712).
const MAX_DECODE_WORK: usize = 512 << 20;
/// The document's pictures, keyed by their location in the Data stream: anchors that share a
/// `fc_pic` share one media entry (or one failure), and the total bytes copied are capped, so a
@@ -28,11 +32,16 @@ pub(crate) struct Pictures {
by_fc: HashMap<u32, Option<Inline>>,
/// BLIP store index → media key (`None`: unreadable, not retried).
by_bse: HashMap<u32, Option<String>>,
/// (foDelay, end) of a BLIP record outside the store → media key: store entries that alias
/// one record share one decode (`None`: failed, not retried).
by_delay: HashMap<(usize, usize), Option<String>>,
/// The media entries, merged into the document at the end.
pub(crate) media: BTreeMap<String, Arc<Vec<u8>>>,
total: usize,
/// The byte cap (`MAX_MEDIA_BYTES`; smaller in tests).
pub(crate) limit: usize,
/// Decode work left (`MAX_DECODE_WORK`; smaller in tests).
pub(crate) work: usize,
/// Anchors whose picture was dropped (only the first few are logged).
dropped: usize,
/// Pictures kept as placeholders (formats we can't draw).
@@ -44,9 +53,11 @@ impl Default for Pictures {
Pictures {
by_fc: HashMap::new(),
by_bse: HashMap::new(),
by_delay: HashMap::new(),
media: BTreeMap::new(),
total: 0,
limit: MAX_MEDIA_BYTES,
work: MAX_DECODE_WORK,
dropped: 0,
placeholders: 0,
}
@@ -67,16 +78,15 @@ impl Pictures {
/// The inline picture at `fc_pic` in the Data stream, copying its bytes the first time.
/// `None` when the header is malformed, the picture can't be read, or a picture
/// count/byte cap is reached.
pub(crate) fn get(&mut self, data: &[u8], fc_pic: u32) -> Option<Inline> {
if let Some(hit) = self.by_fc.get(&fc_pic) {
return hit.clone();
pub(crate) fn get(&mut self, data: &[u8], fc_pic: u32) -> Option<&Inline> {
if !self.by_fc.contains_key(&fc_pic) {
if self.by_fc.len().saturating_add(self.by_bse.len()) >= MAX_PICTURES {
return None;
}
let pic = self.read(data, fc_pic);
self.by_fc.insert(fc_pic, pic);
}
if self.by_fc.len().saturating_add(self.by_bse.len()) >= MAX_PICTURES {
return None;
}
let pic = self.read(data, fc_pic);
self.by_fc.insert(fc_pic, pic.clone());
pic
self.by_fc.get(&fc_pic)?.as_ref()
}
/// Read the inline picture at `fc_pic`: nothing is decoded or copied beyond what is left of
@@ -86,7 +96,7 @@ impl Pictures {
if room == 0 {
return None;
}
match locate(data, fc_pic, room)? {
match locate(data, fc_pic, room, &mut self.work)? {
Found::Range(pic) => {
let bytes = data.get(pic.range.clone()).filter(|b| b.len() <= room)?.to_vec();
let key = self.insert(bytes, pic.ext, false)?;
@@ -109,12 +119,25 @@ impl Pictures {
return None;
}
let fbse = bstore.get((pib as usize).checked_sub(1)?)?;
let room = self.limit.saturating_sub(self.total);
let key = officeart::fbse_blip(fbse, delay, room).and_then(|b| self.insert(b.bytes, b.ext, b.placeholder));
let key = self.decode_bse(fbse, delay);
self.by_bse.insert(pib, key.clone());
key
}
/// Decode a store entry's BLIP, once per delay-stream record however many entries name it.
fn decode_bse(&mut self, fbse: &[u8], delay: &[u8]) -> Option<String> {
let (rec, at) = officeart::fbse_record(fbse, delay)?;
if let Some(hit) = at.and_then(|at| self.by_delay.get(&at)) {
return hit.clone();
}
let room = self.limit.saturating_sub(self.total);
let key = officeart::blip(&rec, room, &mut self.work).and_then(|b| self.insert(b.bytes, b.ext, b.placeholder));
if let Some(at) = at {
self.by_delay.insert(at, key.clone());
}
key
}
/// Add picture bytes as a new media entry, within the byte cap.
fn insert(&mut self, bytes: Vec<u8>, ext: &str, placeholder: bool) -> Option<String> {
let total = self.total.checked_add(bytes.len()).filter(|&t| t <= self.limit)?;
@@ -161,7 +184,7 @@ const MM_SHAPEFILE: u16 = 0x0066;
/// Locate the picture at `fc_pic` in the Data stream ([MS-DOC] §2.9.192 PICFAndOfficeArtData);
/// a shape's picture is decoded only up to `room` bytes.
fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option<Found> {
fn locate(data: &[u8], fc_pic: u32, room: usize, work: &mut usize) -> Option<Found> {
let base = fc_pic as usize;
let pic = data.get(base..)?;
let lcb = u32::from_le_bytes(pic.get(0..4)?.try_into().ok()?) as usize;
@@ -180,7 +203,7 @@ fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option<Found> {
if mm == MM_SHAPEFILE {
at = at.checked_add(1 + *pic.get(at)? as usize)?;
}
let (blip, props) = inline_shape(pic.get(at..end)?, data, room)?;
let (blip, props) = inline_shape(pic.get(at..end)?, data, room, work)?;
// The goal size is before cropping; the crop fractions take their share off it.
let [cl, ct, cr, cb] = props.crop;
let w = (dxa_goal / 20.0 * (1.0 - cl - cr).max(0.01) * scale(mx)).clamp(1.0, 1584.0);
@@ -199,13 +222,13 @@ fn locate(data: &[u8], fc_pic: u32, room: usize) -> Option<Found> {
/// An `OfficeArtInlineSpContainer`: the picture shape, then the `OfficeArtFBSE` records its
/// `pib` indexes (BLIPs embedded, or at `foDelay` in the Data stream). `room` caps the bytes
/// the BLIP decodes to.
fn inline_shape(buf: &[u8], data: &[u8], room: usize) -> Option<(officeart::Blip, officeart::Props)> {
fn inline_shape(buf: &[u8], data: &[u8], room: usize, work: &mut usize) -> Option<(officeart::Blip, officeart::Props)> {
let mut budget = 10_000usize;
let recs = officeart::children(buf, &mut budget);
let sp = recs.iter().find(|r| r.typ == officeart::rt::SP_CONTAINER).map(|r| officeart::shape(r.body, &mut budget))?;
let fbses: Vec<&[u8]> = recs.iter().filter(|r| r.typ == officeart::rt::FBSE).map(|r| r.body).collect();
let fbse = sp.props.pib.and_then(|p| fbses.get((p as usize).checked_sub(1)?)).or(fbses.first())?;
let blip = officeart::fbse_blip(fbse, data, room)?;
let blip = officeart::fbse_blip(fbse, data, room, work)?;
Some((blip, sp.props))
}
crates/docbin/src/officeart.rs
+35 -10
View File
@@ -112,23 +112,32 @@ pub(crate) struct Blip {
/// The picture of an `OfficeArtFBSE` body: its embedded BLIP, or the BLIP stored at `foDelay`
/// in `delay` (the WordDocument stream for the document's BLIP store, the Data stream for
/// inline pictures). `limit` caps the bytes produced.
pub(crate) fn fbse_blip(fbse: &[u8], delay: &[u8], limit: usize) -> Option<Blip> {
pub(crate) fn fbse_blip(fbse: &[u8], delay: &[u8], limit: usize, work: &mut usize) -> Option<Blip> {
let (rec, _) = fbse_record(fbse, delay)?;
blip(&rec, limit, work)
}
/// The BLIP record of an `OfficeArtFBSE`: embedded after it, or in the delay stream, where
/// its (`foDelay`, end) location is returned too (store entries can share one record).
pub(crate) fn fbse_record<'a>(fbse: &'a [u8], delay: &'a [u8]) -> Option<(Rec<'a>, Option<(usize, usize)>)> {
// btWin32, btMacOS, rgbUid[16], tag, size, cRef, foDelay, unused, cbName, unused×2.
let size = u32_at(fbse, 20)? as usize;
let fo_delay = u32_at(fbse, 28)? as usize;
let cb_name = *fbse.get(33)? as usize;
let at = 36usize.checked_add(cb_name)?;
if let Some((rec, _)) = fbse.get(at..).and_then(record) {
return blip(&rec, limit);
return Some((rec, None));
}
// Not embedded: the record sits in the delay stream.
let end = fo_delay.checked_add(size.max(8))?.min(delay.len());
let (rec, _) = record(delay.get(fo_delay..end)?)?;
blip(&rec, limit)
Some((rec, Some((fo_delay, end))))
}
/// Decode one `OfficeArtBlip*` record ([MS-ODRAW] §2.2.23-§2.2.31).
pub(crate) fn blip(rec: &Rec, limit: usize) -> Option<Blip> {
/// Decode one `OfficeArtBlip*` record ([MS-ODRAW] §2.2.23-§2.2.31). `work` is the
/// document's decode budget: inflating a metafile or decoding a DIB is charged to it before
/// it runs, whether it succeeds or not, and nothing is decoded once it is spent.
pub(crate) fn blip(rec: &Rec, limit: usize, work: &mut usize) -> Option<Blip> {
// Each type has two instance values; the odd one carries a second 16-byte UID.
let uids = if rec.inst & 1 == 1 { 32 } else { 16 };
match rec.typ {
@@ -141,7 +150,7 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option<Blip> {
rt::BLIP_PNG => Some(Blip { bytes: data.to_vec(), ext: "png", placeholder: false }),
rt::BLIP_JPEG | rt::BLIP_JPEG_CMYK => Some(Blip { bytes: data.to_vec(), ext: "jpeg", placeholder: false }),
rt::BLIP_TIFF => Some(Blip { bytes: data.to_vec(), ext: "tiff", placeholder: false }),
_ => dib_to_png(data, limit),
_ => dib_to_png(data, limit, work),
}
}
rt::BLIP_EMF | rt::BLIP_WMF | rt::BLIP_PICT => {
@@ -154,7 +163,11 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option<Blip> {
let raw = raw.get(..cb_save.min(raw.len()))?;
let cap = limit.min(MAX_METAFILE);
let bytes = if compression == 0 {
let want = if cb_size > 0 { cb_size.min(cap) } else { cap };
let want = (if cb_size > 0 { cb_size.min(cap) } else { cap }).min(*work);
if want == 0 {
return None;
}
*work -= want;
miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(raw, want).ok()?
} else {
if raw.len() > cap {
@@ -177,8 +190,9 @@ pub(crate) fn blip(rec: &Rec, limit: usize) -> Option<Blip> {
/// A device-independent bitmap (BITMAPINFOHEADER + palette + pixels) as a PNG: the DIB gets
/// the 14-byte BMP file header it lacks, is decoded with size limits and re-encoded. When the
/// bitmap can't be decoded the BMP bytes are kept (and render as a grey placeholder).
fn dib_to_png(dib: &[u8], limit: usize) -> Option<Blip> {
/// bitmap can't be decoded, or its declared pixels (4 bytes each) don't fit in `work`, the
/// BMP bytes are kept (and render as a grey placeholder).
fn dib_to_png(dib: &[u8], limit: usize, work: &mut usize) -> Option<Blip> {
let header = u32_at(dib, 0)? as usize;
if !(12..=124).contains(&header) {
return None;
@@ -203,6 +217,17 @@ fn dib_to_png(dib: &[u8], limit: usize) -> Option<Blip> {
bmp.extend_from_slice(&[0, 0, 0, 0]);
bmp.extend_from_slice(&u32::try_from(off_bits).ok()?.to_le_bytes());
bmp.extend_from_slice(dib);
let placeholder = |bmp| Some(Blip { bytes: bmp, ext: "bmp", placeholder: true });
// The declared size is what decoding allocates, however small the (RLE) data is.
let (w, h) = if header == 12 {
(u64::from(u16_at(dib, 4)?), u64::from(u16_at(dib, 6)?))
} else {
(u64::from(u32_at(dib, 4)?), u64::from((u32_at(dib, 8)? as i32).unsigned_abs()))
};
let Some(left) = w.saturating_mul(h).saturating_mul(4).try_into().ok().and_then(|cost: usize| work.checked_sub(cost)) else {
return placeholder(bmp);
};
*work = left;
let mut reader = image::ImageReader::new(std::io::Cursor::new(&bmp));
reader.set_format(image::ImageFormat::Bmp);
let mut limits = image::Limits::default();
@@ -217,7 +242,7 @@ fn dib_to_png(dib: &[u8], limit: usize) -> Option<Blip> {
{
Some(Blip { bytes: png.bytes, ext: "png", placeholder: false })
}
_ => Some(Blip { bytes: bmp, ext: "bmp", placeholder: true }),
_ => placeholder(bmp),
}
}
crates/docbin/src/piece.rs
+6
View File
@@ -117,6 +117,12 @@ impl PieceTable {
}
}
/// The raw Prm of the piece holding `cp` (0, no properties, when there is none): two CPs
/// with the same value get the same [`PrmRef`].
pub(crate) fn prm_of(&self, cp: u32) -> u16 {
self.piece_of(cp).map_or(0, |p| p.prm)
}
fn piece_of(&self, cp: u32) -> Option<&Pcd> {
let i = self.cps.partition_point(|&c| c <= cp).checked_sub(1)?;
if i >= self.pieces.len() {
crates/docbin/src/tests.rs
+179 -4
View File
@@ -1732,9 +1732,9 @@ fn picture_bytes_are_capped_per_document() {
let data: Vec<u8> = [a.clone(), b].concat();
let mut pics = crate::media::Pictures::default();
pics.limit = 1000;
let first = pics.get(&data, 0).expect("first fits");
let first = pics.get(&data, 0).cloned().expect("first fits");
// Same location again: no new bytes.
assert_eq!(pics.get(&data, 0).map(|p| p.key), Some(first.key.clone()));
assert_eq!(pics.get(&data, 0).map(|p| p.key.clone()), Some(first.key.clone()));
// A second distinct picture would exceed the cap.
assert!(pics.get(&data, a.len() as u32).is_none());
assert_eq!(pics.media.len(), 1);
@@ -1754,8 +1754,8 @@ fn rejected_inline_pictures_are_not_read_again() {
assert!(pics.get(&small, 0).is_none());
// The rejected picture took nothing off the cap, and anchors sharing a picture share it.
let at = big.len() as u32;
let fits = pics.get(&data, at).expect("fits");
assert_eq!(pics.get(&data, at).map(|p| p.key), Some(fits.key));
let fits = pics.get(&data, at).cloned().expect("fits");
assert_eq!(pics.get(&data, at).map(|p| p.key.clone()), Some(fits.key));
assert_eq!(pics.media.len(), 1);
// Filling the cap exactly works; after that, a place never tried before isn't read.
let rest = picf_with_png(700);
@@ -2392,3 +2392,178 @@ fn paragraph_borders_read_from_both_forms() {
assert_eq!((top.style, top.width, top.space), (wordcraft_doc::props::BorderStyle::Single, 1.0, 2.0));
assert_eq!(top.color, Some(wordcraft_doc::Rgb(0xFF, 0, 0)));
}
/// #3642: a small compound file whose DIFAT names one FAT sector thousands of times made `cfb`
/// build a FAT of hundreds of megabytes before checking it. The precheck refuses it first.
#[test]
fn a_difat_repeating_one_fat_sector_is_refused_before_cfb_builds_the_fat() {
let f = wordcraft_doc::cfb_guard::repeating_difat_sample();
// The precheck's own error is the proof that it ran before cfb (no clock to skew).
assert!(matches!(read(&f), Err(super::DocbinError::Container(e)) if e.contains("FAT sectors in a file of")));
assert!(!super::is_encrypted(&f));
// A file cfb wrote itself passes.
assert!(wordcraft_doc::cfb_guard::check(&cfb_file(&[("WordDocument", &[0u8; 4096])])).is_ok());
}
/// A compound file `cfb` writes in `version`: `streams` at its root, then `extra` streams under
/// an `ObjectPool` storage (where Word keeps embedded objects).
fn cfb_file_in(version: cfb::Version, streams: &[(&str, &[u8])], extra: &[(&str, &[u8])]) -> Vec<u8> {
let mut comp = cfb::CompoundFile::create_with_version(version, std::io::Cursor::new(Vec::new())).expect("cfb create");
for (name, data) in streams {
comp.create_stream(format!("/{name}")).expect("cfb stream").write_all(data).expect("cfb write");
}
comp.create_storage("/ObjectPool").expect("cfb storage");
for (name, data) in extra {
comp.create_stream(format!("/ObjectPool/{name}")).expect("cfb stream").write_all(data).expect("cfb write");
}
comp.into_inner().into_inner()
}
/// #3642 review: valid compound files pass the guard and open, so it refuses only the hostile
/// ones. The repository keeps no files Word made, so these are written by the `cfb` crate in
/// the shape of Word 97–2003 files: the WordDocument and table streams of a .doc these tests
/// build, with the summary, Data and embedded-object streams Word adds, in 512-byte sectors
/// (version 3, Word's own) and 4096-byte ones (version 4); and one with a Data stream of 8 MB,
/// past what the header's 109 FAT sector slots map in 512-byte sectors (109 × 128 sectors,
/// about 7 MB), so its FAT sectors are listed in a DIFAT chain too, which the guard walks.
#[test]
fn valid_doc_shaped_compound_files_pass_the_guard_and_open() {
let doc = struct_doc(&structured_spec("Hello world\r", 12));
let mut comp = cfb::CompoundFile::open(std::io::Cursor::new(doc.as_slice())).expect("cfb open");
// The table stream is 1Table or 0Table, as the FIB's fWhichTblStm says: kept under its name.
let table_name = if comp.is_stream("/1Table") { "1Table" } else { "0Table" };
let mut stream = |name: &str| {
let mut v = Vec::new();
std::io::Read::read_to_end(&mut comp.open_stream(name).expect("stream"), &mut v).expect("read");
v
};
let (word, table) = (stream("/WordDocument"), stream(&format!("/{table_name}")));
let summary = vec![0xFEu8; 4096];
let data = |n: usize| (0..n).map(|i| (i % 251) as u8).collect::<Vec<u8>>();
let small = data(200_000);
let big = data(8 << 20);
let ole = data(70_000);
let files = [
("as built", doc.clone()),
(
"version 3",
cfb_file_in(
cfb::Version::V3,
&[("WordDocument", &word), (table_name, &table), ("Data", &small), ("\u{5}SummaryInformation", &summary)],
&[("_1234567890", &ole)],
),
),
("version 4", cfb_file_in(cfb::Version::V4, &[("WordDocument", &word), (table_name, &table), ("Data", &small)], &[("_1", &ole)])),
("with a DIFAT chain", cfb_file_in(cfb::Version::V3, &[("WordDocument", &word), (table_name, &table), ("Data", &big)], &[])),
];
let difat_sectors = |f: &[u8]| f.get(0x48..0x4C).map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
assert!(difat_sectors(&files[3].1) > 0, "the 8 MB file lists FAT sectors past the header's");
for (what, f) in &files {
assert_eq!(wordcraft_doc::cfb_guard::check(f), Ok(()), "{what}");
let doc = read(f).unwrap_or_else(|e| panic!("{what}: {e}"));
assert_eq!(body_texts(&doc).first().map(String::as_str), Some("Hello world"), "{what}");
}
}
/// #3708: a long HYPERLINK target over a long result was copied (and compared) once per result
/// character. It is now copied once per run.
#[test]
fn a_long_hyperlink_over_a_long_result_is_one_linked_run() {
let n = 250_000;
let target = "x".repeat(n);
let text: &'static str = Box::leak(format!("\u{13}HYPERLINK \"http://{target}\"\u{14}{}\u{15}\r", "y".repeat(n)).into_boxed_str());
let spec = structured_spec(text, text.chars().count() as u32);
let doc = read(&struct_doc(&spec)).expect("opens");
let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para");
let linked: Vec<_> = p0.runs.iter().filter(|r| r.props.link.is_some()).collect();
assert_eq!(linked.len(), 1);
assert_eq!(linked.first().map(|r| r.len), Some(n));
// One copy of the target for the one run, not one per character (~2.5 s here): counted by
// what the copies charge to the work budget, which a busy machine can't skew.
let result = vec![('y', wordcraft_doc::props::CharProps::default()); n];
let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"http://{target}\""), sep: Some(result) };
let work = super::Work::new(u64::MAX);
super::finish_field(&mut super::ParaBuild::default(), f, &work);
assert_eq!(u64::MAX - work.left.get(), ("http://".len() + n) as u64, "one copy of the target");
}
/// #3708: a result whose formatting changes every character pays for a target copy per run
/// out of the work budget; past it the text stays, unlinked.
#[test]
fn hyperlink_target_copies_are_charged_to_the_work_budget() {
use wordcraft_doc::props::CharProps;
let bold = CharProps { bold: Some(true), ..Default::default() };
let result: Vec<(char, CharProps)> = (0..1000).map(|i| ('a', if i % 2 == 0 { bold.clone() } else { CharProps::default() })).collect();
let f = super::FieldBuild { depth: 1, instr: format!("HYPERLINK \"{}\"", "u".repeat(100)), sep: Some(result) };
let mut pb = super::ParaBuild::default();
let budget = super::Work::new(10_000);
super::finish_field(&mut pb, f, &budget);
assert_eq!(pb.text.len(), 1000, "all text kept");
let linked: usize = pb.runs.iter().filter_map(|r| r.props.link.as_ref()).map(|l| l.len()).sum();
assert!(linked <= 10_000, "linked bytes {linked}");
assert_eq!(pb.runs.iter().filter(|r| r.props.link.is_some()).count(), 100);
assert_eq!(budget.left.get(), 0);
assert_eq!(budget.dropped(), [super::Cost::LinkTarget], "the unlinked text is noted, to be logged");
}
/// #3871 review: what the work budget leaves out is noted once per kind, in a fixed order, and
/// logged once per kind when the document has been read; nothing is noted while it lasts.
#[test]
fn the_work_budget_notes_each_kind_it_leaves_out_once() {
use super::Cost::*;
let work = super::Work::new(10);
assert!(work.charge(10, AltText));
assert!(work.dropped().is_empty(), "spent to the last byte, nothing left out");
assert!(work.charge(0, LinkTarget), "a free copy is never refused");
for _ in 0..3 {
assert!(!work.charge(1, FloatingObject));
}
assert!(!work.charge(1, AltText));
assert_eq!(work.dropped(), [AltText, FloatingObject]);
work.log();
assert_eq!(super::LAST_DROPPED.with(|d| d.borrow().clone()), [AltText, FloatingObject]);
// A document within the budget leaves nothing out.
read(&struct_doc(&structured_spec("Hello\r", 6))).expect("opens");
assert!(super::LAST_DROPPED.with(|d| d.borrow().is_empty()));
}
/// #3692: a piece's Prm grpprl (a Prc of up to 64 KB) was re-interpreted for every character.
/// It is now resolved once per formatting change.
#[test]
fn a_large_prc_is_interpreted_once_per_formatting_change() {
let prc = grpprl(&vec![(0x0835u16, &[0x01u8][..]); 21_000]);
let text: &'static str = Box::leak(format!("{}\r", "a".repeat(100_000)).into_boxed_str());
let spec = RichSpec {
text,
styles: vec![(1, 0x0FFF, "Normal", Vec::new(), Vec::new())],
fonts: vec!["Times New Roman"],
chpx_runs: vec![(0, Vec::new())],
papx_entries: vec![(0, 0, Vec::new())],
};
let bytes = rich_doc_with(&spec, &RichExtra { prcs: vec![prc], prm: 0x0001, ..Default::default() });
let t = std::time::Instant::now();
let doc = read(&bytes).expect("opens");
// About 60 ms on a loaded machine; interpreting the Prc per character took 88 s.
assert!(t.elapsed().as_secs() < 10, "opened in {:?}", t.elapsed());
let p = para0(&doc);
assert_eq!(p.runs.len(), 1);
assert_eq!(p.runs.first().and_then(|r| r.props.bold), Some(true));
}
/// #3692: Prm grpprls are charged to a document-wide work budget; once it is spent the Prm
/// is skipped (the text keeps its CHPX formatting).
#[test]
fn prm_grpprls_past_the_work_budget_are_skipped() {
use wordcraft_doc::props::CharProps;
let g = grpprl(&[(0x0835, &[0x01])]);
let work = super::Work::new(g.len() as u64 + 1);
let base = CharProps::default();
let mut first = CharProps::default();
super::apply_prm(&mut first, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work);
assert_eq!(first.bold, Some(true));
let mut second = CharProps::default();
super::apply_prm(&mut second, crate::piece::PrmRef::Grpprl(Some(&g)), &[], &base, &work);
assert_eq!(second.bold, None);
assert_eq!(work.left.get(), 1);
assert_eq!(work.dropped(), [super::Cost::Formatting]);
}
crates/docbin/src/tests/drawing.rs
+168 -7
View File
@@ -247,7 +247,7 @@ fn dib_becomes_png_and_compressed_metafiles_inflate() {
dib.extend_from_slice(&[0, 0, 255, 0, 255, 0, 0, 0]);
let r = rec(0, 0x7A8, 0xF01F, &[[0u8; 16].as_slice(), &[0xFF], &dib].concat());
let (dib_rec, _) = officeart::record(&r).unwrap();
let blip = officeart::blip(&dib_rec, 1 << 20).expect("DIB decodes");
let blip = officeart::blip(&dib_rec, 1 << 20, &mut { usize::MAX }).expect("DIB decodes");
assert_eq!(blip.ext, "png");
let img = image::load_from_memory(&blip.bytes).unwrap().to_rgb8();
assert_eq!((img.width(), img.height(), img.get_pixel(0, 0).0), (2, 1, [255, 0, 0]));
@@ -262,7 +262,7 @@ fn dib_becomes_png_and_compressed_metafiles_inflate() {
hdr.extend_from_slice(&[0x00, 0xFE]); // compression = deflate, filter
let r = rec(0, 0x216, 0xF01B, &[[0u8; 16].as_slice(), &hdr, &packed].concat());
let (wmf_rec, _) = officeart::record(&r).unwrap();
let blip = officeart::blip(&wmf_rec, 1 << 20).expect("WMF inflates");
let blip = officeart::blip(&wmf_rec, 1 << 20, &mut { usize::MAX }).expect("WMF inflates");
assert_eq!((blip.ext, blip.bytes), ("wmf", wmf));
}
@@ -281,10 +281,10 @@ fn deeply_nested_groups_and_lying_lengths_are_bounded() {
hdr.extend_from_slice(&[0, 0]);
let r = rec(0, 0x3D4, 0xF01A, &[[0u8; 16].as_slice(), &hdr, &[0x78, 0x9C, 1, 2, 3]].concat());
let (emf, _) = officeart::record(&r).unwrap();
assert!(officeart::blip(&emf, 1 << 20).is_none());
assert!(officeart::blip(&emf, 1 << 20, &mut { usize::MAX }).is_none());
let mut fbse = vec![0u8; 36];
fbse[28..32].copy_from_slice(&u32::MAX.to_le_bytes());
assert!(officeart::fbse_blip(&fbse, &[0u8; 64], 1 << 20).is_none());
assert!(officeart::fbse_blip(&fbse, &[0u8; 64], 1 << 20, &mut { usize::MAX }).is_none());
}
proptest! {
@@ -322,13 +322,174 @@ fn dib_conversion_caps_header_and_encoded_bytes() {
let record = rec(0, 0x7A8, officeart::rt::BLIP_DIB, &[[0u8; 16].as_slice(), &[0xFF], &dib].concat());
let (record, _) = officeart::record(&record).unwrap();
let bmp_size = 14 + dib.len();
assert!(officeart::blip(&record, bmp_size - 1).is_none(), "the BMP header also needs room");
let fallback = officeart::blip(&record, bmp_size).expect("the BMP fallback fits exactly");
assert!(officeart::blip(&record, bmp_size - 1, &mut { usize::MAX }).is_none(), "the BMP header also needs room");
let fallback = officeart::blip(&record, bmp_size, &mut { usize::MAX }).expect("the BMP fallback fits exactly");
assert_eq!((fallback.ext, fallback.placeholder, fallback.bytes.len()), ("bmp", true, bmp_size));
let img = image::load_from_memory(&fallback.bytes).unwrap().to_rgb8();
assert_eq!((img.width(), img.height(), img.get_pixel(0, 0).0), (1, 1, [90, 60, 30]));
let normal = officeart::blip(&record, 512).expect("normal conversion fits");
let normal = officeart::blip(&record, 512, &mut { usize::MAX }).expect("normal conversion fits");
assert_eq!((normal.ext, normal.placeholder), ("png", false));
assert!(normal.bytes.len() > bmp_size && normal.bytes.len() <= 512);
assert_eq!(image::load_from_memory(&normal.bytes).unwrap().width(), 1);
}
/// #3709: every anchor of one inline picture got its own copy of the alt text (up to 4096
/// UTF-16 units). Copies are charged to the document's work budget; past it alt is empty.
#[test]
fn alt_text_copies_across_many_anchors_are_bounded() {
let alt: String = "€".repeat(4096); // 12 KB as UTF-8
let shape = sp(&[fsp(75, 1025, 0xA00), fopt(&[(0x0104, 1)], &[(0x0381, wz(&alt))])]);
let data = picf_shape(&[shape, fbse_png(&tiny_png())].concat());
let text: &'static str = Box::leak(format!("{}\r", "\u{1}".repeat(8000)).into_boxed_str());
let ccp = text.chars().count() as u32;
let spec = StructSpec { text, data_stream: data, chpx_runs: vec![(0, grpprl(&[(0x6A03, &0u32.to_le_bytes())]))], ..structured_spec(text, ccp) };
let doc = read(&struct_doc(&spec)).expect("opens");
let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para");
let alts: Vec<&str> = p0.objects.iter().filter_map(|o| if let InlineObject::Image { alt, .. } = o { Some(alt.as_str()) } else { None }).collect();
assert_eq!(alts.len(), 8000, "every anchor keeps its picture");
assert_eq!(alts.first().copied(), Some(alt.as_str()));
let total: usize = alts.iter().map(|a| a.len()).sum();
assert!(total <= 64 << 20, "alt bytes {total}");
assert_eq!(alts.last().copied(), Some(""));
// Leaving it out is logged, once (#3871 review).
assert_eq!(crate::LAST_DROPPED.with(|d| d.borrow().clone()), [crate::Cost::AltText]);
}
/// #3710: every PlcSpa entry naming a group built another copy of all its members. A shape is
/// anchored once; repeated spids are skipped.
#[test]
fn a_group_named_by_many_anchors_is_built_once() {
let rect = |l: i32| rec(0, 0, 0xF00F, &[l, 0, l + 10, 10].iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<_>>());
let head = sp(&[fsp(0, 3000, 0x201), rec(1, 0, 0xF009, &[0i32, 0, 10_000, 10].iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<_>>())]);
let members: Vec<Vec<u8>> = (0..1000).map(|i| sp(&[fsp(1, 4000 + i as u32, 0xA02), rect(i * 10)])).collect();
let group = container(0xF003, &[vec![head], members].concat());
let art = art_content(&[], &[group]);
let n = 5000u32;
let cps: Vec<u32> = (1..=n + 1).collect();
let spas: Vec<u8> = (0..n).flat_map(|_| spa(3000, [0, 0, 1440, 1440], spa_flags(0, 0, 3, false))).collect();
let text = "A\u{8}\u{8}\u{8}B\r";
let ccp = text.chars().count() as u32;
let spec = StructSpec { text, extra_pairs: vec![(40, plc(&cps, &spas)), (50, art)], ..structured_spec(text, ccp) };
let t = std::time::Instant::now();
let doc = read(&struct_doc(&spec)).expect("opens");
// About 10 ms on a loaded machine, so 10 s is a backstop; the old code (a copy per entry,
// 6-8 s here) is caught by the group count below, one group per anchor ([1000; 3]).
assert!(t.elapsed().as_secs() < 10, "opened in {:?}", t.elapsed());
let p0 = doc.body.first().and_then(|b| b.as_para()).expect("para");
let groups: Vec<usize> =
p0.objects.iter().filter_map(|o| if let InlineObject::Group { children, .. } = o { Some(children.len()) } else { None }).collect();
assert_eq!(groups, [1000]);
}
/// #3710: a floating picture's copy is charged its alt text too, and a group every member's.
#[test]
fn a_floating_copy_is_charged_its_alt_text() {
let image = |alt: String| InlineObject::Image {
media: String::new(),
w: 1.0,
h: 1.0,
alt,
float: Default::default(),
crop: Default::default(),
ole: None,
};
let bare = crate::object_cost(&image(String::new()));
assert_eq!(crate::object_cost(&image("€".repeat(4096))), bare + 3 * 4096);
let member = |obj| wordcraft_doc::para::GroupChild { x: 0.0, y: 0.0, obj };
let group = InlineObject::Group {
w: 1.0,
h: 1.0,
float: Default::default(),
ch_w: 1.0,
ch_h: 1.0,
children: vec![member(image("x".repeat(100))), member(image(String::new()))],
canvas: None,
};
assert_eq!(crate::object_cost(&group), 3 * bare + 100);
}
/// A deflated WMF BLIP record declaring `size` uncompressed bytes.
fn packed_wmf(size: u32, packed: &[u8]) -> Vec<u8> {
let mut hdr = size.to_le_bytes().to_vec();
hdr.extend_from_slice(&[0u8; 24]); // rcBounds, ptSize
hdr.extend_from_slice(&(packed.len() as u32).to_le_bytes());
hdr.extend_from_slice(&[0x00, 0xFE]); // compression = deflate, filter
rec(0, 0x216, officeart::rt::BLIP_WMF, &[[0u8; 16].as_slice(), &hdr, packed].concat())
}
/// An `OfficeArtFBSE` body whose BLIP (of `size` bytes) sits at `fo_delay` in the delay stream.
fn delayed_fbse(size: usize, fo_delay: usize) -> Vec<u8> {
let mut fbse = vec![0u8; 36];
fbse[20..24].copy_from_slice(&(size as u32).to_le_bytes());
fbse[28..32].copy_from_slice(&(fo_delay as u32).to_le_bytes());
fbse
}
/// #3711: store entries aliasing one delay-stream metafile each inflated (and copied) it again.
/// Now the record is decoded once and every entry shares its media key.
#[test]
fn store_entries_aliasing_one_delayed_metafile_decode_it_once() {
let wmf = vec![0u8; 1 << 20];
let delay = packed_wmf(wmf.len() as u32, &miniz_oxide::deflate::compress_to_vec_zlib(&wmf, 6));
let fbse = delayed_fbse(delay.len(), 0);
let store: Vec<&[u8]> = vec![&fbse; 5_000];
let mut pics = crate::media::Pictures::default();
pics.work = 8 << 20;
let keys: Vec<_> = (1..=store.len() as u32).map(|pib| pics.bse(pib, &store, &delay)).collect();
assert!(keys.iter().all(|k| k.is_some() && *k == keys[0]), "every entry shares one picture");
assert_eq!(pics.work, (8 << 20) - wmf.len(), "inflated once");
}
/// #3711: distinct metafiles that fail to inflate are charged their declared size anyway, so
/// a document of them stops decoding once the work budget is spent.
#[test]
fn failing_metafile_inflations_spend_the_decode_budget() {
let records: Vec<Vec<u8>> = (0..10u8).map(|i| packed_wmf(1 << 20, &[0x78, 0x9C, i, 1, 2, 3])).collect();
let delay = records.concat();
let mut at = 0;
let fbses: Vec<Vec<u8>> = records
.iter()
.map(|r| {
at += r.len();
delayed_fbse(r.len(), at - r.len())
})
.collect();
let store: Vec<&[u8]> = fbses.iter().map(Vec::as_slice).collect();
let mut pics = crate::media::Pictures::default();
pics.work = 3 << 20;
for pib in 1..=store.len() as u32 {
assert!(pics.bse(pib, &store, &delay).is_none());
}
assert_eq!(pics.work, 0, "three attempts spent the budget; the rest weren't tried");
}
/// #3712: a DIB's declared pixels are charged before decoding (an RLE bitmap is tiny on disk but
/// decodes to w × h × 4 bytes); past the budget it stays a BMP placeholder, undecoded.
#[test]
fn dib_decoding_is_charged_its_declared_pixels() {
let blip_of = |dib: &[u8]| rec(0, 0x7A8, officeart::rt::BLIP_DIB, &[[0u8; 16].as_slice(), &[0xFF], dib].concat());
let one = blip_of(&one_pixel_dib());
let (one, _) = officeart::record(&one).unwrap();
let mut work = 3;
assert_eq!(officeart::blip(&one, 512, &mut work).map(|b| (b.ext, b.placeholder)), Some(("bmp", true)));
assert_eq!(work, 3, "nothing decoded, nothing charged");
let mut work = 4;
assert_eq!(officeart::blip(&one, 512, &mut work).map(|b| (b.ext, b.placeholder)), Some(("png", false)));
assert_eq!(work, 0);
// 8-bit RLE declaring 16384 × 16384 (1 GiB decoded) in a few bytes.
let mut big = [40u32, 16_384, 16_384].iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<_>>();
big.extend_from_slice(&1u16.to_le_bytes());
big.extend_from_slice(&8u16.to_le_bytes());
big.extend_from_slice(&1u32.to_le_bytes()); // BI_RLE8
big.extend_from_slice(&[0u8; 12]);
big.extend_from_slice(&1u32.to_le_bytes()); // one palette colour
big.extend_from_slice(&[0u8; 4]); // clrImportant
big.extend_from_slice(&[0u8; 4]); // the palette colour
big.extend_from_slice(&[0, 0, 0, 1]); // end of line, end of bitmap
let big = blip_of(&big);
let (big, _) = officeart::record(&big).unwrap();
let mut work = 512 << 20;
assert_eq!(officeart::blip(&big, 1 << 20, &mut work).map(|b| (b.ext, b.placeholder)), Some(("bmp", true)));
assert_eq!(work, 512 << 20);
}
crates/docx/Cargo.toml
+1
View File
@@ -31,6 +31,7 @@ zeroize = { workspace = true }
getrandom = { workspace = true, features = ["wasm_js"] }
[dev-dependencies]
wordcraft-doc = { workspace = true, features = ["test-support"] }
proptest = { workspace = true }
image = { workspace = true }
zip = { workspace = true }
crates/docx/src/chart_spec.rs
+56 -3
View File
@@ -6,7 +6,7 @@
//! [`spec_of`] turns a chart part back into the model only when writing that model again gives
//! the same markup, so a chart with anything the model leaves out stays as it was read.
use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, LegendPos};
use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
use wordcraft_doc::graphic::GraphicItem;
use wordcraft_doc::props::Rgb;
@@ -23,9 +23,21 @@ pub(crate) const CHART_URI: &str = "http://schemas.openxmlformats.org/drawingml/
const CAT_AX: &str = "500000001";
const VAL_AX: &str = "500000002";
/// The chart part for `spec`: a complete XML document. `spec` should be sanitized
/// ([`ChartSpec::sanitize`]); a value that isn't finite is written as a gap either way.
/// The chart part for `spec`: a complete XML document, written as [`ChartSpec::sanitize`] leaves
/// it (a value that isn't finite is a gap).
pub fn chart_xml(spec: &ChartSpec) -> String {
chart_part(spec).0
}
/// [`chart_xml`], and whether the chart limits left series or points of `spec` out of it.
pub(crate) fn chart_part(spec: &ChartSpec) -> (String, bool) {
let kept = bounded(spec);
let points = spec.series.iter().map(|s| s.values.len()).max().unwrap_or(0).max(spec.categories.len());
let cut = kept.series.len() < spec.series.len() || kept.points() < points;
(write_chart(&kept), cut)
}
fn write_chart(spec: &ChartSpec) -> String {
let mut w = W::new();
w.open("c:chartSpace", &[("xmlns:c", NS_C), ("xmlns:a", NS_A), ("xmlns:r", NS_R)]);
w.empty("c:roundedCorners", &[("val", "0")]);
@@ -79,6 +91,26 @@ pub fn chart_xml(spec: &ChartSpec) -> String {
w.s
}
/// `spec` held to the chart limits. A model deserialised as is (a `.json` document) never went
/// through [`ChartSpec::sanitize`], and every series repeats every category, so its size would
/// multiply (#3648). Only what the limits keep is copied before sanitizing.
fn bounded(spec: &ChartSpec) -> ChartSpec {
ChartSpec {
kind: spec.kind,
categories: spec.categories.iter().take(MAX_POINTS).cloned().collect(),
series: spec
.series
.iter()
.take(MAX_SERIES)
.map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() })
.collect(),
title: spec.title.clone(),
legend: spec.legend,
data_labels: spec.data_labels,
}
.sanitized()
}
/// What `spec` draws as, `w` × `h` points, in the colours of `theme` (series take the accents in
/// turn). Empty when the chart has no series.
pub fn chart_items(spec: &ChartSpec, theme: &[Rgb], w: f32, h: f32) -> Vec<GraphicItem> {
@@ -421,6 +453,27 @@ mod tests {
e.els().map(|c| c.name.as_str()).collect()
}
#[test]
fn a_chart_past_the_limits_is_written_within_them() {
// A model deserialised as is (a `.json` document) never went through `sanitize`, and every
// series repeats every category: 300 series of 4,500 points wrote 2.7 million points (#3648).
use wordcraft_doc::chart::MAX_CELLS;
let spec = ChartSpec {
kind: ChartType::Column,
categories: (0..4500).map(|i| i.to_string()).collect(),
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(),
title: None,
legend: None,
data_labels: false,
};
let x = chart_xml(&spec);
let series = x.matches("<c:ser>").count();
let points = x.matches("<c:pt ").count();
assert!((1..=MAX_SERIES).contains(&series), "{series} series");
assert!(points <= 2 * MAX_CELLS, "{points} points");
assert!(!x.contains(&format!("<c:pt idx=\"{MAX_POINTS}\"")), "past the point limit");
}
#[test]
fn every_type_reads_back_as_itself() {
for kind in ChartType::ALL {
crates/docx/src/crypt.rs
+2 -1
View File
@@ -74,7 +74,7 @@ const NS_PASSWORD: &str = "http://schemas.microsoft.com/office/2006/keyEncryptor
/// Is this an encrypted Office package (a compound file with `EncryptionInfo` and
/// `EncryptedPackage` streams) rather than a zip?
pub fn is_encrypted(bytes: &[u8]) -> bool {
if bytes.get(..8) != Some(&CFB_MAGIC[..]) {
if bytes.get(..8) != Some(&CFB_MAGIC[..]) || wordcraft_doc::cfb_guard::check(bytes).is_err() {
return false;
}
match cfb::CompoundFile::open(Cursor::new(bytes)) {
@@ -87,6 +87,7 @@ pub fn is_encrypted(bytes: &[u8]) -> bool {
/// [`DocxError::PasswordRequired`] once it knows the file is one it can decrypt; a password that
/// doesn't fit gives [`DocxError::WrongPassword`].
pub fn decrypt(bytes: &[u8], password: Option<&str>) -> Result<Vec<u8>, DocxError> {
wordcraft_doc::cfb_guard::check(bytes).map_err(|e| bad(format!("not a readable compound file ({e})")))?;
let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)).map_err(|e| bad(format!("not a readable compound file ({e})")))?;
let info = stream(&mut comp, "/EncryptionInfo", MAX_INFO)?;
let package = stream(&mut comp, "/EncryptedPackage", bytes.len() as u64)?;
crates/docx/src/flat.rs
+99 -1
View File
@@ -437,7 +437,47 @@ fn xml_fragment<'a>(ct: &str, bytes: &'a [u8]) -> Option<&'a str> {
Some(rest) => rest.split_once("?>").map(|(_, r)| r)?.trim_start(),
None => s,
};
(s.starts_with('<') && !s.contains("<!DOCTYPE")).then_some(s)
(s.starts_with('<') && !s.contains("<!DOCTYPE") && one_element(s)).then_some(s)
}
/// Whether `s` is exactly one well-formed element, with only whitespace, comments and processing
/// instructions around it. Kept bytes are inlined into `pkg:xmlData` only then; anything else could
/// close the wrapper and add parts of its own, so it goes out as `pkg:binaryData` (#3742).
fn one_element(s: &str) -> bool {
let mut r = quick_xml::Reader::from_str(s);
r.config_mut().check_comments = true;
let (mut depth, mut roots) = (0usize, 0usize);
loop {
let attrs_ok = |e: &BytesStart| e.attributes().all(|a| a.is_ok());
match r.read_event() {
Ok(Event::Start(e)) => {
if !attrs_ok(&e) || (depth == 0 && roots > 0) {
return false;
}
roots += usize::from(depth == 0);
depth += 1;
}
Ok(Event::Empty(e)) => {
if !attrs_ok(&e) || (depth == 0 && roots > 0) {
return false;
}
roots += usize::from(depth == 0);
}
Ok(Event::End(_)) => match depth.checked_sub(1) {
Some(d) => depth = d,
None => return false,
},
Ok(Event::Text(t)) if depth == 0 => {
if !t.iter().all(u8::is_ascii_whitespace) {
return false;
}
}
Ok(Event::CData(_) | Event::GeneralRef(_)) if depth == 0 => return false,
Ok(Event::Decl(_) | Event::DocType(_)) | Err(_) => return false,
Ok(Event::Eof) => return depth == 0 && roots == 1,
Ok(_) => {}
}
}
}
#[cfg(test)]
@@ -494,6 +534,64 @@ mod tests {
assert!(matches!(unpack(package(xml::MAX_DEPTH + 1).as_bytes()), Err(DocxError::Limit(_))));
}
#[test]
fn kept_xml_that_would_break_out_of_its_part_is_written_in_base64() {
// A part kept from a file is written back as it was read; inlined as is, markup that
// closes the wrapper adds parts of its own (here a second core properties part, read
// before the real one) (#3742).
let evil = format!(
"<a/></pkg:xmlData></pkg:part><pkg:part pkg:name=\"/docProps/core.xml\" pkg:contentType=\"application/vnd.openxmlformats-package.core-properties+xml\"><pkg:xmlData><cp:coreProperties xmlns:cp=\"{}\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\"><dc:title>Injected</dc:title></cp:coreProperties></pkg:xmlData></pkg:part><pkg:part pkg:name=\"/x.xml\"><pkg:xmlData><b>",
"http://schemas.openxmlformats.org/package/2006/metadata/core-properties"
);
let w_ns = "xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"";
let rel = |id: &str, kind: &str, target: &str| format!("<Relationship Id=\"{id}\" Type=\"{kind}\" Target=\"{target}\"/>");
let rels = format!(
"<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">{}{}</Relationships>",
rel("rId1", crate::package::rt::OFFICE_DOC, "word/document.xml"),
rel("rId2", crate::package::rt::CUSTOM_XML, "customXml/item1.xml")
);
let types = "<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"xml\" ContentType=\"application/xml\"/></Types>";
let z = crate::package::zip_entries(&[
("[Content_Types].xml".into(), types.as_bytes().to_vec()),
("_rels/.rels".into(), rels.into_bytes()),
("word/document.xml".into(), format!("<w:document {w_ns}><w:body><w:p/></w:body></w:document>").into_bytes()),
("customXml/item1.xml".into(), evil.clone().into_bytes()),
])
.unwrap();
let mut doc = crate::read(&z).unwrap();
assert!(doc.passthrough.values().any(|b| b.as_slice() == evil.as_bytes()), "the part is kept as read");
doc.core.title = "Real".into();
let flat = write(&doc).unwrap();
let text = String::from_utf8_lossy(&flat);
assert_eq!(text.matches("pkg:name=\"/docProps/core.xml\"").count(), 1, "{text}");
let back = read(&flat).unwrap();
assert_eq!(back.core.title, "Real");
assert!(back.passthrough.values().any(|b| b.as_slice() == evil.as_bytes()), "the part survives, in base64");
}
#[test]
fn only_one_well_formed_element_is_inlined() {
let ct = "application/xml";
for ok in ["<a/>", "<?xml version=\"1.0\"?>\n<a x=\"1\"><b>t &amp; <![CDATA[c]]></b></a>\n<!-- end -->", "<!-- c --><a/><?pi x?>"] {
assert!(xml_fragment(ct, ok.as_bytes()).is_some(), "{ok}");
}
for bad in [
"<a/><b/>",
"<a></b>",
"<a>",
"<a/></x>",
"<a/>text",
"<a/>&amp;",
"<a x=1/>",
"<a/><?xml version=\"1.0\"?>",
"<a></a></pkg:xmlData>",
"<a><!-- x -- y --></a>",
"<a/><!-- x--->",
] {
assert!(xml_fragment(ct, bad.as_bytes()).is_none(), "{bad}");
}
}
#[test]
fn repeated_and_nameless_parts_count_against_the_size_limits() {
let (mut files, mut types, mut total) = (BTreeMap::new(), Vec::new(), 0u64);
crates/docx/src/fonts.rs
+79
View File
@@ -4,6 +4,8 @@
//! back. [`embedded_fonts`] hands their font files to whoever lays the document out, undoing the
//! obfuscation Word applies to them (ECMA-376 Part 1, 17.8.1).
use std::collections::HashSet;
use wordcraft_doc::Document;
use crate::package::{EMBEDDED_PARTS, EmbeddedManifest, FONT_TABLE_PART};
@@ -11,6 +13,11 @@ use crate::package::{EMBEDDED_PARTS, EmbeddedManifest, FONT_TABLE_PART};
/// Most font files taken from one document.
const MAX_FONTS: usize = 64;
/// Most bytes of font files copied out of one document: one part's limit, however many
/// references the font table makes (each reference is a copy; 64 of them to one large part
/// were 64 copies, #3692).
const MAX_FONT_BYTES: u64 = crate::package::MAX_PART;
/// The font table's references to embedded font files, one per style.
const EMBEDS: [&str; 4] = ["w:embedRegular", "w:embedBold", "w:embedItalic", "w:embedBoldItalic"];
@@ -28,6 +35,12 @@ pub struct EmbeddedFont {
/// fonts: that is up to the font database. Empty for a document read from anything but a package
/// with a font table.
pub fn embedded_fonts(doc: &Document) -> Vec<EmbeddedFont> {
fonts_within(doc, MAX_FONT_BYTES)
}
/// [`embedded_fonts`], copying at most `budget` bytes of font files.
fn fonts_within(doc: &Document, mut budget: u64) -> Vec<EmbeddedFont> {
let budget_total = budget;
let Some(table) = doc.passthrough.get(FONT_TABLE_PART).and_then(|b| crate::xml::parse(b).ok()) else {
return Vec::new();
};
@@ -36,6 +49,9 @@ pub fn embedded_fonts(doc: &Document) -> Vec<EmbeddedFont> {
return Vec::new();
};
let mut out = Vec::new();
// A reference repeating one already taken (same file, key and name) gives the same font.
let mut taken: HashSet<(&str, [u8; 16], &str)> = HashSet::new();
let mut over_budget = 0usize;
let embeds = table
.children("w:font")
.flat_map(|f| f.els().filter(|e| EMBEDS.contains(&e.name.as_str())).map(move |e| (f.attr("w:name").unwrap_or(""), e)));
@@ -61,12 +77,25 @@ pub fn embedded_fonts(doc: &Document) -> Vec<EmbeddedFont> {
},
None => [0; 16],
};
if !taken.insert((rel.target.as_str(), key, name)) {
continue;
}
// Checked before the copy is made.
let Some(left) = budget.checked_sub(bytes.len() as u64) else {
over_budget += 1;
continue;
};
budget = left;
let mut data = bytes.to_vec();
for (i, b) in data.iter_mut().take(32).enumerate() {
*b ^= key.get(i % 16).copied().unwrap_or(0);
}
out.push(EmbeddedFont { name: name.to_string(), data });
}
if over_budget > 0 {
// Once for the document, not once per font.
log::warn!("docx: embedded fonts past the {budget_total} byte budget; {over_budget} of them are not loaded");
}
out
}
@@ -83,3 +112,53 @@ fn font_key(guid: &str) -> Option<[u8; 16]> {
}
Some(key)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::*;
/// A document whose font table makes `refs` references (`(font name, part)`) to font parts
/// `parts` (`(name, size)`).
fn doc_with(refs: &[(&str, &str)], parts: &[(&str, usize)]) -> Document {
let mut table = String::from(
r#"<w:fonts xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">"#,
);
let mut manifest = format!("P\t{FONT_TABLE_PART}\tapplication/xml\n");
for (i, (font, part)) in refs.iter().enumerate() {
table.push_str(&format!(r#"<w:font w:name="{font}"><w:embedRegular r:id="rId{i}"/></w:font>"#));
manifest.push_str(&format!("R\t{FONT_TABLE_PART}\trId{i}\tfont\t{part}\tI\n"));
}
table.push_str("</w:fonts>");
let mut doc = Document::default();
doc.passthrough.insert(FONT_TABLE_PART.into(), Arc::new(table.into_bytes()));
doc.passthrough.insert(EMBEDDED_PARTS.into(), Arc::new(manifest.into_bytes()));
for (part, size) in parts {
doc.passthrough.insert((*part).into(), Arc::new(vec![7u8; *size]));
}
doc
}
#[test]
fn a_font_referenced_again_is_copied_once() {
// Each reference was its own copy: the 64 references allowed to one 256 MB part were
// 16 GB of copies (#3692).
let refs = vec![("Body", "word/fonts/a.odttf"); MAX_FONTS];
let fonts = embedded_fonts(&doc_with(&refs, &[("word/fonts/a.odttf", 1000)]));
assert_eq!(fonts.len(), 1);
}
#[test]
fn font_copies_stop_at_the_byte_budget() {
let refs: Vec<(String, &str)> = (0..MAX_FONTS).map(|i| (format!("Font {i}"), "word/fonts/a.odttf")).collect();
let refs: Vec<(&str, &str)> = refs.iter().map(|(n, p)| (n.as_str(), *p)).collect();
let doc = doc_with(&refs, &[("word/fonts/a.odttf", 1000)]);
let (fonts, warned) = crate::test_log::warnings(|| fonts_within(&doc, 3500));
assert_eq!(fonts.len(), 3, "three copies fit");
let budget: Vec<_> = warned.iter().filter(|w| w.contains("byte budget")).collect();
assert_eq!(budget.len(), 1, "one warning for the document, not one per font: {warned:?}");
assert!(budget.iter().all(|w| w.contains(&format!("{} of them", MAX_FONTS - 3))), "{budget:?}");
assert_eq!(embedded_fonts(&doc).len(), MAX_FONTS, "well within the real budget");
}
}
crates/docx/src/lib.rs
+2
View File
@@ -16,6 +16,8 @@ mod fonts;
mod package;
mod read;
mod smart_art;
#[cfg(test)]
mod test_log;
mod units;
mod wordml2003;
mod wordml2003_write;
crates/docx/src/package.rs
+127 -13
View File
@@ -1,6 +1,7 @@
//! OPC package access: bounded zip reading, relationships, part names.
use std::collections::{BTreeMap, HashMap};
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{Cursor, Read, Write};
use std::sync::Arc;
@@ -191,6 +192,9 @@ pub fn rel_is(t: &str, full: &str) -> bool {
/// picture, an embedded object) is handed over rather than copied.
pub struct Package {
files: BTreeMap<String, Arc<Vec<u8>>>,
/// Lowercased part name → its stored spelling (the first in name order): part names are
/// case-insensitive (ECMA-376 Part 2 §6.2.2.3), so every spelling is one part.
lower: HashMap<String, String>,
}
impl Package {
@@ -223,24 +227,60 @@ impl Package {
let room = max_part.min(max_total.saturating_sub(total));
let mut buf = Vec::with_capacity(f.size().min(16 * 1024 * 1024) as usize);
let read = (&mut f).take(room + 1).read_to_end(&mut buf);
// Charged whether or not the read succeeds: an entry inflates in full before its CRC
// is checked, so failed entries would otherwise inflate without limit (#3723).
total = total.saturating_add(buf.len() as u64);
if buf.len() as u64 > room {
return Err(DocxError::Limit(format!("part {name} decompresses past the size limit")));
}
if let Err(e) = read {
// A corrupt entry: keep going unless it's a part we can't do without.
log::warn!("docx: zip entry {name} unreadable: {e}");
continue;
}
if buf.len() as u64 > room {
return Err(DocxError::Limit(format!("part {name} decompresses past the size limit")));
}
total = total.saturating_add(buf.len() as u64);
files.insert(name, Arc::new(buf));
}
Ok(Package { files })
Ok(Package::new(files))
}
/// A package from parts already in memory (a Flat OPC file, a converted document), by part
/// name without the leading `/`.
pub fn from_parts(files: BTreeMap<String, Vec<u8>>) -> Package {
Package { files: files.into_iter().map(|(name, bytes)| (name, Arc::new(bytes))).collect() }
Package::new(files.into_iter().map(|(name, bytes)| (name, Arc::new(bytes))).collect())
}
fn new(mut files: BTreeMap<String, Arc<Vec<u8>>>) -> Package {
let mut lower: HashMap<String, String> = HashMap::new();
let mut collides = false;
for name in files.keys() {
match lower.entry(name.to_ascii_lowercase()) {
Entry::Occupied(_) => collides = true,
Entry::Vacant(v) => {
v.insert(name.clone());
}
}
}
if !collides {
return Package { files, lower };
}
// Entries whose names differ only in case are one part, so every spelling of the name
// reaches the same stored entry (#3704). The spelling kept is the one a relationship
// names, so the part the document points at is the one read; the first in byte order only
// when no relationship names any of them.
let named = rel_targets(&files);
for name in files.keys().rev().filter(|n| named.contains(n.as_str())) {
lower.insert(name.to_ascii_lowercase(), name.clone());
}
let before = files.len();
files.retain(|name, _| lower.get(&name.to_ascii_lowercase()) == Some(name));
let dropped = before.saturating_sub(files.len());
log::warn!("docx: {dropped} zip entries are other spellings of parts already read; they are left out");
Package { files, lower }
}
/// The stored spelling of part `name` (leading `/` ignored; case-insensitive).
pub fn canonical(&self, name: &str) -> Option<&str> {
self.lower.get(&name.trim_start_matches('/').to_ascii_lowercase()).map(String::as_str)
}
/// Part bytes by name (leading `/` ignored; case-insensitive fallback).
@@ -251,11 +291,7 @@ impl Package {
/// [`Package::get`], as the shared bytes themselves: for a part the document keeps whole, which
/// then costs no copy (a picture document's pictures used to be held twice while it opened).
pub fn get_shared(&self, name: &str) -> Option<&Arc<Vec<u8>>> {
let n = name.trim_start_matches('/');
if let Some(v) = self.files.get(n) {
return Some(v);
}
self.files.iter().find(|(k, _)| k.eq_ignore_ascii_case(n)).map(|(_, v)| v)
self.canonical(name).and_then(|n| self.files.get(n))
}
/// [`Package::xml`] for the main document part: malformed XML is an error, not a truncated read.
@@ -289,7 +325,14 @@ impl Package {
for r in root.els().filter(|e| e.local() == "Relationship") {
let (Some(id), Some(target)) = (r.attr("Id"), r.attr("Target")) else { continue };
let external = r.attr("TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External"));
let resolved = if external { target.to_string() } else { resolve(dir, target) };
// Internal targets name the part as stored, so a part reached under several spellings
// is kept, cached and written once (#3704).
let resolved = if external {
target.to_string()
} else {
let path = resolve(dir, target);
self.canonical(&path).map_or(path, str::to_string)
};
rels.list.push(Rel { id: id.to_string(), kind: r.attr("Type").unwrap_or("").to_string(), target: resolved, external });
}
rels
@@ -364,6 +407,24 @@ fn split_dir(part: &str) -> (&str, &str) {
}
}
/// Every internal part name the relationship parts in `files` target, spelled as they spell it.
fn rel_targets(files: &BTreeMap<String, Arc<Vec<u8>>>) -> HashSet<String> {
let mut named = HashSet::new();
for (name, bytes) in files {
// `dir/_rels/file.rels` relates `dir/file`, so its targets resolve against `dir`.
let Some(at) = name.rfind("_rels/").filter(|_| name.to_ascii_lowercase().ends_with(".rels")) else { continue };
let dir = name.get(..at).unwrap_or("").trim_end_matches('/');
let Ok(root) = xml::parse(bytes) else { continue };
for r in root.els().filter(|e| e.local() == "Relationship") {
let external = r.attr("TargetMode").is_some_and(|m| m.eq_ignore_ascii_case("External"));
if let (Some(target), false) = (r.attr("Target"), external) {
named.insert(resolve(dir, target));
}
}
}
named
}
/// Resolve a relative target against a source directory into a package path.
pub fn resolve(dir: &str, target: &str) -> String {
let target = target.split('#').next().unwrap_or(target);
@@ -420,6 +481,39 @@ mod tests {
zip_entries(&e).unwrap()
}
#[test]
fn case_colliding_entries_are_one_part() {
let z = zip_of(&[("word/media/Image.png", b"upper".to_vec()), ("word/media/image.png", b"lower".to_vec())]);
let p = Package::open(&z).unwrap();
for spelling in ["word/media/Image.png", "word/media/image.png", "/WORD/MEDIA/IMAGE.PNG"] {
assert_eq!(p.canonical(spelling), Some("word/media/Image.png"), "{spelling}");
assert_eq!(p.get(spelling), Some(&b"upper"[..]), "{spelling}");
}
assert_eq!(p.files.len(), 1);
}
#[test]
fn the_spelling_a_relationship_names_is_kept() {
// The first spelling in byte order ("Image.png") was kept even when the document's
// relationship named the other one, so the picture shown was not the one it pointed at.
let rels = r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="image" Target="media/image.png"/></Relationships>"#;
let z = zip_of(&[
("word/_rels/document.xml.rels", rels.as_bytes().to_vec()),
("word/media/Image.png", b"upper".to_vec()),
("word/media/image.png", b"lower".to_vec()),
("word/media/Other.png", b"first".to_vec()),
("word/media/other.png", b"second".to_vec()),
]);
let (p, warned) = crate::test_log::warnings(|| Package::open(&z).unwrap());
assert_eq!(p.canonical("word/media/IMAGE.png"), Some("word/media/image.png"));
assert_eq!(p.get("word/media/Image.png"), Some(&b"lower"[..]));
assert_eq!(p.get("word/media/OTHER.PNG"), Some(&b"first"[..]), "no relationship names it: first in byte order");
assert_eq!(p.files.len(), 3);
let folded: Vec<_> = warned.iter().filter(|w| w.contains("other spellings")).collect();
assert_eq!(folded.len(), 1, "one warning for the package: {warned:?}");
assert!(folded.iter().all(|w| w.contains("2 zip entries")), "{folded:?}");
}
#[test]
fn decompression_bomb_is_rejected() {
let z = zip_of(&[("word/document.xml", vec![b' '; 2_000_000])]);
@@ -429,6 +523,26 @@ mod tests {
assert!(Package::open_limited(&z, 10_000_000, 10_000_000).is_ok());
}
#[test]
fn entries_that_fail_their_check_still_count_against_the_total() {
// Each entry inflates in full before its CRC is checked; a failed one was dropped without
// being charged, so any number of them inflated (#3723).
let names: Vec<String> = (0..20).map(|i| format!("p{i}.bin")).collect();
let entries: Vec<(&str, Vec<u8>)> = names.iter().map(|n| (n.as_str(), vec![0u8; 1_000_000])).collect();
let mut z = zip_of(&entries);
let mut patched = 0;
for sig in [[0x50u8, 0x4B, 0x01, 0x02], [0x50, 0x4B, 0x03, 0x04]] {
let crc_at = if sig[2] == 1 { 16 } else { 14 };
let found: Vec<usize> = z.windows(4).enumerate().filter(|(_, w)| *w == sig).map(|(i, _)| i).collect();
for pos in found {
z[pos + crc_at] ^= 0xFF;
patched += 1;
}
}
assert_eq!(patched, 40);
assert!(matches!(Package::open_limited(&z, 2_000_000, 4_000_000), Err(DocxError::Limit(_))));
}
#[test]
fn lying_declared_size_is_rejected_or_bounded() {
let mut z = zip_of(&[("a.xml", b"<a/>".to_vec())]);
crates/docx/src/read/mod.rs
+5 -1
View File
@@ -1363,6 +1363,10 @@ impl Reader<'_> {
let mut by_para: HashMap<String, u32> = HashMap::new();
let mut next_id = 0u32;
let mut entries: BTreeMap<u32, Comment> = BTreeMap::new();
// Ids given before this part, looked up rather than scanned for every generated id: a
// scan per comment made many non-numeric ids quadratic (#3692). Ids given here are in
// `entries`.
let given: HashSet<u32> = self.comment_map.values().copied().collect();
for c in root.children("w:comment") {
if self.doc.parts.len() >= MAX_PARTS {
break;
@@ -1374,7 +1378,7 @@ impl Reader<'_> {
let id = match file_id.parse::<u32>() {
Ok(v) if !entries.contains_key(&v) => v,
_ => {
while entries.contains_key(&next_id) || self.comment_map.values().any(|v| *v == next_id) {
while entries.contains_key(&next_id) || given.contains(&next_id) {
next_id += 1;
}
next_id
crates/docx/src/test_log.rs
+36
View File
@@ -0,0 +1,36 @@
//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that
//! a limit which leaves something out says so.
use std::cell::RefCell;
struct Capture;
thread_local! {
static LINES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
impl log::Log for Capture {
fn enabled(&self, m: &log::Metadata) -> bool {
m.level() <= log::Level::Warn
}
fn log(&self, r: &log::Record) {
if self.enabled(r.metadata()) {
LINES.with(|l| l.borrow_mut().push(r.args().to_string()));
}
}
fn flush(&self) {}
}
static CAPTURE: Capture = Capture;
/// `f`'s result and the warnings logged on this thread while it ran.
pub(crate) fn warnings<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
// Only this logger is ever installed in a test binary; a second call keeps it.
let _ = log::set_logger(&CAPTURE);
log::set_max_level(log::LevelFilter::Warn);
LINES.with(|l| l.borrow_mut().clear());
let out = f();
(out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut())))
}
crates/docx/src/write/embed.rs
+160 -2
View File
@@ -48,6 +48,10 @@ pub(crate) struct EmbedWriter<'d> {
pub parts: Vec<(String, Vec<u8>, String, PartRels)>,
/// Leave out the main document's kept glossary (it's written from the model instead).
pub drop_glossary: bool,
/// Kept parts written with these bytes instead of their own (see [`Self::scrub_glossary`]).
scrubbed: HashMap<String, Vec<u8>>,
/// A chart past the chart limits has been written (and the loss logged, once per save).
chart_cut: bool,
}
impl<'d> EmbedWriter<'d> {
@@ -93,6 +97,8 @@ impl<'d> EmbedWriter<'d> {
markup_bytes: MAX_MARKUP_BYTES,
parts: Vec::new(),
drop_glossary: false,
scrubbed: HashMap::new(),
chart_cut: false,
}
}
@@ -230,6 +236,44 @@ impl<'d> EmbedWriter<'d> {
})
}
/// Have the main document's kept glossary, and the parts it reaches, written without the
/// personal information (`names`) and dates the privacy settings remove (#3692). False when a
/// part's markup can't be rewritten: the glossary is then written from the model.
pub fn scrub_glossary(&mut self, names: bool, dates: bool) -> bool {
let Some(roots) = self.passthrough.get(crate::package::UNMODELLED_ROOTS) else { return true };
let mut todo: Vec<String> = String::from_utf8_lossy(roots)
.lines()
.take(MAX_EMBEDDED_RELS * 2)
.filter_map(|line| match line.split('\t').collect::<Vec<_>>().as_slice() {
[source, kind, target] if *source == "D" && rel_is(kind, rt::GLOSSARY) => Some(target.to_string()),
_ => None,
})
.collect();
let mut seen: HashSet<String> = HashSet::new();
while let Some(p) = todo.pop() {
if seen.len() >= MAX_EMBEDDED_PARTS || !seen.insert(p.clone()) {
continue;
}
let (Some(info), Some(bytes)) = (self.manifest.parts.get(&p), self.passthrough.get(&p)) else { continue };
todo.extend(info.rels.iter().filter(|r| !r.external).map(|r| r.target.clone()));
if !(info.content_type.ends_with("xml") || p.to_ascii_lowercase().ends_with(".xml")) {
continue;
}
match scrub_private(bytes, names, dates) {
Some(b) => {
self.scrubbed.insert(p, b);
}
None => {
log::warn!(
"docx: the glossary's part {p} can't be cleaned of personal information; the glossary is written from its entries, without what WordCraft doesn't model"
);
return false;
}
}
}
true
}
/// What copying the parts reached from `roots` costs against the copy budget.
fn tree_cost(&self, roots: &[&str]) -> u64 {
let mut seen: HashSet<&str> = HashSet::new();
@@ -283,8 +327,9 @@ impl<'d> EmbedWriter<'d> {
rels.add_with_id(&r.id, &r.kind, &relative(&new, &target), false);
}
let ct = if info.content_type.is_empty() { "application/octet-stream".to_string() } else { info.content_type };
let bytes = self.scrubbed.get(&orig).cloned().unwrap_or_else(|| bytes.to_vec());
written.push(self.parts.len());
self.parts.push((new, bytes.to_vec(), ct, rels));
self.parts.push((new, bytes, ct, rels));
}
Some(name)
}
@@ -408,7 +453,12 @@ impl super::Writer<'_> {
/// related from the story part through `rels`.
pub(super) fn chart_graphic(&mut self, spec: &wordcraft_doc::chart::ChartSpec, rels: &mut PartRels) -> String {
let name = self.embeds.fresh("word/charts/chart", ".xml");
let bytes = crate::chart_spec::chart_xml(spec).into_bytes();
let (xml, cut) = crate::chart_spec::chart_part(spec);
if cut && !self.embeds.chart_cut {
self.embeds.chart_cut = true;
log::warn!("docx: a chart has more series or points than a chart part holds; the rest are left out of the file");
}
let bytes = xml.into_bytes();
self.embeds.parts.push((name.clone(), bytes, crate::chart_spec::CHART_CT.to_string(), PartRels::default()));
let id = rels.add(crate::package::rt::CHART, &relative(STORY_PART, &name), false);
let mut w = xml::W::default();
@@ -640,6 +690,80 @@ fn patch_rel_ids(bytes: &[u8], ids: &HashMap<String, String>) -> Vec<u8> {
out
}
/// `bytes` (kept XML) without the personal information and dates the privacy settings remove, as
/// the writer leaves them out of the parts it writes: annotation authors become "Author" and
/// comment initials "A" (`names`); annotation dates and UTC dates go (`dates`); and with `names`,
/// a people part's authors become "Author" and their presence information goes. Attributes are
/// matched by namespace, so any prefix is caught. `None` when the markup can't be rewritten
/// (malformed, or not UTF-8).
fn scrub_private(bytes: &[u8], names: bool, dates: bool) -> Option<Vec<u8>> {
use std::borrow::Cow;
use quick_xml::events::Event;
use quick_xml::events::attributes::Attribute;
use quick_xml::name::{QName, ResolveResult};
use quick_xml::reader::NsReader;
use quick_xml::writer::Writer;
const WML: &[u8] = b"http://schemas.openxmlformats.org/wordprocessingml/2006/main";
const WML_STRICT: &[u8] = b"http://purl.oclc.org/ooxml/wordprocessingml/main";
const W15: &[u8] = b"http://schemas.microsoft.com/office/word/2012/wordml";
// UTF-16 (or other wide encodings) would pass through unread.
if bytes.contains(&0) || bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {
return None;
}
let ns_is = |r: &ResolveResult, ns: &[u8]| matches!(r, ResolveResult::Bound(n) if n.as_ref() == ns);
let wml = |r: &ResolveResult| ns_is(r, WML) || ns_is(r, WML_STRICT);
let mut r = NsReader::from_reader(bytes);
r.config_mut().check_end_names = false;
let mut out = Writer::new(Vec::with_capacity(bytes.len()));
let mut buf = Vec::new();
let mut skip = Vec::new();
loop {
buf.clear();
let (res, ev) = r.read_resolved_event_into(&mut buf).ok()?;
let (in_wml, in_w15) = (wml(&res), ns_is(&res, W15));
let ev = match ev {
Event::Eof => break,
Event::Start(ref e) | Event::Empty(ref e) if names && in_w15 && e.local_name().as_ref() == b"presenceInfo" => {
if matches!(ev, Event::Start(_)) {
let end = e.name().as_ref().to_vec();
r.read_to_end_into(QName(&end), &mut skip).ok()?;
skip.clear();
}
continue;
}
Event::Start(ref e) | Event::Empty(ref e) => {
let mut el = e.to_owned();
el.clear_attributes();
for a in e.attributes().with_checks(false) {
let a = a.ok()?;
let (ares, local) = r.resolver_mut().resolve_attribute(a.key);
// An unprefixed attribute of a WordprocessingML element is read as its own.
let wml_attr = wml(&ares) || (matches!(ares, ResolveResult::Unbound) && in_wml);
let value: Option<&[u8]> = match local.as_ref() {
b"author" if names && (wml_attr || ns_is(&ares, W15)) => Some(super::ANONYMOUS_AUTHOR.as_bytes()),
b"initials" if names && wml_attr => Some(super::ANONYMOUS_INITIALS.as_bytes()),
b"date" if dates && wml_attr => None,
b"dateUtc" if dates => None,
_ => {
el.push_attribute(a);
continue;
}
};
if let Some(v) = value {
el.push_attribute(Attribute { key: a.key, value: Cow::Borrowed(v) });
}
}
if matches!(ev, Event::Start(_)) { Event::Start(el) } else { Event::Empty(el) }
}
ev => ev,
};
out.write_event(ev).ok()?;
}
Some(out.into_inner())
}
/// A VML `style` with the object's size, rotation and flips, and its offset when it floats
/// (`position:absolute`).
fn vml_style_with(style: &str, w: f32, h: f32, float: &Float) -> String {
@@ -709,6 +833,40 @@ fn valid_part_name(name: &str) -> String {
mod tests {
use super::*;
#[test]
fn charts_cut_to_the_limits_on_save_say_so_once() {
// A model deserialised as is can hold more than a chart part keeps; the save left the rest
// out without a word (review of #3872).
use std::sync::Arc;
use wordcraft_doc::chart::{ChartSeries, ChartSpec, ChartType, MAX_SERIES};
use wordcraft_doc::graphic::{Graphic, GraphicKind};
use wordcraft_doc::{CharProps, InlineObject, Pos};
let chart = |series: usize| {
let spec = ChartSpec {
kind: ChartType::Column,
categories: vec!["a".into(), "b".into()],
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(),
title: None,
legend: None,
data_labels: false,
};
let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() };
InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) }
};
let save = |charts: Vec<InlineObject>| {
let mut doc = wordcraft_doc::Document::new();
for c in charts {
doc.insert_object(&Pos::body(0, 0), c, &CharProps::default()).unwrap();
}
crate::test_log::warnings(|| crate::write(&doc).unwrap()).1
};
let cut = |w: &Vec<String>| w.iter().filter(|w| w.contains("chart")).count();
let warned = save(vec![chart(MAX_SERIES + 1), chart(MAX_SERIES + 5), chart(2)]);
assert_eq!(cut(&warned), 1, "one warning for the save, not one per chart: {warned:?}");
let warned = save(vec![chart(3)]);
assert_eq!(cut(&warned), 0, "nothing left out: {warned:?}");
}
#[test]
fn data_part_drawing_ids_are_rewritten_once() {
let ids: HashMap<String, String> = [("rId9".to_string(), "rId2".to_string()), ("rId2".to_string(), "rId3".to_string())].into_iter().collect();
crates/docx/src/write/mod.rs
+8 -2
View File
@@ -272,7 +272,13 @@ pub(crate) fn write_parts(doc: &Document, flavor: Flavor) -> Result<Vec<(String,
// The glossary: kept verbatim while its entries are as read and its parts were kept, else
// written from the model (before the notes: references its entries hold are collected too).
if glossary::edited(doc) || (!doc.glossary.is_empty() && !wr.embeds.kept_glossary()) {
// When the privacy settings ask for changes, its kept parts are written without the names and
// dates of its tracked changes and comments (#3692), or from the model if they can't be.
let (names, dates) = (doc.settings.remove_personal_info, doc.settings.remove_date_time);
if glossary::edited(doc)
|| (!doc.glossary.is_empty() && !wr.embeds.kept_glossary())
|| ((names || dates) && !wr.embeds.scrub_glossary(names, dates))
{
wr.embeds.drop_glossary = true;
wr.glossary(&mut entries, &mut overrides, &mut rels);
}
@@ -1231,7 +1237,7 @@ fn theme_xml(doc: &Document) -> Vec<u8> {
/// The author the saved file gives a comment or tracked change when the document removes personal
/// information on save (`w:removePersonalInformation`), as Word does.
pub(crate) const ANONYMOUS_AUTHOR: &str = "Author";
const ANONYMOUS_INITIALS: &str = "A";
pub(crate) const ANONYMOUS_INITIALS: &str = "A";
/// The `w:author` written for an annotation by `author`: "Author" when the document removes
/// personal information on save, "Unknown" when it has none.
crates/docx/tests/it/glossary.rs
+125
View File
@@ -234,6 +234,131 @@ fn edited_glossary_is_written_from_the_model() {
assert_eq!(twice.get(&target), files.get(&target));
}
#[test]
fn an_unchanged_glossary_follows_the_privacy_settings() {
// An untouched glossary was written back byte for byte, so the names and dates of its
// tracked changes survived a save that removes personal information (#3692).
let tracked = r#"<w:docPart><w:docPartPr><w:name w:val="Clause"/><w:category><w:name w:val="General"/><w:gallery w:val="docParts"/></w:category></w:docPartPr><w:docPartBody><w:p><w:ins w:id="1" w:author="Quill Ramsey" w:date="2026-03-04T05:06:07Z"><w:r><w:t>Added clause</w:t></w:r></w:ins></w:p></w:docPartBody></w:docPart>"#;
let bytes = package(
&[("rId7", GLOSSARY_REL, "glossary/document.xml")],
&[
("[Content_Types].xml", content_types(&[("/word/glossary/document.xml", GLOSSARY_CT)])),
("word/glossary/document.xml", glossary_xml(tracked)),
],
);
let doc = wordcraft_docx::read(&bytes).unwrap();
let as_read = save(&doc);
let kept = text(&as_read, &glossary_target(&as_read).expect("a glossary"));
assert!(kept.contains("Quill Ramsey"), "the fixture's glossary is kept verbatim: {kept}");
for (personal, dates) in [(true, false), (false, true), (true, true)] {
let mut private = doc.clone();
private.settings.remove_personal_info = personal;
private.settings.remove_date_time = dates;
let files = save(&private);
let g = text(&files, &glossary_target(&files).expect("a glossary"));
assert!(g.contains("Added clause"), "{g}");
assert_eq!(g.contains("Quill Ramsey"), !personal, "{g}");
assert_eq!(g.contains("2026-03-04T05:06:07Z"), !dates, "{g}");
}
}
#[test]
fn a_private_save_keeps_the_glossary_entries_the_model_passed_over() {
// The privacy settings had the glossary written from the model, which holds only the
// entries the reader took: one without a name was lost, and with no other entry the whole
// glossary went. The kept parts are now written without the names and dates instead.
let nameless = r#"<w:docPart><w:docPartPr><w:category><w:name w:val="General"/><w:gallery w:val="docParts"/></w:category></w:docPartPr><w:docPartBody><w:p><w:ins w:id="1" w:author="Quill Ramsey" w:date="2026-03-04T05:06:07Z"><w:r><w:t>Nameless entry</w:t></w:r></w:ins><w:commentRangeStart w:id="0"/><w:r><w:t>noted</w:t></w:r><w:commentRangeEnd w:id="0"/><w:r><w:commentReference w:id="0"/></w:r></w:p></w:docPartBody></w:docPart>"#;
// The comments part binds WordprocessingML to another prefix: names are matched by namespace.
let comments = format!(
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><x:comments xmlns:x="{W}"><x:comment x:id="0" x:author="Quill Ramsey" x:date="2026-03-04T05:06:07Z" x:initials="QR"><x:p><x:r><x:t>A note</x:t></x:r></x:p></x:comment></x:comments>"#
);
let bytes = package(
&[("rId7", GLOSSARY_REL, "glossary/document.xml")],
&[
(
"[Content_Types].xml",
content_types(&[
("/word/glossary/document.xml", GLOSSARY_CT),
("/word/glossary/comments.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"),
]),
),
("word/glossary/document.xml", glossary_xml(nameless)),
("word/glossary/_rels/document.xml.rels", rels(&[("rId1", &format!("{REL}/comments"), "comments.xml")])),
("word/glossary/comments.xml", comments),
],
);
let doc = wordcraft_docx::read(&bytes).unwrap();
assert!(doc.glossary.is_empty(), "the reader passes over an entry without a name");
for (personal, dates) in [(true, false), (false, true), (true, true)] {
let mut private = doc.clone();
private.settings.remove_personal_info = personal;
private.settings.remove_date_time = dates;
let files = save(&private);
let target = glossary_target(&files).expect("the glossary survives");
let g = text(&files, &target);
assert!(g.contains("Nameless entry"), "{g}");
assert_eq!(g.contains("Quill Ramsey"), !personal, "{g}");
assert_eq!(g.contains("2026-03-04T05:06:07Z"), !dates, "{g}");
let c = files.iter().find(|(n, _)| n.starts_with("word/glossary/comments")).map(|(_, b)| String::from_utf8_lossy(b).into_owned());
let c = c.expect("the glossary's comments survive");
assert!(c.contains("A note"), "{c}");
assert_eq!(c.contains("Quill Ramsey"), !personal, "{c}");
assert_eq!(c.contains("x:initials=\"QR\""), !personal, "{c}");
assert_eq!(c.contains("2026-03-04T05:06:07Z"), !dates, "{c}");
}
}
#[test]
fn a_kept_glossary_that_cant_be_rewritten_privately_is_written_from_the_model() {
// UTF-16 markup isn't rewritten in place: the model, which applies the settings, is written.
let tracked = r#"<w:docPart><w:docPartPr><w:name w:val="Clause"/><w:category><w:name w:val="General"/><w:gallery w:val="docParts"/></w:category></w:docPartPr><w:docPartBody><w:p><w:ins w:id="1" w:author="Quill Ramsey" w:date="2026-03-04T05:06:07Z"><w:r><w:t>Added clause</w:t></w:r></w:ins></w:p></w:docPartBody></w:docPart>"#;
let utf16: Vec<u8> = [0xFF, 0xFE].into_iter().chain(glossary_xml(tracked).encode_utf16().flat_map(u16::to_le_bytes)).collect();
let bytes = zip(&[
("_rels/.rels", rels(&[("rIdMain", &format!("{REL}/officeDocument"), "word/document.xml")]).into_bytes()),
(
"word/document.xml",
format!(r#"<w:document xmlns:w="{W}"><w:body><w:p><w:r><w:t>Body</w:t></w:r></w:p></w:body></w:document>"#).into_bytes(),
),
("word/_rels/document.xml.rels", rels(&[("rId7", GLOSSARY_REL, "glossary/document.xml")]).into_bytes()),
("[Content_Types].xml", content_types(&[("/word/glossary/document.xml", GLOSSARY_CT)]).into_bytes()),
("word/glossary/document.xml", utf16),
]);
let mut doc = wordcraft_docx::read(&bytes).unwrap();
assert_eq!(doc.glossary.len(), 1);
doc.settings.remove_personal_info = true;
let (files, warned) = warnings(|| save(&doc));
let g = text(&files, &glossary_target(&files).expect("a glossary"));
assert!(g.contains("Added clause") && !g.contains("Quill Ramsey"), "{g}");
// What the model doesn't hold is lost: the save says so (review of #3872).
assert_eq!(warned.iter().filter(|w| w.contains("written from its entries")).count(), 1, "{warned:?}");
}
/// `f`'s result and the warnings logged on this thread while it ran.
fn warnings<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
use std::cell::RefCell;
struct Capture;
thread_local! {
static LINES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
impl log::Log for Capture {
fn enabled(&self, m: &log::Metadata) -> bool {
m.level() <= log::Level::Warn
}
fn log(&self, r: &log::Record) {
if self.enabled(r.metadata()) {
LINES.with(|l| l.borrow_mut().push(r.args().to_string()));
}
}
fn flush(&self) {}
}
static CAPTURE: Capture = Capture;
let _ = log::set_logger(&CAPTURE);
log::set_max_level(log::LevelFilter::Warn);
LINES.with(|l| l.borrow_mut().clear());
let out = f();
(out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut())))
}
/// The relationships part of a part in `word/`.
fn glossary_rels(part: &str) -> String {
let (dir, file) = part.rsplit_once('/').unwrap();
crates/docx/tests/it/malformed.rs
+35
View File
@@ -342,3 +342,38 @@ fn long_header_reference_chain_is_cut_off() {
assert!(!d.parts.is_empty());
assert!(d.parts.len() <= 8, "followed {} chained headers", d.parts.len());
}
/// #3642: a compound file whose DIFAT names one FAT sector thousands of times is refused by the
/// encrypted-package sniff (reached for any `.docx` name) before `cfb` builds its FAT.
#[test]
fn a_repeating_difat_is_refused_before_cfb_opens_it() {
let f = wordcraft_doc::cfb_guard::repeating_difat_sample();
assert!(!wordcraft_docx::is_encrypted(&f));
assert!(matches!(wordcraft_docx::decrypt(&f, None), Err(e) if e.to_string().contains("FAT sectors in a file of")));
}
#[test]
fn comments_with_non_numeric_ids_open_in_linear_time() {
// Each generated id was checked against every id given so far, so a comments part with many
// ids that aren't numbers took time quadratic in their count (#3692).
const RELS_NS: &str = "http://schemas.openxmlformats.org/package/2006/relationships";
const COMMENTS_REL: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments";
let n = 49_000;
let mut comments = format!("<w:comments {W_NS}>");
for i in 0..n {
comments.push_str(&format!(r#"<w:comment w:id="c{i}" w:author="A"><w:p/></w:comment>"#));
}
comments.push_str("</w:comments>");
let doc = format!("<w:document {W_NS}><w:body><w:p><w:r><w:t>body</w:t></w:r></w:p></w:body></w:document>");
let rels = format!(r#"<Relationships xmlns="{RELS_NS}"><Relationship Id="rId1" Type="{COMMENTS_REL}" Target="comments.xml"/></Relationships>"#);
let z = zip(&[
("_rels/.rels", ROOT_RELS.as_bytes()),
("word/document.xml", doc.as_bytes()),
("word/_rels/document.xml.rels", rels.as_bytes()),
("word/comments.xml", comments.as_bytes()),
]);
let d = read(&z).unwrap();
assert_eq!(d.comments.len(), n);
let ids: std::collections::HashSet<_> = d.comments.keys().collect();
assert_eq!(ids.len(), n, "every comment has its own id");
}
crates/docx/tests/it/passthrough.rs
+46
View File
@@ -357,3 +357,49 @@ fn extended_properties_are_kept_without_words_counts_and_personal_ones_follow_th
assert!(!private.contains("Company") && !private.contains("Manager"), "{private}");
assert!(private.contains("<Template>Normal.dotm</Template>"));
}
/// `name` with its letters' case flipped by the bits of `i`.
fn spelling(name: &str, i: u32) -> String {
let mut bit = 0;
name.chars()
.map(|c| {
if !c.is_ascii_alphabetic() {
return c;
}
bit += 1;
if i >> (bit - 1) & 1 == 1 { c.to_ascii_uppercase() } else { c.to_ascii_lowercase() }
})
.collect()
}
#[test]
fn a_part_reached_under_many_spellings_is_kept_and_written_once() {
// Part names are case-insensitive (ECMA-376 Part 2 §6.2.2.3): 64 relationships naming one picture
// and one unmodelled part in 64 casings are two parts, not 128 copies on save (#3704).
let ns = r#"xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture""#;
let n = 64;
let mut body = String::new();
let mut doc_rels = Vec::new();
for i in 0..n {
body.push_str(&format!(
r#"<w:p><w:r><w:drawing><wp:inline {ns}><wp:extent cx="9525" cy="9525"/><wp:docPr id="{}" name="p"/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/picture"><pic:pic><pic:blipFill><a:blip r:embed="rIdP{i}"/></pic:blipFill></pic:pic></a:graphicData></a:graphic></wp:inline></w:drawing></w:r></w:p>"#,
i + 1
));
doc_rels.push((format!("rIdP{i}"), format!("{REL}/image"), spelling("media/image1.png", i)));
doc_rels.push((format!("rIdU{i}"), "urn:example:future-relationship".to_string(), spelling("unknownPart.xml", i)));
}
let doc_rels: Vec<(&str, &str, &str)> = doc_rels.iter().map(|(a, b, c)| (a.as_str(), b.as_str(), c.as_str())).collect();
let picture = format!("\u{89}PNG\r\n\u{1a}\n{}", "0".repeat(64 * 1024));
let unknown = format!(r#"<x:thing xmlns:x="urn:example:future">{}</x:thing>"#, "x".repeat(64 * 1024));
let ct = CT.replace("<Default Extension=\"jpeg\"", "<Default Extension=\"png\" ContentType=\"image/png\"/><Default Extension=\"jpeg\"");
let bytes =
package(&body, &doc_rels, &[], &[("[Content_Types].xml", &ct), ("word/media/image1.png", &picture), ("word/unknownPart.xml", &unknown)]);
let doc = wordcraft_docx::read(&bytes).unwrap();
assert_eq!(doc.media.len(), 1, "{:?}", doc.media.keys());
let files = save(&doc);
let named = |s: &str| files.keys().filter(|k| k.to_ascii_lowercase().contains(s)).count();
assert_eq!(named("unknownpart"), 1, "{:?}", files.keys());
assert_eq!(files.keys().filter(|k| k.starts_with("word/media/")).count(), 1, "{:?}", files.keys());
let written: usize = files.values().map(Vec::len).sum();
assert!(written < 4 * 64 * 1024, "{written} bytes written");
}
crates/engine/src/cmd/font.rs
+35
View File
@@ -43,6 +43,15 @@ fn read_font_file(path: &str) -> Result<Vec<u8>, CmdError> {
crate::io::read_file_capped(path, wordcraft_fonts::MAX_FONT_FILE_BYTES as u64, "font file").map_err(CmdError::Failed)
}
/// Adding or removing a font writes to the user font folder: automation confined to a write root
/// (#769) may do that only when the folder is inside it.
fn check_font_store(dir: Option<&std::path::Path>) -> Result<(), CmdError> {
match dir {
Some(dir) => crate::file_access::check_write(&dir.to_string_lossy()).map_err(CmdError::Failed),
None => Ok(()),
}
}
fn add(s: &mut Session, v: &Value) -> CmdResult {
if p::str(v, "url").is_some() {
return Err(CmdError::Failed(
@@ -71,6 +80,7 @@ fn add(s: &mut Session, v: &Value) -> CmdResult {
s.push_ui_request(json!({"open": "addFont"}));
return Ok(json!({"pending": true}));
};
check_font_store(FontDb::global().user_font_dir().as_deref())?;
let r = FontDb::global().install_font(&name, &bytes).map_err(|e| CmdError::Failed(e.to_string()))?;
// Text already set in the family (shown in a fallback until now) takes the new font.
s.relayout();
@@ -100,8 +110,33 @@ fn list(_: &mut Session, v: &Value) -> CmdResult {
fn remove(s: &mut Session, v: &Value) -> CmdResult {
let family = p::str(v, "family").filter(|f| !f.trim().is_empty()).ok_or_else(|| CmdError::Params("family is required".into()))?;
check_font_store(FontDb::global().user_font_dir().as_deref())?;
let gone = FontDb::global().remove_user_font(family.trim()).map_err(|e| CmdError::Failed(e.to_string()))?;
s.relayout();
s.status = format!("Removed {}", family.trim());
Ok(json!({"removed": gone.iter().map(font_json).collect::<Vec<_>>()}))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn confined_automation_changes_only_a_font_store_inside_its_write_root() {
let root = std::env::temp_dir().join(format!("wordcraft-font-store-{}", std::process::id()));
let outside = std::env::temp_dir().join(format!("wordcraft-font-store-out-{}", std::process::id()));
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(&outside).unwrap();
let write = crate::file_access::AutomationRoots::new(None, Some(&root)).unwrap();
let read_only = crate::file_access::AutomationRoots::new(Some(&root), None).unwrap();
crate::file_access::confine(write.as_ref(), || {
assert!(check_font_store(None).is_ok(), "no folder: the font lasts for the session");
assert!(check_font_store(Some(&root.join("fonts"))).is_ok());
assert!(check_font_store(Some(&outside)).is_err());
});
crate::file_access::confine(read_only.as_ref(), || assert!(check_font_store(Some(&root)).is_err()));
assert!(check_font_store(Some(&outside)).is_ok(), "unconfined: the person at the keyboard");
let _ = std::fs::remove_dir_all(&root);
let _ = std::fs::remove_dir_all(&outside);
}
}
crates/fonts/Cargo.toml
+1
View File
@@ -17,6 +17,7 @@ kurbo = { workspace = true }
skrifa = { workspace = true }
harfrust = { workspace = true }
wuff = { workspace = true }
brotli-decompressor = { workspace = true }
unicode-bidi = { workspace = true }
unicode-script = { workspace = true }
log = { workspace = true }
crates/fonts/src/color.rs
+101 -13
View File
@@ -18,6 +18,9 @@ use crate::FontFace;
/// Upper bound on the ops one colour glyph may expand to.
pub const MAX_OPS: usize = 20_000;
/// Upper bound on the gradient stops and clip-path elements one colour glyph may copy in total: a
/// paint graph can reference one long stop list or outline from every op.
pub const MAX_PAYLOAD: usize = 1 << 16;
/// Upper bound on a bitmap glyph's width and height in pixels.
pub const MAX_BITMAP_SIDE: u32 = 2048;
@@ -124,10 +127,10 @@ impl FontFace {
if let Some(cg) = ColorGlyphCollection::new(&f).get(glyph) {
let palette: Vec<[u8; 4]> =
ColorPalettes::new(&f).get(0).map(|p| p.colors().iter().map(|c| [c.red, c.green, c.blue, c.alpha]).collect()).unwrap_or_default();
let mut rec = Recorder::new(&palette, |g: GlyphId| {
let mut p = self.glyph_outline(g.to_u32());
let mut rec = Recorder::new(&palette, |g: GlyphId, room: usize| {
let mut p = self.glyph_outline_capped(g.to_u32(), room)?;
p.apply_affine(Affine::FLIP_Y);
p
Some(p)
});
if cg.paint(self.location(), &mut rec).is_ok()
&& let Some(ops) = rec.finish()
@@ -175,20 +178,34 @@ pub fn bitmap_placement(bg: &skrifa::bitmap::BitmapGlyph<'_>, upem: f64) -> Opti
finite.then_some(ColorBitmap { data, width: w, height: h, rect, ppem: bg.ppem_y })
}
/// Approximate heap bytes held by `ops`, counting each op's copied stops and path elements.
pub fn ops_bytes(ops: &[ColorOp]) -> usize {
let payload = |op: &ColorOp| match op {
ColorOp::PushClip(p) => p.elements().len().saturating_mul(std::mem::size_of::<kurbo::PathEl>()),
ColorOp::Fill(ColorPaint::Linear { stops, .. } | ColorPaint::Radial { stops, .. } | ColorPaint::Sweep { stops, .. }) => {
stops.len().saturating_mul(std::mem::size_of::<Stop>())
}
_ => 0,
};
ops.iter().fold(ops.len().saturating_mul(std::mem::size_of::<ColorOp>()), |n, op| n.saturating_add(payload(op)))
}
/// A [`ColorPainter`] that records the paint calls as [`ColorOp`]s, resolving palette colours and
/// clip glyphs. Stops recording (and [`Recorder::finish`] gives None) past [`MAX_OPS`].
pub struct Recorder<'a, F: Fn(GlyphId) -> BezPath> {
/// clip glyphs. Stops recording (and [`Recorder::finish`] gives None) past [`MAX_OPS`] or
/// [`MAX_PAYLOAD`].
pub struct Recorder<'a, F: Fn(GlyphId, usize) -> Option<BezPath>> {
palette: &'a [[u8; 4]],
outline: F,
ops: Vec<ColorOp>,
payload: usize,
overflow: bool,
}
impl<'a, F: Fn(GlyphId) -> BezPath> Recorder<'a, F> {
impl<'a, F: Fn(GlyphId, usize) -> Option<BezPath>> Recorder<'a, F> {
/// A recorder with palette `palette` (straight RGBA) and clip outlines (font units, y up) from
/// `outline`.
/// `outline`, which is given the path elements left in the budget and gives None past them.
pub fn new(palette: &'a [[u8; 4]], outline: F) -> Self {
Recorder { palette, outline, ops: vec![], overflow: false }
Recorder { palette, outline, ops: vec![], payload: 0, overflow: false }
}
/// The recorded ops, None when the glyph was too big or its pushes and pops don't balance.
@@ -218,6 +235,15 @@ impl<'a, F: Fn(GlyphId) -> BezPath> Recorder<'a, F> {
self.ops.push(op);
}
/// Charge `n` stops or path elements to the glyph's payload; false (and overflow) past the budget.
fn charge(&mut self, n: usize) -> bool {
self.payload = self.payload.saturating_add(n);
if self.payload > MAX_PAYLOAD {
self.overflow = true;
}
!self.overflow
}
fn color(&self, palette_index: u16, alpha: f32) -> GlyphColor {
let alpha = if alpha.is_finite() { alpha.clamp(0.0, 1.0) } else { 1.0 };
if palette_index == 0xFFFF {
@@ -241,7 +267,7 @@ fn pt(p: skrifa::raw::types::Point<f32>) -> Point {
Point::new(p.x as f64, p.y as f64)
}
impl<F: Fn(GlyphId) -> BezPath> ColorPainter for Recorder<'_, F> {
impl<F: Fn(GlyphId, usize) -> Option<BezPath>> ColorPainter for Recorder<'_, F> {
fn push_transform(&mut self, t: Transform) {
self.push(ColorOp::PushTransform(Affine::new([t.xx, t.yx, t.xy, t.yy, t.dx, t.dy].map(|v| v as f64))));
}
@@ -249,8 +275,17 @@ impl<F: Fn(GlyphId) -> BezPath> ColorPainter for Recorder<'_, F> {
self.push(ColorOp::PopTransform);
}
fn push_clip_glyph(&mut self, glyph_id: GlyphId) {
let p = (self.outline)(glyph_id);
self.push(ColorOp::PushClip(p));
if self.overflow {
return;
}
// Built within what is left of the budget: a huge outline is refused while it is drawn.
let Some(p) = (self.outline)(glyph_id, MAX_PAYLOAD.saturating_sub(self.payload)) else {
self.overflow = true;
return;
};
if self.charge(p.elements().len()) {
self.push(ColorOp::PushClip(p));
}
}
fn push_clip_box(&mut self, b: BoundingBox<f32>) {
let r = Rect::new(b.x_min as f64, b.y_min as f64, b.x_max as f64, b.y_max as f64);
@@ -260,6 +295,15 @@ impl<F: Fn(GlyphId) -> BezPath> ColorPainter for Recorder<'_, F> {
self.push(ColorOp::PopClip);
}
fn fill(&mut self, brush: Brush<'_>) {
let stops = match &brush {
Brush::Solid { .. } => 0,
Brush::LinearGradient { color_stops, .. } | Brush::RadialGradient { color_stops, .. } | Brush::SweepGradient { color_stops, .. } => {
color_stops.len()
}
};
if !self.charge(stops) {
return;
}
let paint = match brush {
Brush::Solid { palette_index, alpha } => ColorPaint::Solid(self.color(palette_index, alpha)),
Brush::LinearGradient { p0, p1, color_stops, extend } => {
@@ -286,8 +330,8 @@ impl<F: Fn(GlyphId) -> BezPath> ColorPainter for Recorder<'_, F> {
mod tests {
use super::*;
fn square(_: GlyphId) -> BezPath {
kurbo::Shape::to_path(&Rect::new(0.0, 0.0, 10.0, 10.0), 0.1)
fn square(_: GlyphId, room: usize) -> Option<BezPath> {
Some(kurbo::Shape::to_path(&Rect::new(0.0, 0.0, 10.0, 10.0), 0.1)).filter(|p| p.elements().len() <= room)
}
#[test]
@@ -337,6 +381,50 @@ mod tests {
assert!(r.finish().is_none());
}
#[test]
fn gradient_stops_and_clip_paths_share_a_payload_budget() {
// A few hundred fills reusing one long stop list would copy it per fill, far under MAX_OPS.
let stops = vec![ColorStop { offset: 0.5, palette_index: 0, alpha: 1.0 }; 4096];
let fills = MAX_PAYLOAD / stops.len() + 1;
assert!(fills < MAX_OPS);
let outlines = std::cell::Cell::new(0);
let count = |g, room| {
outlines.set(outlines.get() + 1);
square(g, room)
};
let mut r = Recorder::new(&[], count);
for _ in 0..fills {
r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad });
}
assert!(r.ops.len() < fills, "stops past the budget are not copied");
r.push_clip_glyph(GlyphId::new(1));
r.pop_clip();
assert_eq!(outlines.get(), 0, "no clip outline is built once the budget is spent");
assert!(r.finish().is_none());
// Within budget nothing changes, and the byte estimate counts the copied stops.
let mut r = Recorder::new(&[], square);
r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad });
let ops = r.finish().unwrap();
assert!(ops_bytes(&ops) >= stops.len() * std::mem::size_of::<Stop>());
}
#[test]
fn a_clip_outline_is_drawn_within_the_payload_left() {
// #3716 review: the outline is told the room left, so an oversized one is never built whole.
let stops = vec![ColorStop { offset: 0.5, palette_index: 0, alpha: 1.0 }; MAX_PAYLOAD - 3];
let rooms = std::cell::RefCell::new(vec![]);
let seen = |g, room| {
rooms.borrow_mut().push(room);
square(g, room)
};
let mut r = Recorder::new(&[], seen);
r.fill(Brush::LinearGradient { p0: Default::default(), p1: Default::default(), color_stops: &stops, extend: Extend::Pad });
r.push_clip_glyph(GlyphId::new(1));
r.pop_clip();
assert_eq!(*rooms.borrow(), [3]);
assert!(r.finish().is_none(), "a square needs more than the 3 elements left");
}
fn bitmap(origin: skrifa::bitmap::Origin, data: skrifa::bitmap::BitmapData<'_>) -> skrifa::bitmap::BitmapGlyph<'_> {
skrifa::bitmap::BitmapGlyph {
data,
crates/fonts/src/fontdb.rs
+304 -35
View File
@@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex, RwLock};
use kurbo::BezPath;
use skrifa::instance::{Location, LocationRef, Size};
use skrifa::outline::{DrawSettings, Engine, GlyphStyles, HintingInstance, HintingOptions, OutlineGlyphFormat, OutlinePen, SmoothMode, Target};
use skrifa::outline::{DrawSettings, Engine, GlyphStyles, Hinting, HintingInstance, HintingOptions, OutlineGlyphFormat, OutlinePen, SmoothMode, Target};
use skrifa::raw::FileRef;
use skrifa::string::StringId;
use skrifa::{GlyphId, MetadataProvider};
@@ -92,8 +92,44 @@ pub struct FontFace {
outline_hinter: std::sync::OnceLock<Option<HintingInstance>>,
/// The autohinter's analysis of the font for screen hinting, made once for every size.
glyph_styles: std::sync::OnceLock<GlyphStyles>,
/// Hinted draws so far, those that ran too long, and whether that turned hinting off
/// ([`FontFace::count_hinted`]).
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
hint_draws: std::sync::atomic::AtomicU32,
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
hint_slow: std::sync::atomic::AtomicU32,
hinting_off: std::sync::atomic::AtomicBool,
}
/// Limits on what a face's TrueType instructions may run over (#3692). skrifa's interpreter budget
/// counts instructions, not the points each one moves (SHZ, IUP… walk a whole zone), so a font with
/// huge zones turns a short program into billions of point moves. Real fonts stay far below: a
/// twilight zone of tens to hundreds of points, a stack of up to ~2,000 values, a few thousand
/// points a glyph. Past them, glyphs are drawn unhinted.
const MAX_HINT_TWILIGHT: u16 = 1024;
const MAX_HINT_STACK: u16 = 4096;
/// skrifa's buffer for one hinted glyph (points, stack, twilight zone, CVT and storage).
const MAX_HINT_GLYPH_BYTES: usize = 256 * 1024;
/// A hinted glyph slower than this is a strike against its face. A face under the limits above can
/// still take seconds for one glyph, so [`MIN_HINT_STRIKES`] strikes that are also at least one
/// in [`HINT_SLOW_RATIO`] of its hinted draws turn hinting off: a face that is slow throughout
/// stops after a few glyphs, while the odd draw slowed by a busy machine leaves a face hinted.
///
/// Wall-clock, not interpreter work: skrifa's own budget (a million instructions a program, not
/// settable) counts instructions, not the points each one moves, and it exposes no count of the
/// work done, so time is the only measure there is (#3827). Whether such a face stays hinted can
/// therefore depend on the machine, by design: only faces that need the interpreter to assemble
/// their contours (legacy CJK fonts such as DFKai-SB) are hinted at all, and those typically hint
/// a glyph in microseconds, so 50 ms is far above an honest face on a loaded machine; a face
/// that trips it is drawn unhinted (its outlines possibly misassembled) rather than freezing the
/// app. Native only (no clock on wasm).
#[cfg(not(target_arch = "wasm32"))]
const HINT_STRIKE: std::time::Duration = std::time::Duration::from_millis(50);
#[cfg(not(target_arch = "wasm32"))]
const MIN_HINT_STRIKES: u32 = 3;
#[cfg(not(target_arch = "wasm32"))]
const HINT_SLOW_RATIO: u32 = 16;
/// A cheap `Copy` handle to a face. Faces are never unloaded, so the handle lives for the rest of
/// the process; unlike cloning an `Arc`, copying it touches no shared reference count (which made
/// parallel composition scale negatively).
@@ -178,18 +214,66 @@ impl FontFace {
pub fn id(&self) -> u32 {
self.id
}
/// Counts a hinted draw, `slow` when it ran past [`HINT_STRIKE`]; true when it is the one that
/// turns hinting off.
#[cfg(not(target_arch = "wasm32"))]
fn count_hinted(&self, slow: bool) -> bool {
let draws = self.hint_draws.fetch_add(1, Ordering::Relaxed).saturating_add(1);
if !slow {
return false;
}
let slow = self.hint_slow.fetch_add(1, Ordering::Relaxed).saturating_add(1);
slow >= MIN_HINT_STRIKES && slow.saturating_mul(HINT_SLOW_RATIO) >= draws && !self.hinting_off.swap(true, Ordering::Relaxed)
}
/// Glyph `gid`'s outline in font units, y-down (flipped), uncached: for faces the database
/// doesn't keep (font list previews); others go through [`FontDb::outline`]'s cache.
pub fn glyph_outline(&self, gid: u32) -> BezPath {
let mut pen = FlipPen(BezPath::new());
self.draw_outline(gid, usize::MAX).path
}
/// [`FontFace::glyph_outline`], or None when the outline has more than `max_elements` path
/// elements: drawing stops copying them at the limit, so a huge glyph is never built whole.
pub fn glyph_outline_capped(&self, gid: u32, max_elements: usize) -> Option<BezPath> {
let pen = self.draw_outline(gid, max_elements);
(!pen.over).then_some(pen.path)
}
fn draw_outline(&self, gid: u32, max_elements: usize) -> FlipPen {
let mut pen = FlipPen { path: BezPath::new(), room: max_elements, over: false };
if let Some(f) = self.skrifa()
&& let Some(g) = f.outline_glyphs().get(GlyphId::new(gid))
{
let hinter = self.outline_hinter.get_or_init(|| {
let hinter =
self.hinter().filter(|_| !self.hinting_off.load(Ordering::Relaxed) && g.draw_memory_size(Hinting::Embedded) <= MAX_HINT_GLYPH_BYTES);
#[cfg(not(target_arch = "wasm32"))]
let started = std::time::Instant::now();
let settings = hinter.map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false));
let drawn = g.draw(settings, &mut pen);
#[cfg(not(target_arch = "wasm32"))]
if hinter.is_some() && self.count_hinted(started.elapsed() > HINT_STRIKE) {
log::warn!("Hinting glyph {gid} in {:?} took {:?}: drawing the face unhinted", self.family, started.elapsed());
}
if let Err(e) = drawn {
log::warn!("Cannot draw glyph {gid} in {:?}: {e}", self.family);
pen.path = BezPath::new();
}
}
pen
}
/// The face's TrueType interpreter, for the fonts that need one to assemble their contours and
/// whose zones are within the limits ([`MAX_HINT_TWILIGHT`]).
fn hinter(&self) -> Option<&HintingInstance> {
self.outline_hinter
.get_or_init(|| {
use skrifa::raw::TableProvider;
let f = self.skrifa()?;
let outlines = f.outline_glyphs();
if !outlines.require_interpreter() {
return None;
}
let zones = f.maxp().ok().map(|m| (m.max_twilight_points().unwrap_or(0), m.max_stack_elements().unwrap_or(0)));
if !zones.is_some_and(|(twilight, stack)| twilight <= MAX_HINT_TWILIGHT && stack <= MAX_HINT_STACK) {
log::warn!("{:?} asks for hinting zones of {zones:?}: drawing it unhinted", self.family);
return None;
}
// At UPEM pixels, hinted coordinates remain in font units. This
// executes contour assembly without tying the cache to view zoom.
HintingInstance::new(
@@ -198,17 +282,10 @@ impl FontFace {
self.location(),
HintingOptions { engine: Engine::Interpreter, target: Target::Mono },
)
.map_err(|e| log::warn!("Cannot initialize outline interpreter for {}: {e}", self.family))
.map_err(|e| log::warn!("Cannot initialize outline interpreter for {:?}: {e}", self.family))
.ok()
});
let settings =
hinter.as_ref().map_or_else(|| DrawSettings::unhinted(Size::unscaled(), self.location()), |h| DrawSettings::hinted(h, false));
if let Err(e) = g.draw(settings, &mut pen) {
log::warn!("Cannot draw glyph {gid} in {}: {e}", self.family);
return BezPath::new();
}
}
pen.0
})
.as_ref()
}
/// Glyph `gid`'s outline hinted for screen text at `ppem` device pixels per em, in device
/// pixels, y-down (flipped), or `None` when the face can't be hinted. The hinting is light,
@@ -217,9 +294,9 @@ impl FontFace {
fn hinted_glyph_outline(&self, gid: u32, hinter: &HintingInstance) -> Option<BezPath> {
let f = self.skrifa()?;
let g = f.outline_glyphs().get(GlyphId::new(gid))?;
let mut pen = FlipPen(BezPath::new());
g.draw(DrawSettings::hinted(hinter, false), &mut pen).map_err(|e| log::warn!("Cannot hint glyph {gid} in {}: {e}", self.family)).ok()?;
Some(pen.0)
let mut pen = FlipPen { path: BezPath::new(), room: usize::MAX, over: false };
g.draw(DrawSettings::hinted(hinter, false), &mut pen).map_err(|e| log::warn!("Cannot hint glyph {gid} in {:?}: {e}", self.family)).ok()?;
Some(pen.path)
}
/// A light hinting instance for `ppem` device pixels per em, with the engine FreeType's light
@@ -243,7 +320,7 @@ impl FontFace {
(Engine::Interpreter, light)
};
HintingInstance::new(&outlines, Size::new(ppem), self.location(), HintingOptions { engine, target })
.map_err(|e| log::warn!("Cannot hint {} at {ppem} px: {e}", self.family))
.map_err(|e| log::warn!("Cannot hint {:?} at {ppem} px: {e}", self.family))
.ok()
}
@@ -875,7 +952,8 @@ fn font_faces(f: &skrifa::FontRef<'_>, index: u32) -> Vec<Found> {
let east_asian = east_asian_font(f);
let axes = f.axes();
let mut named: Vec<Found> = Vec::new();
for ni in f.named_instances().iter() {
// At most one more than a file may add, so a caller can tell there are more (#3716).
for ni in f.named_instances().iter().take(MAX_FILE_FACES + 1) {
let Some(style) = name(f, &[ni.subfamily_name_id()]) else { continue };
if named.iter().any(|n| n.style.eq_ignore_ascii_case(&style)) {
continue;
@@ -904,15 +982,25 @@ fn font_faces(f: &skrifa::FontRef<'_>, index: u32) -> Vec<Found> {
}]
}
/// Parse every face in `data` (a font file or collection); a variable font yields one face per
/// named instance.
fn enumerate_faces(data: &[u8]) -> Vec<Found> {
/// Most faces one font file adds (collection members, variable fonts' named instances).
const MAX_FILE_FACES: usize = 256;
/// Parse the faces in `data` (a font file or collection), at most `limit` of them; a variable
/// font yields one face per named instance. A face with the family and style of an earlier one
/// is left out (a collection can list one font many times, #3716).
fn enumerate_faces(data: &[u8], limit: usize) -> Vec<Found> {
let count = match FileRef::new(data) {
Ok(FileRef::Font(_)) => 1,
Ok(FileRef::Collection(c)) => c.len(),
Ok(FileRef::Collection(c)) => c.len().min(u32::try_from(limit).unwrap_or(u32::MAX)),
Err(_) => 0,
};
(0..count).filter_map(|i| skrifa::FontRef::from_index(data, i).ok().map(|f| font_faces(&f, i))).flatten().collect()
let mut seen = std::collections::HashSet::new();
(0..count)
.filter_map(|i| skrifa::FontRef::from_index(data, i).ok().map(|f| font_faces(&f, i)))
.flatten()
.filter(|f| seen.insert((f.family.to_ascii_lowercase(), f.style.to_ascii_lowercase())))
.take(limit)
.collect()
}
/// The compiled-in Source Sans 3 Regular, for a database that somehow has no faces at all.
@@ -975,6 +1063,9 @@ fn make_face(bytes: FontBytes, index: u32, family: String, style: String, legacy
bmp: std::sync::OnceLock::new(),
outline_hinter: std::sync::OnceLock::new(),
glyph_styles: std::sync::OnceLock::new(),
hint_draws: Default::default(),
hint_slow: Default::default(),
hinting_off: Default::default(),
})
}
@@ -1041,7 +1132,7 @@ impl FontDb {
fn with_static_fonts(font_dirs: Vec<std::path::PathBuf>, fonts: impl Iterator<Item = &'static [u8]>) -> Self {
let mut faces = Vec::new();
for data in fonts {
for f in enumerate_faces(data) {
for f in enumerate_faces(data, MAX_FILE_FACES) {
if let Some(mut face) = make_face(FontBytes::Static(data), f.index, f.family, f.style, f.legacy, f.coords) {
face.local_names = f.local;
faces.push(Arc::new(face));
@@ -1248,7 +1339,7 @@ impl FontDb {
for (bytes, name) in files {
let data = Arc::new(bytes);
let name: Option<String> = name.map(|n| n.trim().chars().take(MAX_FAMILY_NAME).collect()).filter(|n: &String| !n.is_empty());
for f in enumerate_faces(&data) {
for f in enumerate_faces(&data, MAX_FILE_FACES) {
if let Some(mut face) = make_face(FontBytes::Owned(data.clone()), f.index, f.family, f.style, f.legacy, f.coords) {
face.local_names = f.local;
if let Some(n) = name.as_deref().filter(|n| !face.named(n) && !face.local_names.iter().any(|l| l.name.eq_ignore_ascii_case(n))) {
@@ -1260,9 +1351,12 @@ impl FontDb {
}
}
let mut faces = self.faces.write().unwrap_or_else(std::sync::PoisonError::into_inner);
// Indexed: a document's fonts may add hundreds of faces to thousands (#3716).
let key = |f: &FontFace| (f.family.to_ascii_lowercase(), f.style.to_ascii_lowercase());
let mut present: std::collections::HashSet<_> = faces.iter().map(|g| key(g)).collect();
let mut added = 0;
for f in new {
if !faces.iter().any(|g| g.family.eq_ignore_ascii_case(&f.family) && g.style.eq_ignore_ascii_case(&f.style)) {
if present.insert(key(&f)) {
faces.push(f);
added += 1;
}
@@ -1462,7 +1556,9 @@ impl FontDb {
(best.path.clone(), best.family.clone(), best.style.clone())
};
let data = Arc::new(read_fallback_file(&path)?);
let found = enumerate_faces(&data).into_iter().find(|f| f.family.eq_ignore_ascii_case(&fam) && f.style.eq_ignore_ascii_case(&style))?;
let found = enumerate_faces(&data, MAX_FILE_FACES)
.into_iter()
.find(|f| f.family.eq_ignore_ascii_case(&fam) && f.style.eq_ignore_ascii_case(&style))?;
let mut face = make_face(FontBytes::Owned(data), found.index, found.family, found.style, found.legacy, found.coords)?;
face.local_names = found.local;
Some(Arc::new(face))
@@ -2006,23 +2102,55 @@ impl FontDb {
}
}
struct FlipPen(BezPath);
/// Collects an outline flipped to y-down, keeping at most `room` path elements (`over` once it
/// had to drop one).
struct FlipPen {
path: BezPath,
room: usize,
over: bool,
}
impl FlipPen {
/// Take one element's room; false (and `over`) when there is none left.
fn take(&mut self) -> bool {
match self.room.checked_sub(1) {
Some(room) if !self.over => {
self.room = room;
true
}
_ => {
self.over = true;
false
}
}
}
}
impl OutlinePen for FlipPen {
fn move_to(&mut self, x: f32, y: f32) {
self.0.move_to((x as f64, -y as f64));
if self.take() {
self.path.move_to((x as f64, -y as f64));
}
}
fn line_to(&mut self, x: f32, y: f32) {
self.0.line_to((x as f64, -y as f64));
if self.take() {
self.path.line_to((x as f64, -y as f64));
}
}
fn quad_to(&mut self, cx0: f32, cy0: f32, x: f32, y: f32) {
self.0.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64));
if self.take() {
self.path.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64));
}
}
fn curve_to(&mut self, cx0: f32, cy0: f32, cx1: f32, cy1: f32, x: f32, y: f32) {
self.0.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64));
if self.take() {
self.path.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64));
}
}
fn close(&mut self) {
self.0.close_path();
if self.take() {
self.path.close_path();
}
}
}
@@ -2033,6 +2161,24 @@ mod tests_sysfonts;
#[cfg(test)]
mod tests {
#[test]
#[cfg(not(target_arch = "wasm32"))]
fn occasional_slow_hinted_glyphs_do_not_turn_hinting_off() {
let off = |pattern: &[bool]| {
let face =
super::make_face(super::FontBytes::Static(super::BUNDLED[0]), 0, "T".into(), "Regular".into(), "T".into(), Vec::new()).unwrap();
pattern.iter().position(|&slow| face.count_hinted(slow))
};
// A face slow throughout stops at the third strike.
assert_eq!(off(&[true; 5]), Some(2));
// Three draws slowed by a busy machine among many: hinting stays on.
let mut busy = vec![false; 100];
busy.extend([true; 3]);
assert_eq!(off(&busy), None);
// Slow glyphs spread among fast ones still stop it once they are one in 16.
let spread: Vec<bool> = (0..64).map(|i| i % 8 == 7).collect();
assert_eq!(off(&spread), Some(23));
}
#[test]
#[ignore = "requires a locally licensed kaiu.ttf; set WORDCRAFT_TEST_KAI_FONT"]
fn kai_contours_include_complete_character_body() {
@@ -2065,6 +2211,21 @@ mod tests {
}
}
#[test]
fn a_capped_outline_is_refused_while_it_is_drawn() {
// #3716 review: a colour glyph's clip outline stops growing at the payload left.
let db = FontDb::with_font_dirs(Vec::new());
db.set_system_fallback(false);
let face = db.face(FALLBACK_FAMILY, "Regular");
let gid = face.glyph_for('g');
let whole = face.glyph_outline(gid);
let n = whole.elements().len();
assert!(n > 4, "{n}");
assert_eq!(face.glyph_outline_capped(gid, n).as_ref().map(BezPath::elements), Some(whole.elements()));
assert!(face.glyph_outline_capped(gid, n - 1).is_none());
assert!(face.glyph_outline_capped(gid, 0).is_none());
}
#[test]
fn run_fallback_is_cached_per_char_set() {
let db = FontDb::with_font_dirs(Vec::new());
@@ -2128,9 +2289,9 @@ mod tests {
let target = Target::Smooth { mode: SmoothMode::Light, symmetric_rendering: true, preserve_linear_metrics: true };
let drawn = |engine: Engine, gid: u32| {
let h = HintingInstance::new(&outlines, Size::new(ppem), face.location(), HintingOptions { engine, target }).unwrap();
let mut pen = FlipPen(BezPath::new());
let mut pen = FlipPen { path: BezPath::new(), room: usize::MAX, over: false };
outlines.get(GlyphId::new(gid)).unwrap().draw(DrawSettings::hinted(&h, false), &mut pen).unwrap();
pen.0
pen.path
};
let mut differs = false;
for c in "Every Thursday".chars().filter(|c| !c.is_whitespace()) {
@@ -2300,4 +2461,112 @@ mod tests {
assert!(!db.has_family("NotAFont-Bold"));
assert_eq!(db.face("Source Serif 4", "Regular").style, "Regular");
}
/// A variable font holding just `name` and `fvar` tables: family "Var Test", one `wght` axis
/// and `instances` named instances, each with its own style name.
fn variable_font(instances: u16) -> Vec<u8> {
let mut names: Vec<(u16, String)> = vec![(1, "Var Test".into()), (2, "Regular".into())];
names.extend((0..instances).map(|i| (256 + i, format!("Style {i}"))));
let mut name = vec![0, 0];
name.extend_from_slice(&(names.len() as u16).to_be_bytes());
name.extend_from_slice(&(6 + 12 * names.len() as u16).to_be_bytes());
let mut strings = Vec::new();
for (id, text) in &names {
let utf16: Vec<u8> = text.encode_utf16().flat_map(u16::to_be_bytes).collect();
for v in [3, 1, 0x409, *id, utf16.len() as u16, strings.len() as u16] {
name.extend_from_slice(&v.to_be_bytes());
}
strings.extend(utf16);
}
name.extend(strings);
let mut fvar = Vec::new();
for v in [1u16, 0, 16, 2, 1, 20, instances, 8] {
fvar.extend_from_slice(&v.to_be_bytes());
}
fvar.extend_from_slice(b"wght");
for v in [100u32 << 16, 400 << 16, 900 << 16] {
fvar.extend_from_slice(&v.to_be_bytes());
}
fvar.extend_from_slice(&[0, 0, 0, 2]); // flags, axisNameId
for i in 0..instances {
fvar.extend_from_slice(&(256 + i).to_be_bytes());
fvar.extend_from_slice(&[0, 0]);
fvar.extend_from_slice(&(400u32 << 16).to_be_bytes());
}
mini_font(&[(*b"fvar", fvar), (*b"name", name)])
}
/// A collection listing `font` `members` times.
fn collection_of(font: &[u8], members: u32) -> Vec<u8> {
let start = 12 + 4 * members;
let mut out = b"ttcf\0\x01\0\0".to_vec();
out.extend_from_slice(&members.to_be_bytes());
for _ in 0..members {
out.extend_from_slice(&start.to_be_bytes());
}
let mut font = font.to_vec();
let tables = u16::from_be_bytes([font[4], font[5]]) as usize;
for i in 0..tables {
let at = 12 + 16 * i + 8;
let offset = u32::from_be_bytes(font[at..at + 4].try_into().unwrap()) + start;
font[at..at + 4].copy_from_slice(&offset.to_be_bytes());
}
out.extend(font);
out
}
/// JetBrains Mono renamed "MingLiU JBMono" (a name the interpreter is required for), with its
/// `maxp` twilight-zone size set to `twilight`.
fn hint_reliant_font(twilight: u16) -> Vec<u8> {
let mut data = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/fonts/JetBrainsMono-Regular.ttf")).unwrap();
let utf16 = |s: &str| s.encode_utf16().flat_map(u16::to_be_bytes).collect::<Vec<u8>>();
for (from, to) in [(utf16("JetBrains Mono"), utf16("MingLiU JBMono")), (b"JetBrains Mono".to_vec(), b"MingLiU JBMono".to_vec())] {
while let Some(at) = data.windows(from.len()).position(|w| w == from) {
data[at..at + to.len()].copy_from_slice(&to);
}
}
let tables = u16::from_be_bytes([data[4], data[5]]) as usize;
let entry = (0..tables).map(|i| 12 + 16 * i).find(|e| &data[*e..e + 4] == b"maxp").unwrap();
let maxp = u32::from_be_bytes(data[entry + 8..entry + 12].try_into().unwrap()) as usize;
data[maxp + 16..maxp + 18].copy_from_slice(&twilight.to_be_bytes());
data
}
#[test]
fn a_font_asking_for_huge_hinting_zones_is_drawn_unhinted() {
// #3692: skrifa's budget counts instructions, not the points each one moves.
let db = FontDb::with_font_dirs(vec![]);
db.add_font(hint_reliant_font(660));
let face = db.face("MingLiU JBMono", "Regular");
assert_eq!(face.family, "MingLiU JBMono");
assert!(face.hinter().is_some(), "a real font's zones are hinted");
assert!(!face.glyph_outline(face.glyph_for('a')).elements().is_empty());
let db = FontDb::with_font_dirs(vec![]);
db.add_font(hint_reliant_font(u16::MAX));
let face = db.face("MingLiU JBMono", "Regular");
assert!(face.hinter().is_none(), "a huge twilight zone is not hinted");
assert!(!face.glyph_outline(face.glyph_for('a')).elements().is_empty(), "drawn unhinted");
}
#[test]
fn named_instances_and_collection_members_are_capped() {
// #3716: a file's faces are capped before they are built, however the file multiplies them.
let font = variable_font(300);
let faces = font_faces(&skrifa::FontRef::new(&font).unwrap(), 0);
assert_eq!(faces.len(), MAX_FILE_FACES + 1, "one more than the cap, so a caller can tell");
assert_eq!(faces[0].style, "Style 0");
assert_eq!(enumerate_faces(&font, MAX_FILE_FACES).len(), MAX_FILE_FACES);
// A million members, all the same font: read once, and no more than the cap.
let started = std::time::Instant::now();
let ttc = collection_of(BUNDLED[0], 1_000_000);
let found = enumerate_faces(&ttc, MAX_FILE_FACES);
assert_eq!(found.iter().map(|f| (f.family.as_str(), f.style.as_str())).collect::<Vec<_>>(), [(FALLBACK_FAMILY, "Regular")]);
let db = FontDb::with_font_dirs(vec![]);
assert_eq!(db.add_font(collection_of(&font, 1_000_000)), MAX_FILE_FACES);
// A font the user adds is refused instead.
assert!(matches!(db.install_font("v.ttf", &variable_font(300)), Err(user::UserFontError::Unusable(e)) if e.contains("more than 256")));
// The face counts above are what tell the old code (a face per member) apart; the time
// (under 0.1 s on a loaded machine) only backs up "read once", with room for a busy CI.
assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed());
}
}
crates/fonts/src/fontdb/user.rs
+313 -11
View File
@@ -149,8 +149,24 @@ pub fn decode_font_file(bytes: &[u8]) -> Result<Vec<u8>, UserFontError> {
}
let decoded = match sniff(bytes).ok_or(UserFontError::NotAFont)? {
FontFormat::TrueType | FontFormat::OpenType | FontFormat::Collection => return Ok(bytes.to_vec()),
FontFormat::Woff => unwrap_web_font(|| wuff::decompress_woff1(bytes))?,
FontFormat::Woff2 => unwrap_web_font(|| wuff::decompress_woff2(bytes))?,
FontFormat::Woff => {
check_woff1_size(bytes)?;
unwrap_web_font(|| wuff::decompress_woff1(bytes))?
}
FontFormat::Woff2 => {
let mut too_many_points = false;
let unwrapped = unwrap_web_font(|| {
wuff::decompress_woff2_with_custom_brotli(bytes, &mut |data, size| {
let stream = unbrotli(data, size)?;
too_many_points = !woff2_points_fit(bytes, &stream);
if too_many_points { Err(TOO_MANY_POINTS.into()) } else { Ok(stream) }
})
});
if too_many_points {
return Err(UserFontError::BadWebFont(TOO_MANY_POINTS.into()));
}
unwrapped?
}
};
if decoded.len() > MAX_DECODED_BYTES {
return Err(UserFontError::TooLarge { bytes: decoded.len() });
@@ -161,6 +177,126 @@ pub fn decode_font_file(bytes: &[u8]) -> Result<Vec<u8>, UserFontError> {
}
}
/// WOFF 1.0: the font its table directory describes must fit [`MAX_DECODED_BYTES`]. The decoder
/// sizes each table's buffer from the directory's `origLength` before inflating it (#3703).
/// A short header or directory is left to the decoder to report.
fn check_woff1_size(bytes: &[u8]) -> Result<(), UserFontError> {
let u32_at = |at: usize| bytes.get(at..at.saturating_add(4)).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes);
let tables = bytes.get(12..14).and_then(|n| n.try_into().ok()).map_or(0, |n| usize::from(u16::from_be_bytes(n)));
let mut total = 12 + 16 * tables;
for i in 0..tables {
let entry = 44 + 20 * i;
let (Some(compressed), Some(orig)) = (u32_at(entry + 8), u32_at(entry + 12)) else { break };
total = total.saturating_add((compressed.max(orig) as usize).saturating_add(3) & !3);
}
if total > MAX_DECODED_BYTES {
return Err(UserFontError::TooLarge { bytes: total });
}
Ok(())
}
/// Why a WOFF2 file whose glyph has more points than a glyph may is refused.
const TOO_MANY_POINTS: &str = "a glyph has more than 65536 points";
/// WOFF2's brotli stream, which must decompress to exactly `size` bytes (the decoder has checked
/// `size` against its own cap and the compression ratio).
fn unbrotli(data: &[u8], size: usize) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
use std::io::Read;
let mut out = Vec::with_capacity(size);
brotli_decompressor::Decompressor::new(data, 4096).take(size as u64 + 1).read_to_end(&mut out)?;
if out.len() != size {
return Err("the compressed data doesn't have the size the directory gives".into());
}
Ok(out)
}
/// Does every glyph in the WOFF2 file's transformed `glyf` tables have at most 65536 points, as
/// the decoder requires? It allocates a glyph's points from its counts before checking them, so a
/// small file could claim gigabytes (#3703). `stream` is the decompressed table data; anything
/// that doesn't parse is refused too (the decoder would refuse it).
fn woff2_points_fit(file: &[u8], stream: &[u8]) -> bool {
woff2_transformed_glyf(file, stream).is_some_and(|tables| tables.iter().all(|t| glyf_points_fit(t).unwrap_or(false)))
}
/// The transformed `glyf` tables in `stream`, found from the WOFF2 table directory in `file`
/// (tables follow each other in directory order).
fn woff2_transformed_glyf<'a>(file: &[u8], stream: &'a [u8]) -> Option<Vec<&'a [u8]>> {
let tables = u16::from_be_bytes(file.get(12..14)?.try_into().ok()?);
let (mut at, mut offset, mut glyf) = (48usize, 0usize, Vec::new());
for _ in 0..tables {
let flags = *file.get(at)?;
at += 1;
let tag: Option<&[u8]> = if flags & 63 == 63 {
at += 4;
Some(file.get(at - 4..at)?)
} else {
None
};
let is = |index: u8, name: &[u8]| flags & 63 == index || tag == Some(name);
let (is_glyf, is_loca) = (is(10, b"glyf"), is(11, b"loca"));
let orig = base128(file, &mut at)?;
// glyf and loca are transformed at version 0, other tables at any other version.
let transformed = if is_glyf || is_loca { flags >> 6 == 0 } else { flags >> 6 != 0 };
let len = if transformed { base128(file, &mut at)? } else { orig };
let end = offset.checked_add(len)?;
if is_glyf && transformed {
glyf.push(stream.get(offset..end)?);
}
offset = end;
}
Some(glyf)
}
/// WOFF2's UIntBase128.
fn base128(b: &[u8], at: &mut usize) -> Option<usize> {
let mut v: u32 = 0;
for i in 0..5 {
let byte = *b.get(*at)?;
*at += 1;
if (i == 0 && byte == 0x80) || v >> 25 != 0 {
return None;
}
v = v << 7 | u32::from(byte & 0x7f);
if byte & 0x80 == 0 {
return usize::try_from(v).ok();
}
}
None
}
/// Does every glyph of the transformed `glyf` table `t` have at most 65536 points?
fn glyf_points_fit(t: &[u8]) -> Option<bool> {
let u32_at = |at: usize| t.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes).map(|v| v as usize);
let glyphs = usize::from(u16::from_be_bytes(t.get(4..6)?.try_into().ok()?));
let contours_end = 36usize.checked_add(u32_at(8)?)?;
let contours = t.get(36..contours_end)?;
let points = t.get(contours_end..contours_end.checked_add(u32_at(12)?)?)?;
let mut at = 0;
for g in 0..glyphs {
let n = i16::from_be_bytes(contours.get(2 * g..2 * g + 2)?.try_into().ok()?);
let mut total = 0usize;
for _ in 0..n.max(0) {
// 255UInt16
let code = *points.get(at)?;
at += 1;
let mut byte = || {
at += 1;
points.get(at - 1).copied().map(usize::from)
};
total += match code {
253 => byte()? << 8 | byte()?,
254 => byte()? + 253 * 2,
255 => byte()? + 253,
v => usize::from(v),
};
if total > 1 << 16 {
return Some(false);
}
}
}
Some(true)
}
/// Run a WOFF/WOFF2 decoder. The decoder checks its input, but it is third-party code reading
/// hostile bytes: a panic in it is caught (native) and reported as a damaged file.
fn unwrap_web_font(decode: impl FnOnce() -> Result<Vec<u8>, wuff::WuffErr>) -> Result<Vec<u8>, UserFontError> {
@@ -230,7 +366,8 @@ impl FontDb {
None
}
Err(e) => {
log::warn!("couldn't store the font {name} in {}: {e}", dir.display());
// Debug-quoted: a line break in the supplied name must not start a forged log record.
log::warn!("couldn't store the font {name:?} in {}: {e}", dir.display());
Some(format!("added for this session only: couldn't store it in {}: {e}", dir.display()))
}
},
@@ -266,7 +403,7 @@ impl FontDb {
if FileRef::new(&sfnt).is_err() {
return Err(UserFontError::Unusable("not a valid OpenType file".into()));
}
let found = enumerate_faces(&sfnt);
let found = enumerate_faces(&sfnt, MAX_FACES_PER_FILE + 1);
if found.is_empty() {
return Err(UserFontError::Unusable("no face with a family name".into()));
}
@@ -412,12 +549,29 @@ fn unique_name(name: &str, taken: impl Fn(&str) -> bool) -> String {
/// Write `bytes` to a new file in `dir` named `name` (or a numbered variant when that exists),
/// through a temporary file so a crash never leaves half a font.
fn write_unique(dir: &Path, name: &str, bytes: &[u8]) -> std::io::Result<PathBuf> {
// One store at a time: two adds racing for the same name would otherwise pick the same file
// and unlink each other's temporary. ponytail: in-process only; two app instances adding the
// same name at the same instant can still collide (the later rename wins).
static STORE: std::sync::Mutex<()> = std::sync::Mutex::new(());
let _held = STORE.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
std::fs::create_dir_all(dir)?;
let name = unique_name(name, |n| dir.join(n).exists());
let path = dir.join(&name);
let tmp = dir.join(format!(".{name}.part"));
std::fs::write(&tmp, bytes)?;
if let Err(e) = std::fs::rename(&tmp, &path) {
// Created afresh, never opened through what is there: a link planted at the temporary name
// would otherwise carry the write to a file outside the folder. A leftover from a crash (or
// such a link) is unlinked, not followed, and the create tried once more.
let create = || std::fs::OpenOptions::new().write(true).create_new(true).open(&tmp);
let mut file = match create() {
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
std::fs::remove_file(&tmp)?;
create()?
}
r => r?,
};
let written = std::io::Write::write_all(&mut file, bytes);
drop(file);
if let Err(e) = written.and_then(|()| std::fs::rename(&tmp, &path)) {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
@@ -564,25 +718,89 @@ pub(crate) mod tests {
base128(data.len() as u32, &mut dir);
stream.extend_from_slice(data);
}
let compressed = brotli_stored(&stream);
let sfnt: usize = 12 + 16 * t.len() + t.iter().map(|(_, d)| d.len().next_multiple_of(4)).sum::<usize>();
woff2_file(&font[..4], t.len(), &dir, &stream, sfnt)
}
/// A WOFF2 file: header, the table directory `dir` of `n` tables, `stream` in stored brotli blocks.
fn woff2_file(flavor: &[u8], n: usize, dir: &[u8], stream: &[u8], sfnt: usize) -> Vec<u8> {
let compressed = brotli_stored(stream);
let mut out = Vec::new();
out.extend_from_slice(b"wOF2");
out.extend_from_slice(&font[..4]);
out.extend_from_slice(flavor);
let total = (48 + dir.len() + compressed.len()).next_multiple_of(4);
out.extend_from_slice(&(total as u32).to_be_bytes());
out.extend_from_slice(&(t.len() as u16).to_be_bytes());
out.extend_from_slice(&(n as u16).to_be_bytes());
out.extend_from_slice(&0u16.to_be_bytes());
let sfnt: usize = 12 + 16 * t.len() + t.iter().map(|(_, d)| d.len().next_multiple_of(4)).sum::<usize>();
out.extend_from_slice(&(sfnt as u32).to_be_bytes());
out.extend_from_slice(&(compressed.len() as u32).to_be_bytes());
out.extend_from_slice(&[0, 1, 0, 0]);
out.extend_from_slice(&[0; 20]);
out.extend(dir);
out.extend_from_slice(dir);
out.extend(compressed);
out.resize(total, 0);
out
}
/// A WOFF2 file holding a transformed `glyf` (and `loca`) whose one glyph has two contours of
/// `points` points each (and no coordinates: the decoder finds it damaged in the end).
fn woff2_glyph_with(points: u16) -> Vec<u8> {
let total = 2 * points as usize;
let mut glyf = vec![0, 0, 0, 0, 0, 1, 0, 0]; // reserved, optionFlags, numGlyphs 1, indexFormat
// nContour, nPoints, flag, glyph, composite, bbox and instruction stream sizes.
for size in [2, 6, total, 0, 0, 4, 0] {
glyf.extend_from_slice(&(size as u32).to_be_bytes());
}
glyf.extend_from_slice(&2i16.to_be_bytes());
let [hi, lo] = points.to_be_bytes();
glyf.extend_from_slice(&[253, hi, lo, 253, hi, lo]); // 255UInt16 in its word form
glyf.resize(glyf.len() + total, 1); // on-curve flags
glyf.extend_from_slice(&[0; 4]); // no explicit bounding boxes
let mut dir = vec![10]; // glyf, transform version 0
base128(4096, &mut dir);
base128(glyf.len() as u32, &mut dir);
dir.push(11); // loca, transformed away
base128(8, &mut dir);
base128(0, &mut dir);
woff2_file(&[0, 1, 0, 0], 2, &dir, &glyf, 12 + 32 + 4096 + 8)
}
/// A WOFF 1.0 file whose one table is a zlib stream of nothing that claims to inflate to `orig`
/// bytes.
fn woff1_claiming(orig: u32) -> Vec<u8> {
let mut out = b"wOFF\0\x01\0\0".to_vec();
out.extend_from_slice(&72u32.to_be_bytes()); // length
out.extend_from_slice(&1u16.to_be_bytes()); // numTables
out.extend_from_slice(&0u16.to_be_bytes());
out.extend_from_slice(&orig.saturating_add(28).to_be_bytes()); // totalSfntSize
out.extend_from_slice(&[0, 1, 0, 0]);
out.extend_from_slice(&[0; 20]);
out.extend_from_slice(b"glyf");
for v in [64, 8, orig, 0] {
out.extend_from_slice(&u32::to_be_bytes(v)); // offset, compLength, origLength, checksum
}
out.extend_from_slice(&[0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01]);
out
}
#[test]
fn a_woff_claiming_a_huge_table_is_refused_before_inflating() {
// #3703: the decoder sizes a table's buffer from the directory before inflating it.
for orig in [200 << 20, u32::MAX] {
assert!(matches!(decode_font_file(&woff1_claiming(orig)), Err(UserFontError::TooLarge { .. })), "{orig}");
}
// A small claim reaches the decoder, which finds the data doesn't inflate to it.
assert!(matches!(decode_font_file(&woff1_claiming(64)), Err(UserFontError::BadWebFont(_))));
}
#[test]
fn a_woff2_glyph_with_too_many_points_is_refused_before_unpacking() {
// #3703: the decoder allocates a glyph's points from these counts before checking them.
assert_eq!(decode_font_file(&woff2_glyph_with(40_000)), Err(UserFontError::BadWebFont(TOO_MANY_POINTS.into())));
// Within the limit the scan passes the file on, to the decoder.
assert!(matches!(decode_font_file(&woff2_glyph_with(3)), Err(UserFontError::BadWebFont(e)) if e != TOO_MANY_POINTS));
}
#[test]
fn an_added_font_is_listed_resolved_and_measured() {
let db = db();
@@ -703,6 +921,44 @@ pub(crate) mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
/// The messages logged while this test binary runs.
fn logged() -> &'static std::sync::Mutex<Vec<String>> {
static LOG: std::sync::OnceLock<std::sync::Mutex<Vec<String>>> = std::sync::OnceLock::new();
struct Capture;
impl log::Log for Capture {
fn enabled(&self, _: &log::Metadata) -> bool {
true
}
fn log(&self, r: &log::Record) {
if let Some(l) = LOG.get() {
l.lock().unwrap().push(r.args().to_string());
}
}
fn flush(&self) {}
}
LOG.get_or_init(|| {
let _ = log::set_logger(&Capture);
log::set_max_level(log::LevelFilter::Warn);
std::sync::Mutex::new(Vec::new())
})
}
#[test]
fn a_supplied_name_is_logged_escaped() {
// #3692: a line break in the name must not start a forged record in the log file.
let log = logged();
let not_a_folder = temp_dir("not-a-folder");
std::fs::write(&not_a_folder, b"x").unwrap();
let db = db();
db.load_user_fonts(&not_a_folder);
let r = db.install_font("a\nERROR forged record.ttf", &renamed("SourceSans3-Regular.ttf", FAMILY)).unwrap();
assert!(r.warning.is_some(), "the font can't be stored");
let lines = log.lock().unwrap();
let line = lines.iter().find(|l| l.contains("forged record")).unwrap();
assert!(!line.contains('\n'), "{line}");
let _ = std::fs::remove_file(&not_a_folder);
}
#[test]
fn stored_names_are_safe_and_unique() {
let ttf = renamed("SourceSans3-Regular.ttf", FAMILY);
@@ -711,4 +967,50 @@ pub(crate) mod tests {
assert_eq!(stored_name("-rf.otf", b"OTTO"), "font-rf.otf");
assert_eq!(unique_name("a.ttf", |n| n == "a.ttf" || n == "a-2.ttf"), "a-3.ttf");
}
#[cfg(unix)]
#[test]
fn a_link_planted_at_the_temporary_name_is_not_written_through() {
// #3703 review: the temporary file is created afresh, so a planted link can't carry the
// font bytes to a file outside the folder.
let root = std::env::temp_dir().join(format!("wordcraft-font-part-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let dir = root.join("fonts");
std::fs::create_dir_all(&dir).unwrap();
let outside = root.join("outside.txt");
std::fs::write(&outside, b"keep me").unwrap();
std::os::unix::fs::symlink(&outside, dir.join(".a.ttf.part")).unwrap();
let path = write_unique(&dir, "a.ttf", b"font bytes").unwrap();
assert_eq!(std::fs::read(&outside).unwrap(), b"keep me");
assert_eq!(path, dir.join("a.ttf"));
assert_eq!(std::fs::read(&path).unwrap(), b"font bytes");
assert!(!std::fs::symlink_metadata(&path).unwrap().file_type().is_symlink());
assert!(std::fs::symlink_metadata(dir.join(".a.ttf.part")).is_err(), "the temporary name is gone");
// A leftover from a crash doesn't block the next add either.
std::fs::write(dir.join(".b.ttf.part"), b"half a font").unwrap();
assert_eq!(std::fs::read(write_unique(&dir, "b.ttf", b"whole").unwrap()).unwrap(), b"whole");
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn fonts_stored_at_once_under_one_name_each_get_their_own_file() {
// #3703 review: concurrent adds of one name never share a temporary or a destination.
let dir = std::env::temp_dir().join(format!("wordcraft-font-race-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let paths: Vec<_> = (0..8u8)
.map(|i| {
let dir = dir.clone();
std::thread::spawn(move || (i, write_unique(&dir, "a.ttf", &[i; 64]).unwrap()))
})
.collect::<Vec<_>>()
.into_iter()
.map(|t| t.join().unwrap())
.collect();
let distinct: std::collections::HashSet<_> = paths.iter().map(|(_, p)| p.clone()).collect();
assert_eq!(distinct.len(), 8);
for (i, p) in &paths {
assert_eq!(std::fs::read(p).unwrap(), vec![*i; 64], "{}", p.display());
}
let _ = std::fs::remove_dir_all(&dir);
}
}
crates/fonts/src/tests_sysfonts.rs
+1 -1
View File
@@ -270,7 +270,7 @@ fn the_scan_lists_a_variable_fonts_named_instances_as_loading_it_does() {
return;
};
let data = std::fs::read(path).unwrap();
let loaded: Vec<(String, String, String)> = enumerate_faces(&data).into_iter().map(|f| (f.family, f.style, f.legacy)).collect();
let loaded: Vec<(String, String, String)> = enumerate_faces(&data, MAX_FILE_FACES).into_iter().map(|f| (f.family, f.style, f.legacy)).collect();
assert_eq!(names(path), loaded);
if loaded.len() > 1 {
// Instances other than Regular, Bold and Italic have names of their own.
crates/formats/Cargo.toml
+1
View File
@@ -25,4 +25,5 @@ chrono = { workspace = true }
js-sys = "0.3"
[dev-dependencies]
wordcraft-doc = { workspace = true, features = ["test-support"] }
proptest = { workspace = true }
crates/formats/src/html.rs
+19 -3
View File
@@ -724,7 +724,7 @@ fn apply_style(css: &str, f: &mut Fmt, align: &mut Option<Align>, preserve: &mut
if model::is_mono(&fam) || fam.eq_ignore_ascii_case("monospace") {
f.code = true;
} else if !fam.is_empty() && !matches!(fam.to_ascii_lowercase().as_str(), "serif" | "sans-serif" | "inherit" | "initial") {
f.font = Some(fam);
f.font = model::kept_font(&fam).or(f.font.take());
}
}
"vertical-align" => {
@@ -1226,7 +1226,7 @@ impl<'r> Builder<'r> {
if model::is_mono(&fam) {
fmt.code = true;
} else if !fam.is_empty() {
fmt.font = Some(fam);
fmt.font = model::kept_font(&fam).or(fmt.font.take());
}
}
if let Some(sz) = attr(attrs, "size") {
@@ -1245,7 +1245,7 @@ impl<'r> Builder<'r> {
}
"a" => {
if let Some(h) = attr(attrs, "href").filter(|h| wordcraft_doc::link::is_safe_href(h)) {
fmt.link = Some(h.trim().to_string());
fmt.link = model::kept_link(h.trim());
}
}
_ => {}
@@ -1588,17 +1588,20 @@ fn collapse_ws(s: &str) -> String {
/// Parse HTML into a document.
pub fn import(s: &str) -> Document {
model::begin_import();
model::to_doc(&parse(s))
}
/// [`import`], loading pictures that aren't `data:` URIs through `images`.
pub fn import_with(s: &str, images: ImageLoader) -> Document {
model::begin_import();
model::to_doc(&parse_with(s, images))
}
/// [`import_with`] onto `page` (its size and margins), which tables size to: HTML gives no page
/// size, so the caller picks it (the region's paper, #1381).
pub fn import_on(s: &str, images: ImageLoader, page: &wordcraft_doc::SectionProps) -> Document {
model::begin_import();
model::to_doc_in(&parse_with(s, images), page.clone())
}
@@ -2424,4 +2427,17 @@ mod tests {
assert!(html.contains("vertical-align:middle"));
assert!(html.contains("vertical-align:bottom"));
}
#[test]
fn many_formatted_leading_breaks_trim_quickly() {
// #3705: a paragraph opening with alternating bold and italic `<br>` was trimmed one
// span at a time with `Vec::remove(0)`.
let html = format!("<p>{}x</p>", "<b><br></b><i><br></i>".repeat(100_000));
let t = std::time::Instant::now();
let f = parse(&html);
// About a second on a loaded machine; trimming one span at a time ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed());
let texts: Vec<String> = f.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect();
assert_eq!(texts, ["x"]);
}
}
crates/formats/src/latex.rs
+51 -12
View File
@@ -243,8 +243,10 @@ impl Writer {
}
self.out.push_str("\\begin{verbatim}\n");
for l in lines {
// The environment ends at the first `\end{verbatim}`, so keep it from appearing.
let l: String = l.replace("\\end{verbatim}", "\\end {verbatim}").chars().filter(|c| *c == '\t' || !c.is_control()).collect();
// The environment ends at the first `\end{verbatim}`, so keep it from appearing. The
// control characters go first: removing one afterwards could join a delimiter back up.
let l: String = l.chars().filter(|c| *c == '\t' || !c.is_control()).collect();
let l = l.replace("\\end{verbatim}", "\\end {verbatim}");
self.out.push_str(&l);
self.out.push('\n');
}
@@ -1055,6 +1057,7 @@ fn math_seq(t: &[MTok], pos: &mut usize, depth: usize, in_opt: bool) -> String {
/// Import a LaTeX source file.
pub fn import(bytes: &[u8]) -> Document {
model::begin_import();
let mut d = model::to_doc(&parse(&crate::txt::decode(bytes.get(..MAX_SOURCE).unwrap_or(bytes))));
d.ensure_nonempty();
d
@@ -2333,20 +2336,21 @@ impl<'a> Parser<'a> {
}
}
"href" => {
let url = self.arg_raw();
self.with_arg(move |f| f.link = Some(url));
let url = model::kept_link(&self.arg_raw());
self.with_arg(move |f| f.link = url);
}
"url" | "nolinkurl" => {
let url = self.arg_raw();
let mut f = self.fmt();
if name == "url" {
f.link = Some(url.clone());
f.link = model::kept_link(&url);
}
self.para.push_text(&url, &f);
}
"hyperlink" => {
let target = self.arg_raw();
self.with_arg(move |f| f.link = Some(format!("#{target}")));
let link = model::kept_link(&format!("#{target}"));
self.with_arg(move |f| f.link = link);
}
"hypertarget" => {
let target = self.arg_raw();
@@ -2360,7 +2364,7 @@ impl<'a> Parser<'a> {
"ref" | "eqref" | "pageref" | "autoref" | "cref" | "Cref" | "nameref" => {
let l = self.arg_raw();
let mut f = self.fmt();
f.link = Some(format!("#{l}"));
f.link = model::kept_link(&format!("#{l}"));
self.para.push_text(&l, &f);
}
"cite" | "citep" | "citet" | "parencite" | "textcite" | "autocite" => {
@@ -2671,19 +2675,24 @@ fn table(toks: &[Tok], spec: &str, depth: usize) -> FTable {
for cell_toks in r {
let mut cell = Cell::default();
let mut content = trim_cell(cell_toks);
// `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`.
// `\multicolumn{n}{spec}{content}`, `\multirow{n}{width}{content}`. Each wrapper
// rescans its content, so only MAX_NEST of them are unwrapped (a cell nests one or
// two); further ones stay in the content.
let mut sub = Parser::new(content, depth + 1);
let mut wrappers = 0usize;
loop {
sub.skip_spaces();
match sub.t.get(sub.pos) {
Some(Tok::Cmd(c)) if c == "multicolumn" => {
Some(Tok::Cmd(c)) if c == "multicolumn" && wrappers < MAX_NEST => {
wrappers += 1;
sub.pos += 1;
cell.colspan = sub.arg_raw().trim().parse::<u32>().unwrap_or(1).clamp(1, 63);
let _ = sub.arg_raw();
content = sub.arg_tokens();
sub = Parser::new(content, depth + 1);
}
Some(Tok::Cmd(c)) if c == "multirow" || c == "multirow*" => {
Some(Tok::Cmd(c)) if (c == "multirow" || c == "multirow*") && wrappers < MAX_NEST => {
wrappers += 1;
sub.pos += 1;
let _ = sub.opt_arg();
cell.rowspan = sub.arg_raw().trim().parse::<i64>().map(|n| n.unsigned_abs().clamp(1, 1000) as u32).unwrap_or(1);
@@ -3161,13 +3170,13 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after.
let FBlock::Para(p) = &flow.blocks[0] else { panic!() };
assert_eq!(
p.inlines[0],
Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() })
Inline::Text("Homepage".to_string(), Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() })
);
assert_eq!(
p.inlines[2],
Inline::Text(
"https://example.com/~user/page".to_string(),
Fmt { link: Some("https://example.com/~user/page".to_string()), ..Default::default() }
Fmt { link: Some("https://example.com/~user/page".into()), ..Default::default() }
)
);
}
@@ -3234,4 +3243,34 @@ Caf\'e na\"ive \c{c}a \v{s}\'{e} and {\bfseries grouped bold} after.
let Some(Block::Table(bt_bordered)) = back_bordered.body.first().map(|b| &**b) else { panic!() };
assert!(bt_bordered.props.style.is_some(), "imported table should remain bordered");
}
#[test]
fn verbatim_end_is_kept_out_after_control_characters_go() {
// A control character inside `\end{verbatim}` was removed after the check for it, so
// the code line closed the environment and the rest ran as TeX.
let mut d = Document::new();
let line = "\\end{verb\u{E}atim}\\input{/etc/passwd}";
d.body = vec![wordcraft_doc::para_block(wordcraft_doc::Paragraph::with_text(line, Default::default()).styled(model::CODE_STYLE))];
let out = export(&d);
assert_eq!(out.matches("\\end{verbatim}").count(), 1, "{out}");
assert!(out.contains("\\end {verbatim}\\input"), "{out}");
}
#[test]
fn nested_multicolumn_wrappers_are_bounded() {
// Each `\multicolumn` unwrapped rescanned the rest of the cell; nested ones were all
// unwrapped, quadratic in the nesting.
let n = 50_000;
let src = format!("\\begin{{tabular}}{{l}}{}x{}\\\\\\end{{tabular}}", "\\multicolumn{1}{c}{".repeat(n), "}".repeat(n));
let t = std::time::Instant::now();
let f = parse(&src);
// Under a second on a loaded machine; unwrapping every wrapper took 213 s.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert!(f.blocks.iter().any(|b| matches!(b, FBlock::Table(_))));
// One or two wrappers, as written in real tables, are still read.
let f = parse("\\begin{tabular}{ll}\\multicolumn{2}{c}{\\multirow{2}{*}{Both}}\\\\\\end{tabular}");
let Some(FBlock::Table(t)) = f.blocks.first() else { panic!("{:?}", f.blocks) };
assert_eq!((t.rows[0][0].colspan, t.rows[0][0].rowspan), (2, 2));
assert_eq!(t.rows[0][0].text(), "Both");
}
}
crates/formats/src/lib.rs
+2
View File
@@ -91,5 +91,7 @@ pub fn export_with(ext: &str, doc: &Document, opts: &ExportOptions) -> Option<Re
Some(out.map(|bytes| comments::append(&ext, doc, bytes)))
}
#[cfg(test)]
mod test_log;
#[cfg(test)]
mod tests;
crates/formats/src/markdown.rs
+119 -28
View File
@@ -10,6 +10,7 @@
//! Export writes the same constructs; pictures become `![alt](media-name)` placeholders.
use std::collections::HashMap;
use std::sync::Arc;
use wordcraft_doc::{Align, Document};
@@ -24,16 +25,19 @@ const MAX_NEST: usize = 32;
/// Parse Markdown text into a document.
pub fn import(text: &str) -> Document {
model::begin_import();
model::to_doc(&parse(text))
}
/// [`import`] onto `page`: Markdown gives no page size (#1381).
pub fn import_on(text: &str, page: &wordcraft_doc::SectionProps) -> Document {
model::begin_import();
model::to_doc_in(&parse(text), page.clone())
}
/// Link reference definitions: the normalised label to its destination.
pub type Refs = HashMap<String, String>;
/// A destination is shared by every reference to it, not copied into each (#3713).
pub type Refs = HashMap<String, Arc<str>>;
/// A label as CommonMark compares them: case-folded, and any run of white space one space
/// (§4.7, "matches"). `[FOO bar]` and `[foo BAR]` are the same label.
@@ -141,7 +145,7 @@ fn take_definitions(lines: &mut Vec<String>) -> Refs {
if starts_block && let Some((label, dest)) = link_definition(line) {
// The first definition of a label wins, as CommonMark says, and definitions may
// follow one another.
refs.entry(label).or_insert(dest);
refs.entry(label).or_insert(dest.into());
return false;
}
starts_block = is_blank(line) || atx_heading(t).is_some() || thematic_break(t);
@@ -799,10 +803,18 @@ fn link_dest(s: &str, at: usize) -> Option<(String, usize)> {
i += ws;
let r = s.get(i..)?;
let (dest, used) = if let Some(inner) = r.strip_prefix('<') {
let end = inner.find(['>', '\n'])?;
if inner.get(end..)?.starts_with('\n') {
return None;
}
// The first `>` not escaped by a backslash ends it; a line break first means no link.
let b = inner.as_bytes();
let mut k = 0;
let end = loop {
match b.get(k)? {
b'\\' => k += 1,
b'>' => break k,
b'\n' => return None,
_ => {}
}
k += 1;
};
(inner.get(..end)?.to_string(), end + 2)
} else {
let mut depth = 0i32;
@@ -983,7 +995,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
let dest = link_dest(s, close + 1);
scan_budget = scan_budget.saturating_sub(dest.as_ref().map_or(s.len(), |d| d.1).saturating_sub(close));
let (dest, end) = dest?;
return Some((text.to_string(), dest, end));
return Some((text.to_string(), Arc::from(dest), end));
}
// A reference link: `[text][label]`, `[text][]` and `[text]` all take
// their destination from a definition collected before parsing (#3238).
@@ -1020,7 +1032,11 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
Some((inner, dest, end)) if base.link.is_none() => {
flush!();
let mut f = base.clone();
f.link = Some(dest);
// A destination too long to keep leaves its text unlinked.
f.link = (dest.len() <= model::MAX_LINK).then_some(dest);
if f.link.is_none() {
model::dropped(model::Dropped::Link);
}
for x in parse_inlines(&inner, &f, depth + 1, refs) {
nodes.push(Node::Inl(x));
}
@@ -1047,7 +1063,7 @@ fn tokenize(s: &str, base: &Fmt, depth: usize, refs: &Refs) -> Vec<Node> {
if is_auto {
flush!();
let mut f = base.clone();
f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}") } else { inner.to_string() });
f.link = Some(if inner.contains('@') && !inner.contains(':') { format!("mailto:{inner}").into() } else { inner.into() });
push_text(&mut nodes, inner, &f);
i += inner.len() + 2;
text_start = i;
@@ -1202,18 +1218,16 @@ fn emit(mut nodes: Vec<Node>) -> Vec<Inline> {
}
out.push(Inline::Text(t.to_string(), f));
};
let base_of = |nodes: &[Node], k: usize| -> Fmt {
// Delimiter literals take the formatting (link, …) of neighbouring text.
for j in (0..k).rev() {
if let Some(Node::Text(_, f)) = nodes.get(j) {
return Fmt { bold: false, italic: false, strike: false, ..f.clone() };
}
}
Fmt::default()
};
for (k, n) in nodes.iter().enumerate() {
// Delimiter literals take the formatting (link, …) of the text before them: the last text
// node, tracked here rather than searched for backwards from every delimiter (quadratic on
// a paragraph of delimiters).
let mut last_text: Option<&Fmt> = None;
for n in &nodes {
match n {
Node::Text(t, f) => push(&mut out, t, f, em, strong, strike),
Node::Text(t, f) => {
push(&mut out, t, f, em, strong, strike);
last_text = Some(f);
}
Node::Inl(Inline::Text(t, f)) => push(&mut out, t, f, em, strong, strike),
Node::Inl(other) => out.push(other.clone()),
Node::Delim { ch, left, opens, closes, .. } => {
@@ -1224,9 +1238,11 @@ fn emit(mut nodes: Vec<Node>) -> Vec<Inline> {
Emph::Strike => strike = strike.saturating_sub(1),
}
}
let lit: String = std::iter::repeat_n(*ch, *left).collect();
let f = base_of(&nodes, k);
push(&mut out, &lit, &f, em, strong, strike);
if *left > 0 {
let lit: String = std::iter::repeat_n(*ch, *left).collect();
let f = last_text.map_or_else(Fmt::default, |f| Fmt { bold: false, italic: false, strike: false, ..f.clone() });
push(&mut out, &lit, &f, em, strong, strike);
}
for e in opens {
match e {
Emph::Em => em += 1,
@@ -1378,7 +1394,7 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String {
}
let mut out = String::new();
let mut stack: Vec<Mark> = Vec::new();
let mut link: Option<String> = None;
let mut link: Option<Arc<str>> = None;
let close_all = |out: &mut String, stack: &mut Vec<Mark>| {
while let Some(m) = stack.pop() {
out.push_str(marker(m));
@@ -1436,8 +1452,35 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String {
out
}
/// `l` as a Markdown link destination that reads back as exactly `l`, the target
/// `is_safe_href` passed. What Markdown would decode (a backslash escape, an entity reference)
/// or end the destination on (`<`, `>`) is backslash-escaped, never deleted: deleting `<` or `>`
/// could turn an accepted relative target into `javascript:…` or `//host`. Control characters
/// can't be written in a destination and are left out, as the safety check ignores them.
fn link_target(l: &str) -> String {
if l.contains([' ', '(', ')', '<', '>']) { format!("<{}>", l.replace(['<', '>', '\n'], "")) } else { l.to_string() }
let l: String = l.chars().filter(|c| !c.is_ascii_control()).collect();
let mut out = String::with_capacity(l.len() + 2);
for (i, c) in l.char_indices() {
let rest = l.get(i + c.len_utf8()..).unwrap_or("");
match c {
'<' | '>' => out.push('\\'),
// A backslash is literal unless punctuation (or the closing `)`/`>`) follows it.
'\\' if rest.chars().next().is_none_or(|n| n.is_ascii_punctuation()) => out.push('\\'),
'&' if entity_like(rest) => out.push('\\'),
_ => {}
}
out.push(c);
}
if l.contains(|c: char| c.is_whitespace() || matches!(c, '(' | ')' | '<' | '>')) { format!("<{out}>") } else { out }
}
/// Whether `rest` (the text after a `&`) starts like an entity reference (`amp;`, `#106;`,
/// `#x6A;`), which a Markdown reader would decode.
fn entity_like(rest: &str) -> bool {
let Some(end) = within(rest, 34).find(';') else { return false };
let body = rest.get(..end).unwrap_or("");
let body = body.strip_prefix('#').map_or(body, |n| n.strip_prefix(['x', 'X']).unwrap_or(n));
!body.is_empty() && body.chars().all(|c| c.is_ascii_alphanumeric())
}
fn marker(m: Mark) -> &'static str {
@@ -1640,7 +1683,7 @@ mod tests {
let s = inl("see [the *site*](http://x.org \"T\") and `a*b*` \\*lit\\* &amp; <https://y.com>");
let link: Vec<_> = s
.iter()
.filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.clone(), f.italic)) } else { None })
.filter_map(|i| if let Inline::Text(t, f) = i { f.link.as_ref().map(|l| (t.clone(), l.to_string(), f.italic)) } else { None })
.collect();
assert_eq!(link[0], ("the ".into(), "http://x.org".into(), false));
assert_eq!(link[1], ("site".into(), "http://x.org".into(), true));
@@ -1661,7 +1704,7 @@ mod tests {
.inlines
.iter()
.filter_map(|x| match x {
Inline::Text(t, f) => f.link.clone().map(|l| (t.clone(), l)),
Inline::Text(t, f) => f.link.as_deref().map(|l| (t.clone(), l.to_string())),
_ => None,
})
.collect(),
@@ -2067,7 +2110,8 @@ mod tests {
use wordcraft_doc::{CharProps, Document, Paragraph, StoryRef, para_block};
let cases = ["Shopping\n- milk\n# not a heading", "a\n1. x", "a\n2) x", "a\n+ x", "a\n---", "a\n***", "a\n==", "a\n--", "a\n> q"];
for s in cases {
let d = Document { body: vec![para_block(Paragraph::with_text(s, CharProps::default()))], ..Default::default() };
let mut d = Document::new();
d.body = vec![para_block(Paragraph::with_text(s, CharProps::default()))];
let md = export(&d);
let back = import(&md);
assert_eq!(back.body.len(), 1, "{s:?} -> {md:?}");
@@ -2152,4 +2196,51 @@ mod tests {
let texts: Vec<String> = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect();
assert_eq!(texts, ["Hello world!", "After comment."]);
}
#[test]
fn a_paragraph_of_delimiters_parses_in_linear_time() {
// #3714: each delimiter's literal looked back for the text before it, to the start of
// the paragraph when there was none (quadratic).
let src = "*_".repeat(200_000);
let t = std::time::Instant::now();
let out = inl(&src);
// Under a second on a loaded machine; the backwards search ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
let text: String = out.iter().filter_map(|i| if let Inline::Text(t, _) = i { Some(t.as_str()) } else { None }).collect();
assert!(text.chars().all(|c| matches!(c, '*' | '_')), "only delimiters come out");
// Literal delimiters still take the link of the text before them.
let s = inl("[a](http://x.org) [b*](http://y.org)");
assert!(s.iter().any(|i| matches!(i, Inline::Text(t, f) if t == "b*" && f.link.as_deref() == Some("http://y.org"))), "{s:?}");
}
#[test]
fn exported_link_destinations_read_back_unchanged() {
// Removing `<` and `>` from a destination after the safety check could make a relative
// target a script or network-path one (`java<script:…` → `javascript:…`). They are
// escaped instead, as are backslash escapes and entity references a reader would decode.
let md = |l: &str| inlines_md(&[Inline::Text("x".into(), Fmt { link: Some(l.into()), ..Default::default() })]);
assert_eq!(md("java<script:alert(1)"), "[x](<java\\<script:alert(1)>)");
for l in [
"java<script:alert(1)",
"/</evil.example/(x)",
"a>b c",
"&#106;avascript:alert(1)",
"&#x6A;avascript:x",
"&amp;",
"javascript\\:x",
"dir\\",
"dir\\(1)\\",
"q?a=1&b=2",
] {
let out = md(l);
let back = inl(&out);
let got: Vec<_> =
back.iter().filter_map(|i| if let Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect();
assert_eq!(got, [l.to_string()], "{l:?} → {out:?}");
}
// Ordinary destinations are written as before.
assert_eq!(md("https://example.com/a?b=1&c=2"), "[x](https://example.com/a?b=1&c=2)");
assert_eq!(md("https://example.com/a b"), "[x](<https://example.com/a b>)");
assert_eq!(md("https://e.com/x\ny"), "[x](https://e.com/xy)");
}
}
crates/formats/src/model.rs
+140 -18
View File
@@ -19,6 +19,84 @@ pub const MONO_FONT: &str = "Courier New";
pub const CODE_STYLE: &str = "Code";
/// Nesting limit for tables in tables (hostile input).
pub const MAX_DEPTH: usize = 8;
/// Longest font name the importers keep, in bytes (as the .docx reader); a longer one (no real
/// font has one) is dropped (#3713).
pub const MAX_FONT_NAME: usize = 256;
/// Longest hyperlink target the importers keep, in bytes; a longer link is dropped and its text
/// kept (#3713).
pub const MAX_LINK: usize = 8 << 10;
/// Link and font bytes the document's runs may carry between them. A flow's spans share one
/// copy of a name ([`Fmt`]), but every run of a document owns its own: past this budget, runs
/// lose their link and font rather than multiply a small file into gigabytes (#3713).
const MAX_RUN_STRINGS: usize = 64 << 20;
/// `name` as a span's font, unless it is longer than [`MAX_FONT_NAME`] (then reported, see
/// [`dropped`]).
pub fn kept_font(name: &str) -> Option<Arc<str>> {
let kept = (name.len() <= MAX_FONT_NAME).then(|| name.into());
if kept.is_none() {
dropped(Dropped::Font);
}
kept
}
/// `target` as a span's link, unless it is longer than [`MAX_LINK`] (then reported, see
/// [`dropped`]).
pub fn kept_link(target: &str) -> Option<Arc<str>> {
let kept = (target.len() <= MAX_LINK).then(|| target.into());
if kept.is_none() {
dropped(Dropped::Link);
}
kept
}
/// What an importer leaves out to stay within a limit (review of #3873: no limit drops data
/// without saying so).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Dropped {
/// A font name over [`MAX_FONT_NAME`].
Font,
/// A link target over [`MAX_LINK`].
Link,
/// Runs' links and fonts past [`MAX_RUN_STRINGS`].
RunStrings,
/// A style name cut to [`MAX_FONT_NAME`].
StyleName,
}
impl Dropped {
fn message(self) -> String {
match self {
Dropped::Font => format!("import: font names longer than {MAX_FONT_NAME} bytes are dropped; that text keeps the surrounding font"),
Dropped::Link => format!("import: links longer than {MAX_LINK} bytes are dropped; their text is kept"),
Dropped::RunStrings => {
format!("import: the document's runs carry more than {MAX_RUN_STRINGS} bytes of links and font names; later runs lose them")
}
Dropped::StyleName => format!("import: style names longer than {MAX_FONT_NAME} bytes are cut short"),
}
}
}
thread_local! {
/// The kinds of [`Dropped`] already reported for the document being imported on this thread.
/// Per thread because a document is parsed and built on one, and the parsers drop things
/// from places that have no log of their own to hand.
static REPORTED: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };
}
/// Log that the importer left something of `kind` out: once per kind per imported document
/// ([`begin_import`]), not once per item.
pub(crate) fn dropped(kind: Dropped) {
let bit = 1u8 << (kind as u8);
if REPORTED.with(|r| r.replace(r.get() | bit)) & bit == 0 {
log::warn!("{}", kind.message());
}
}
/// Start a new document's reports: each importer's entry point calls this first.
pub(crate) fn begin_import() {
REPORTED.with(|r| r.set(0));
}
/// Character formatting the text formats can express.
#[derive(Clone, Debug, Default, PartialEq)]
@@ -30,11 +108,13 @@ pub struct Fmt {
pub code: bool,
pub sup: bool,
pub sub: bool,
pub link: Option<String>,
/// Shared: every span of a link holds the same target.
pub link: Option<Arc<str>>,
pub color: Option<Rgb>,
/// Points.
pub size: Option<f32>,
pub font: Option<String>,
/// Shared, as `link`.
pub font: Option<Arc<str>>,
/// Background (highlight / shading).
pub background: Option<Rgb>,
/// Tracked insertion of this text (formats that keep revisions: RTF).
@@ -239,15 +319,13 @@ impl Para {
/// Remove the characters in `start` from the start of the paragraph's text and those in
/// `end` from its end.
pub fn trim_with(&mut self, start: &[char], end: &[char]) {
while let Some(Inline::Text(t, _)) = self.inlines.first_mut() {
let n = t.trim_start_matches(start).len();
let cut = t.len() - n;
// The leading spans that trim to nothing go in one `drain`: removing them one at a time
// shifted the rest each time, quadratic in a run of separately formatted line breaks.
let k = self.inlines.iter().take_while(|i| matches!(i, Inline::Text(t, _) if t.trim_start_matches(start).is_empty())).count();
self.inlines.drain(..k);
if let Some(Inline::Text(t, _)) = self.inlines.first_mut() {
let cut = t.len() - t.trim_start_matches(start).len();
t.replace_range(..cut, "");
if t.is_empty() {
self.inlines.remove(0);
} else {
break;
}
}
while let Some(Inline::Text(t, _)) = self.inlines.last_mut() {
let n = t.trim_end_matches(end).len();
@@ -431,10 +509,10 @@ pub fn char_props(f: &Fmt) -> CharProps {
} else {
None
},
link: f.link.clone().filter(|l| !l.is_empty()),
link: f.link.as_deref().filter(|l| !l.is_empty()).map(str::to_string),
color: f.color.map(TextColor::Rgb),
size: f.size.filter(|s| s.is_finite() && *s >= 1.0 && *s <= 1638.0),
font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.clone().filter(|s| !s.is_empty()) },
font: if f.code { Some(MONO_FONT.to_string()) } else { f.font.as_deref().filter(|s| !s.is_empty()).map(str::to_string) },
shading: f.background,
..Default::default()
}
@@ -518,6 +596,8 @@ struct Builder<'a> {
doc: &'a mut Document,
/// Revisions already in `doc` by (kind, author, date), so each change is added once.
revs: std::collections::HashMap<(u8, RevMark), u32>,
/// Link and font bytes given to runs so far ([`MAX_RUN_STRINGS`]).
run_strings: usize,
}
fn rev_key(kind: RevisionKind) -> u8 {
@@ -537,7 +617,7 @@ impl<'a> Builder<'a> {
revs.entry((rev_key(r.kind), mark)).or_insert(i as u32);
}
}
Builder { doc, revs }
Builder { doc, revs, run_strings: 0 }
}
/// The document revision for a tracked change, added the first time it is seen.
@@ -552,6 +632,22 @@ impl<'a> Builder<'a> {
i
}
/// `cp` as a run may keep it: without its link and font once the runs' copies of them are
/// over budget.
fn budget(&mut self, mut cp: CharProps) -> CharProps {
let n = cp.link.as_ref().map_or(0, String::len).saturating_add(cp.font.as_ref().map_or(0, String::len));
match self.run_strings.checked_add(n).filter(|&t| t <= MAX_RUN_STRINGS) {
Some(t) => self.run_strings = t,
None => {
if cp.link.is_some() || cp.font.is_some() {
dropped(Dropped::RunStrings);
}
(cp.link, cp.font) = (None, None);
}
}
cp
}
fn para(&mut self, p: &Para, list_num: Option<u32>) -> Paragraph {
let mut out = Paragraph::new();
out.props = para_props(p, list_num);
@@ -577,11 +673,12 @@ impl<'a> Builder<'a> {
match i {
Inline::Text(t, f) => {
let t = clean_text(t);
let mut cp = char_props(f);
let mut cp = self.budget(char_props(f));
cp.ins = f.ins.as_ref().map(|m| self.revision(RevisionKind::Insert, m));
cp.del = f.del.as_ref().map(|m| self.revision(RevisionKind::Delete, m));
if let Some(c) = &f.fmt_change {
cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), char_props(&c.old));
let old = self.budget(char_props(&c.old));
cp.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, &c.mark), old);
}
if p.kind == Kind::Code {
// Code's font comes from its style, before a change as after it.
@@ -616,7 +713,7 @@ impl<'a> Builder<'a> {
}
Inline::Ruby { base, ruby, fmt } => {
let (base, ruby) = (clean_text(base), clean_text(ruby));
let cp = char_props(fmt);
let cp = self.budget(char_props(fmt));
// A source without a base or without a reading (malformed markup) is just its
// text: a guide that annotates nothing is no guide.
if base.is_empty() || ruby.is_empty() {
@@ -883,13 +980,13 @@ fn fmt_of(doc: &Document, cp: &CharProps, in_code: bool) -> Fmt {
code,
sup: e.vert_align == Some(VertAlign::Superscript),
sub: e.vert_align == Some(VertAlign::Subscript),
link: e.link.clone(),
link: e.link.as_deref().map(Arc::from),
color: match e.color {
Some(TextColor::Rgb(c)) => Some(c),
_ => None,
},
size: e.size,
font: if code || in_code { None } else { e.shown_font().map(str::to_string) },
font: if code || in_code { None } else { e.shown_font().map(Arc::from) },
background: e.highlight.and_then(|h| h.rgb()).or(e.shading),
ins: None,
del: None,
@@ -1558,4 +1655,29 @@ mod tests {
assert_eq!(t.rows[1].len(), 2);
assert!(t.rows[1][0].covered);
}
#[test]
fn trim_drops_a_long_run_of_blank_spans_in_linear_time() {
// Separately formatted line breaks don't merge; each used to be removed with
// `Vec::remove(0)`, shifting all the rest (quadratic).
let mut p = Para::default();
let (b, i) = (Fmt { bold: true, ..Default::default() }, Fmt { italic: true, ..Default::default() });
for _ in 0..200_000 {
p.push_text("\n", &b);
p.push_text(" ", &i);
}
p.push_text(" x ", &Fmt::default());
let t = std::time::Instant::now();
p.trim();
// A fraction of a second; `Vec::remove(0)` per span ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert_eq!(p.inlines, vec![Inline::Text("x".into(), Fmt::default())]);
// The first span left keeps its formatting and loses only its leading blanks.
let mut p = Para::default();
p.push_text(" \t", &b);
p.push_text(" a ", &i);
p.push_text("b ", &b);
p.trim();
assert_eq!(p.inlines, vec![Inline::Text("a ".into(), i), Inline::Text("b".into(), b)]);
}
}
crates/formats/src/odt.rs
+6 -4
View File
@@ -19,6 +19,7 @@
use std::collections::HashMap;
use std::io::{Cursor, Read, Write};
use std::sync::Arc;
use quick_xml::XmlVersion;
use quick_xml::events::{BytesStart, Event};
@@ -1034,7 +1035,7 @@ struct FmtDelta {
color: Option<Rgb>,
background: Option<Rgb>,
size: Option<f32>,
font: Option<String>,
font: Option<Arc<str>>,
code: Option<bool>,
}
@@ -1340,7 +1341,7 @@ impl Styles {
if model::is_mono(&fam) || get(&a, "style:font-pitch").is_some_and(|p| p.trim().eq_ignore_ascii_case("fixed")) {
d.code = Some(true);
} else if !fam.is_empty() {
d.font = Some(fam);
d.font = model::kept_font(&fam);
}
}
}
@@ -1446,7 +1447,7 @@ impl Styles {
/// The document's default font: the Standard paragraph style's, else the default style's.
fn doc_font(&self) -> Option<String> {
self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.clone()).or_else(|| self.default_font.clone())
self.map.get("Standard").filter(|s| !s.auto).and_then(|s| s.fmt.font.as_deref().map(str::to_string)).or_else(|| self.default_font.clone())
}
fn chain<'a>(&'a self, name: &'a str) -> Vec<(&'a str, &'a StyleInfo)> {
@@ -2053,7 +2054,7 @@ impl Body<'_> {
"a" => {
let mut f = self.cur_fmt();
if let Some(h) = get(a, "xlink:href").filter(|h| !h.is_empty()) {
f.link = Some(h.to_string());
f.link = model::kept_link(h);
}
if let Some(s) = get(a, "text:style-name") {
f = self.styles.text_fmt(s, &f);
@@ -2626,6 +2627,7 @@ fn parse_tracked(bytes: &[u8]) -> Result<(Flow, SectionProps, Option<bool>, Opti
/// Parse an `.odt` package into a document.
pub fn import(bytes: &[u8]) -> Result<Document, String> {
model::begin_import();
let (flow, last, track, font, parts) = parse_tracked(bytes)?;
let mut doc = model::to_doc(&flow);
if font.is_some() {
crates/formats/src/rtf.rs
+17 -8
View File
@@ -19,6 +19,7 @@
//! tables. Group nesting and picture sizes are capped.
use std::collections::HashMap;
use std::sync::Arc;
use wordcraft_doc::props::VAlign;
use wordcraft_doc::{Align, Document, Rgb};
@@ -306,11 +307,13 @@ impl Writer {
"\\nosupersub"
});
}
let font = |f: &Fmt| if f.code { Some(model::MONO_FONT.to_string()) } else { f.font.clone() };
fn font(f: &Fmt) -> Option<&str> {
if f.code { Some(model::MONO_FONT) } else { f.font.as_deref() }
}
let old_font = font(old);
if old_font != font(cur) {
// No font is the document's default font, font 0 (Normal's).
let i = old_font.map_or(0, |n| self.t.font(&n));
let i = old_font.map_or(0, |n| self.t.font(n));
ctl.push_str(&format!("\\f{i}"));
}
let half_points = |s: Option<f32>| s.map(|s| (s * 2.0).round().clamp(2.0, 3276.0) as i32);
@@ -1106,7 +1109,7 @@ struct Reader {
st: St,
stack: Vec<St>,
overflow: usize,
fonts: HashMap<i32, String>,
fonts: HashMap<i32, Arc<str>>,
font_cur: i32,
font_name: String,
/// Code pages of the fonts whose charset names one (`\fcharset204` → 1251).
@@ -1398,7 +1401,9 @@ impl Reader {
fn commit_font(&mut self) {
let name = std::mem::take(&mut self.font_name).trim().trim_end_matches(';').trim().to_string();
if !name.is_empty() && self.fonts.len() < 4096 {
if let Some(name) = model::kept_font(&name).filter(|n| !n.is_empty())
&& self.fonts.len() < 4096
{
self.fonts.insert(self.font_cur, name);
if let Some(cp) = self.font_cp_cur.take() {
self.font_cps.insert(self.font_cur, cp);
@@ -1626,7 +1631,8 @@ impl Reader {
for c in s.chars() {
if c == ';' {
self.commit_font();
} else {
} else if self.font_name.len() <= model::MAX_FONT_NAME {
// One past the limit is enough for `commit_font` to drop the name.
self.font_name.push(c);
}
}
@@ -1646,8 +1652,10 @@ impl Reader {
for c in s.chars() {
if c == ';' {
self.commit_style();
} else {
} else if self.style_name.len() < model::MAX_FONT_NAME {
self.style_name.push(c);
} else {
model::dropped(model::Dropped::StyleName);
}
}
}
@@ -2329,7 +2337,7 @@ impl Reader {
"fldrslt" => {
let link = self.st.field.and_then(|i| self.fields.get(i)).and_then(|s| hyperlink(s));
if link.is_some() {
self.st.fmt.link = link;
self.st.fmt.link = link.as_deref().and_then(model::kept_link);
}
}
_ => {}
@@ -2807,7 +2815,7 @@ fn parse_full(bytes: &[u8]) -> Result<Parsed, String> {
if blocks.is_empty() {
blocks.push(FBlock::Para(Para::default()));
}
let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).cloned();
let default_font = r.fonts.get(&r.deff).filter(|n| !model::is_mono(n)).map(|n| n.to_string());
Ok(Parsed { flow: Flow { blocks, meta: r.meta }, page: r.page, last_section, default_font, headers_footers })
}
@@ -2850,6 +2858,7 @@ fn column_grid(rows: &mut [Vec<Cell>], rights: &[Vec<i32>]) -> Vec<f32> {
/// Parse RTF bytes into a document.
pub fn import(bytes: &[u8]) -> Result<Document, String> {
model::begin_import();
let Parsed { flow, last_section, default_font, headers_footers, .. } = parse_full(bytes)?;
let mut doc = model::to_doc_in(&flow, last_section);
// Each section's own headers and footers; those it leaves out stay linked to the previous.
crates/formats/src/sheet.rs
+138 -18
View File
@@ -18,7 +18,7 @@
//! all), sheets, rows, columns, cells and string lengths are capped, repeat counts are clamped to
//! what is left, and nothing recurses, so XML depth costs nothing.
use std::collections::{BTreeMap, HashMap};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{Cursor, Read};
use quick_xml::XmlVersion;
@@ -138,18 +138,28 @@ pub fn read_rows(bytes: &[u8], pick: Pick<'_>) -> Result<(Vec<String>, Vec<Vec<S
/// repeats numbered (`Name`, `Name 2`), ignoring case as merge fields do.
pub fn field_names(row: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::with_capacity(row.len());
// The names given so far, ASCII-lowercased, and per base name the next number to try: a set
// lookup per candidate, and each base resumes where it stopped, instead of rescanning every
// name from `2` for every repeat (cubic on a row of equal headers).
let mut taken: HashSet<String> = HashSet::with_capacity(row.len());
let mut next: HashMap<String, usize> = HashMap::new();
for (i, h) in row.iter().enumerate() {
let base = match h.trim() {
"" => format!("Column {}", i + 1),
t => t.to_string(),
};
let taken = |n: &str, out: &[String]| out.iter().any(|o| o.eq_ignore_ascii_case(n));
let mut name = base.clone();
let mut k = 2usize;
while taken(&name, &out) {
name = format!("{base} {k}");
k = k.saturating_add(1);
if taken.contains(&name.to_ascii_lowercase()) {
let k = next.entry(base.to_ascii_lowercase()).or_insert(2);
loop {
name = format!("{base} {k}");
*k = k.saturating_add(1);
if !taken.contains(&name.to_ascii_lowercase()) {
break;
}
}
}
taken.insert(name.to_ascii_lowercase());
out.push(name);
}
out
@@ -333,6 +343,8 @@ struct Grid {
rows: Vec<Vec<String>>,
cells: usize,
text: TextBudget,
/// A row has been left out for its text (and logged, once per sheet).
refused: bool,
}
impl Grid {
@@ -340,6 +352,15 @@ impl Grid {
self.rows.len() >= MAX_ROWS || self.cells >= MAX_CELLS || self.text.0 >= MAX_TEXT_BYTES
}
/// Note that a row is left out because its text is over what is left of [`MAX_TEXT_BYTES`]:
/// logged once per sheet, not once per row.
fn refuse(&mut self) {
if !self.refused {
self.refused = true;
log::warn!("sheet: rows with more text than the {MAX_TEXT_BYTES} byte budget has left are left out of the recipient list");
}
}
/// Keep `row` (trailing blanks trimmed) `times` times, as far as the caps allow.
fn push(&mut self, mut row: Vec<String>, times: u64) {
while row.last().is_some_and(String::is_empty) {
@@ -350,14 +371,15 @@ impl Grid {
}
let mut left = times;
while left > 0 && !self.full() {
let room = MAX_CELLS.saturating_sub(self.cells);
let mut r = row.clone();
r.truncate(room);
// What fits is measured on `row` itself; only a copy that will be kept is made.
let keep = row.len().min(MAX_CELLS.saturating_sub(self.cells));
let r = row.get(..keep).unwrap_or_default();
if !self.text.take(r.iter().map(String::len).sum()) {
self.refuse();
return;
}
self.cells = self.cells.saturating_add(r.len());
self.rows.push(r);
self.rows.push(r.to_vec());
left -= 1;
}
}
@@ -854,7 +876,9 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
// one is the wanted sheet.
let mut tables = 0usize;
let mut collecting = false;
let mut row: Option<(Vec<String>, u64, u64)> = None; // cells, pending blank cells, repeat
// Cells, pending blank cells, repeat, and the cells' text bytes (`None` once the row is over
// the text budget and so refused).
let mut row: Option<(Vec<String>, u64, u64, Option<usize>)> = None;
let mut cell: Option<OCell> = None;
let (mut in_p, mut skip) = (0usize, 0usize);
walk(xml, |ev| match ev {
@@ -875,7 +899,7 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
match n {
b"table-row" => {
let repeat = attr(e, b"number-rows-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1);
row = Some((Vec::new(), 0, repeat));
row = Some((Vec::new(), 0, repeat, Some(0)));
}
b"table-cell" | b"covered-table-cell" if row.is_some() => {
let repeat = attr(e, b"number-columns-repeated").and_then(|v| v.trim().parse().ok()).unwrap_or(1u64).max(1);
@@ -959,22 +983,33 @@ fn ods_read(xml: &[u8], want: Option<usize>) -> Result<(Vec<String>, Vec<Vec<Str
if n == b"covered-table-cell" {
skip = skip.saturating_sub(1);
}
let (Some(c), Some((cells, pending, _))) = (cell.take(), row.as_mut()) else { return };
let (Some(c), Some((cells, pending, _, budget))) = (cell.take(), row.as_mut()) else { return };
let Some(bytes) = budget else { return };
let text = ods_value(&c);
if text.is_empty() {
*pending = pending.saturating_add(c.repeat);
return;
}
// Blank cells before this one, then the value as often as it repeats, as far
// as the columns allow.
let blanks = (*pending).min(MAX_COLS.saturating_sub(cells.len()) as u64) as usize;
// as the columns and the grid's cells left (all `Grid::push` keeps) allow.
let room = MAX_COLS.min(MAX_CELLS.saturating_sub(grid.cells));
let blanks = (*pending).min(room.saturating_sub(cells.len()) as u64) as usize;
let times = c.repeat.min(room.saturating_sub(cells.len().saturating_add(blanks)) as u64) as usize;
// The row's text has to fit what is left of the budget to be kept at all
// (`Grid::push`): one that can't is refused now, before the copies are made.
*bytes = bytes.saturating_add(text.len().saturating_mul(times));
if *bytes > MAX_TEXT_BYTES.saturating_sub(grid.text.0) {
cells.clear();
*budget = None;
grid.refuse();
return;
}
cells.resize(cells.len() + blanks, String::new());
*pending = 0;
let times = c.repeat.min(MAX_COLS.saturating_sub(cells.len()) as u64) as usize;
cells.extend(std::iter::repeat_n(text, times));
}
b"table-row" => {
if let Some((cells, _, repeat)) = row.take() {
if let Some((cells, _, repeat, Some(_))) = row.take() {
grid.push(cells, repeat);
}
}
@@ -997,7 +1032,12 @@ fn ods_value(c: &OCell) -> String {
("time", Some(v)) => iso_time(v).unwrap_or_else(|| c.text.clone()),
("boolean", Some(v)) => if v.eq_ignore_ascii_case("true") || v == "1" { "TRUE" } else { "FALSE" }.into(),
("percentage" | "currency", Some(v)) if c.text.trim().is_empty() => v.parse::<f64>().map(number).unwrap_or_else(|_| c.text.clone()),
("string", Some(v)) if c.text.trim().is_empty() => v.to_string(),
("string", Some(v)) if c.text.trim().is_empty() => {
// Attribute text obeys the same per-cell cap as `<text:p>` text.
let mut s = String::new();
push_capped(&mut s, v);
s
}
_ => c.text.clone(),
}
}
@@ -1204,4 +1244,84 @@ mod tests {
assert_eq!(rows.len(), 1);
assert_eq!(rows[0], vec!["TRUE", "World", "Name"]);
}
#[test]
fn repeated_headers_are_numbered_in_linear_time() {
// Every repeat used to restart at `Name 2` and compare each candidate with every name
// so far: cubic in the number of equal headers.
let row = vec!["Name".to_string(); 5_000];
let t = std::time::Instant::now();
let names = field_names(&row);
// Milliseconds; the old numbering ran past 6 minutes.
assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed());
assert_eq!(names.len(), 5_000);
assert_eq!(names[0], "Name");
assert_eq!(names[4_999], "Name 5000");
// Names a header already has stay collision-free, ignoring case.
let row: Vec<String> = ["Name", "Name 2", "Name", "name", "", "Column 5", "NAME 2"].iter().map(|s| s.to_string()).collect();
assert_eq!(field_names(&row), ["Name", "Name 2", "Name 3", "name 4", "Column 5", "Column 5 2", "NAME 2 2"]);
}
fn ods(rows: &str) -> Vec<u8> {
let content = format!(
r#"<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"><office:body><office:spreadsheet><table:table table:name="Sheet1">{rows}</table:table></office:spreadsheet></office:body></office:document-content>"#
);
zip(&[("mimetype", "application/vnd.oasis.opendocument.spreadsheet"), ("content.xml", &content)])
}
#[test]
fn ods_string_attributes_are_capped_before_repeating() {
// An `office:string-value` is a cell's text: it obeys MAX_CELL_CHARS like `<text:p>`
// does, so repeating it across the columns can't build a huge row first.
let long = "v".repeat(MAX_CELL_CHARS * 4);
let book = ods(&format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{long}" table:number-columns-repeated="3"/></table:table-row>"#
));
let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].len(), 3);
assert!(rows[0].iter().all(|c| c.chars().count() == MAX_CELL_CHARS));
// A row whose repeated text alone is over the text budget is dropped as it is built
// (it was built in full, then cloned, then refused); the rows after it still count.
let wide = "€".repeat(MAX_CELL_CHARS);
let book = ods(&format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row><table:table-row><table:table-cell><text:p>next</text:p></table:table-cell></table:table-row>"#
));
let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert_eq!(rows, vec![vec!["next".to_string()]]);
// The row left out is said to be (review of #3873).
assert_eq!(warned.iter().filter(|w| w.contains("are left out of the recipient list")).count(), 1, "{warned:?}");
}
#[test]
fn rows_over_the_text_budget_are_logged_once_per_sheet() {
let wide = "€".repeat(MAX_CELL_CHARS);
let over = format!(
r#"<table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row>"#
);
let book = ods(&format!("{over}{over}{over}<table:table-row><table:table-cell><text:p>next</text:p></table:table-cell></table:table-row>"));
let ((_, rows), warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert_eq!(rows, vec![vec!["next".to_string()]]);
assert_eq!(warned.iter().filter(|w| w.contains("recipient list")).count(), 1, "one warning, not one per row: {warned:?}");
let book = ods("<table:table-row><table:table-cell><text:p>fine</text:p></table:table-cell></table:table-row>");
let (_, warned) = crate::test_log::warnings(|| read_rows(&book, Pick::Index(0)).unwrap());
assert!(warned.is_empty(), "{warned:?}");
}
#[test]
fn ods_repeated_text_is_budgeted_for_the_cells_the_grid_keeps() {
// One cell left in the grid: of a long value repeated a thousand times only the first
// copy is kept, and it fits the text budget, so the row isn't refused for the others.
let wide = "€".repeat(MAX_CELL_CHARS);
let full = MAX_CELLS / MAX_COLS - 1;
let book = ods(&format!(
r#"<table:table-row table:number-rows-repeated="{full}"><table:table-cell office:value-type="string" office:string-value="x" table:number-columns-repeated="{MAX_COLS}"/></table:table-row><table:table-row><table:table-cell office:value-type="string" office:string-value="x" table:number-columns-repeated="{}"/></table:table-row><table:table-row><table:table-cell office:value-type="string" office:string-value="{wide}" table:number-columns-repeated="1000"/></table:table-row>"#,
MAX_COLS - 1
));
let (_, rows) = read_rows(&book, Pick::Index(0)).unwrap();
assert_eq!(rows.len(), full + 2);
assert_eq!(rows.last(), Some(&vec![wide]));
assert_eq!(rows.iter().map(Vec::len).sum::<usize>(), MAX_CELLS);
}
}
crates/formats/src/sheet/xls.rs
+55 -1
View File
@@ -44,7 +44,7 @@ const TABLE: u16 = 0x0236;
/// Whether `bytes` is a compound file holding an Excel workbook stream.
pub fn is_xls(bytes: &[u8]) -> bool {
if !bytes.starts_with(&MAGIC) {
if !bytes.starts_with(&MAGIC) || wordcraft_doc::cfb_guard::check(bytes).is_err() {
return false;
}
let Ok(comp) = cfb::CompoundFile::open(Cursor::new(bytes)) else { return false };
@@ -68,6 +68,7 @@ pub fn read_rows(bytes: &[u8], pick: super::Pick<'_>) -> Result<(Vec<String>, Ve
}
fn workbook_stream(bytes: &[u8]) -> Result<Vec<u8>, String> {
wordcraft_doc::cfb_guard::check(bytes).map_err(|e| format!("not a valid Excel 97–2003 workbook ({e})"))?;
let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)).map_err(|e| format!("not a valid Excel 97–2003 workbook ({e})"))?;
if !comp.is_stream("/Workbook") {
if comp.is_stream("/Book") {
@@ -540,6 +541,59 @@ mod tests {
r.iter().map(|r| r.iter().map(|c| c.to_string()).collect()).collect()
}
/// #3642: a DIFAT naming one FAT sector thousands of times is refused before `cfb` builds
/// a FAT far larger than the file.
#[test]
fn a_repeating_difat_is_refused_before_cfb_opens_it() {
let f = wordcraft_doc::cfb_guard::repeating_difat_sample();
assert!(!is_xls(&f));
assert!(sheet_names(&f).is_err_and(|e| e.contains("FAT sectors in a file of")));
}
/// #3642 review: valid workbooks pass the guard and open. The repository keeps no files
/// Excel made, so these are written by the `cfb` crate in the shape of Excel 97–2003 ones:
/// the Workbook stream [`build`] writes with the summary streams Excel adds, in 512-byte
/// sectors (version 3, Excel's own) and 4096-byte ones (version 4); and one with an 8 MB
/// stream beside it, past what the header's 109 FAT sector slots map in 512-byte sectors, so
/// its FAT sectors are listed in a DIFAT chain too, which the guard walks.
#[test]
fn valid_xls_shaped_compound_files_pass_the_guard_and_open() {
use std::io::{Read, Write};
let book = build(&[("Sheet1", rows(&[&["a", "1"], &["b", "2"]]))]).unwrap();
let mut workbook = Vec::new();
cfb::CompoundFile::open(Cursor::new(book.as_slice())).unwrap().open_stream("/Workbook").unwrap().read_to_end(&mut workbook).unwrap();
let file = |version: cfb::Version, streams: &[(&str, &[u8])]| {
let mut comp = cfb::CompoundFile::create_with_version(version, Cursor::new(Vec::new())).unwrap();
for (name, data) in streams {
comp.create_stream(format!("/{name}")).unwrap().write_all(data).unwrap();
}
comp.into_inner().into_inner()
};
let summary = vec![0xFEu8; 4096];
let big: Vec<u8> = (0..8usize << 20).map(|i| (i % 251) as u8).collect();
let files = [
("as built", book.clone()),
(
"version 3",
file(
cfb::Version::V3,
&[("Workbook", &workbook), ("\u{5}SummaryInformation", &summary), ("\u{5}DocumentSummaryInformation", &summary)],
),
),
("version 4", file(cfb::Version::V4, &[("Workbook", &workbook), ("\u{5}SummaryInformation", &summary)])),
("with a DIFAT chain", file(cfb::Version::V3, &[("Workbook", &workbook), ("Ctls", &big)])),
];
let difat_sectors = |f: &[u8]| f.get(0x48..0x4C).map_or(0, |b| u32::from_le_bytes([b[0], b[1], b[2], b[3]]));
assert!(difat_sectors(&files[3].1) > 0, "the 8 MB file lists FAT sectors past the header's");
for (what, f) in &files {
assert_eq!(wordcraft_doc::cfb_guard::check(f), Ok(()), "{what}");
assert!(is_xls(f), "{what}");
assert_eq!(sheet_names(f).unwrap(), vec!["Sheet1"], "{what}");
let (_, r) = read_rows(f, Pick::Index(0)).unwrap();
assert_eq!(r, rows(&[&["a", "1"], &["b", "2"]]), "{what}");
}
}
#[test]
fn reads_a_biff8_workbook() {
let book = build(&[
crates/formats/src/test_log.rs
+36
View File
@@ -0,0 +1,36 @@
//! Test helper: the warnings this crate logs, captured per test thread, so a test can check that
//! a limit which leaves something out says so.
use std::cell::RefCell;
struct Capture;
thread_local! {
static LINES: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}
impl log::Log for Capture {
fn enabled(&self, m: &log::Metadata) -> bool {
m.level() <= log::Level::Warn
}
fn log(&self, r: &log::Record) {
if self.enabled(r.metadata()) {
LINES.with(|l| l.borrow_mut().push(r.args().to_string()));
}
}
fn flush(&self) {}
}
static CAPTURE: Capture = Capture;
/// `f`'s result and the warnings logged on this thread while it ran.
pub(crate) fn warnings<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
// Only this logger is ever installed in a test binary; a second call keeps it.
let _ = log::set_logger(&CAPTURE);
log::set_max_level(log::LevelFilter::Warn);
LINES.with(|l| l.borrow_mut().clear());
let out = f();
(out, LINES.with(|l| std::mem::take(&mut *l.borrow_mut())))
}
crates/formats/src/tests.rs
+132 -5
View File
@@ -748,8 +748,11 @@ fn code_spans_and_links_around_backticks_read_as_before() {
// inside a code span in link text, and link text with a backtick that nothing closes.
let flow = crate::markdown::parse("\\``e` `` a ` b `` then ``` and `c` [x `]` y](http://a.org) [p ` q](http://b.org) ``f");
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
let spans: Vec<(String, bool, Option<String>)> =
p.inlines.iter().filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.clone())) } else { None }).collect();
let spans: Vec<(String, bool, Option<String>)> = p
.inlines
.iter()
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.code, f.link.as_deref().map(str::to_string))) } else { None })
.collect();
let code: Vec<&str> = spans.iter().filter(|s| s.1).map(|s| s.0.as_str()).collect();
assert_eq!(code, ["e", "a ` b", "c", "]"], "{spans:?}");
let linked = |url: &str| spans.iter().filter(|s| s.2.as_deref() == Some(url)).map(|s| s.0.as_str()).collect::<String>();
@@ -766,7 +769,7 @@ fn latex_href_keeps_ampersand() {
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.clone()) } else { None })
.filter_map(|i| if let model::Inline::Text(_, f) = i { Some(f.link.as_deref().map(str::to_string)) } else { None })
.collect();
assert_eq!(links, [Some("https://e.com/a?b=1&c=2".to_string())]);
@@ -775,7 +778,7 @@ fn latex_href_keeps_ampersand() {
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.clone())) } else { None })
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.to_string(), f.link.as_deref().map(str::to_string))) } else { None })
.collect();
let url = "https://e.com/a?b=1&c=2".to_string();
assert_eq!(texts, [(url.clone(), Some(url))]);
@@ -789,7 +792,8 @@ fn brackets_do_not_turn_later_links_into_text() {
let links = |md: &str| {
let flow = crate::markdown::parse(md);
let FBlock::Para(p) = &flow.blocks[0] else { panic!("{:?}", flow.blocks) };
let mut urls: Vec<String> = p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.clone() } else { None }).collect();
let mut urls: Vec<String> =
p.inlines.iter().filter_map(|i| if let model::Inline::Text(_, f) = i { f.link.as_deref().map(str::to_string) } else { None }).collect();
urls.dedup();
urls
};
@@ -863,6 +867,21 @@ fn exported_links_keep_only_safe_targets() {
}
}
/// A link target with any whitespace (a no-break or ideographic space too) is written in `<…>`,
/// so the reader, which ends a bare destination at the first whitespace, reads it back whole.
#[test]
fn markdown_link_targets_with_unicode_spaces_read_back() {
for l in ["https://e.com/a\u{a0}b", "https://e.com/a\u{3000}b", "https://e.com/a b"] {
let mut d = Document::new();
let mut p = Paragraph::with_text("", CharProps::default());
p.insert_text(0, "go", &CharProps { link: Some(l.to_string()), ..Default::default() }).unwrap();
d.body = vec![para_block(p)];
let md = String::from_utf8(export("md", &d).unwrap().unwrap()).unwrap();
let sp = spans(&crate::markdown::parse(&md));
assert_eq!(sp.iter().find(|(t, _)| t == "go").and_then(|(_, f)| f.link.as_deref()), Some(l), "md: {md}");
}
}
#[test]
fn hidden_drawings_are_left_out_of_exports() {
let mut d = wordcraft_doc::Document::from_text("x");
@@ -875,3 +894,111 @@ fn hidden_drawings_are_left_out_of_exports() {
// Two pictures, one hidden: one comes out.
assert_eq!(crate::html::export(&d).matches("<img").count(), 1);
}
/// Every text span of the flow, with its formatting.
fn spans(flow: &model::Flow) -> Vec<(String, model::Fmt)> {
flow.blocks
.iter()
.filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None })
.flat_map(|p| p.inlines.iter())
.filter_map(|i| if let model::Inline::Text(t, f) = i { Some((t.clone(), f.clone())) } else { None })
.collect()
}
/// A font name or link target too long for any real one is dropped, its text kept: every run
/// carried its own copy, so one long name times many runs was gigabytes (#3713).
#[test]
fn overlong_font_names_and_links_are_dropped() {
let long = "x".repeat(1 << 20);
let rtf = format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}}}\\f1 {}}}", "{\\b a}a".repeat(1000));
let flow = crate::rtf::parse(rtf.as_bytes()).unwrap();
let sp = spans(&flow);
assert!(sp.iter().map(|s| s.0.len()).sum::<usize>() >= 2000);
assert!(sp.iter().all(|(_, f)| f.font.is_none()), "an overlong RTF font name is no font");
// A real one still is.
let flow = crate::rtf::parse(b"{\\rtf1{\\fonttbl{\\f0 Arial;}{\\f1 Georgia;}}\\f1 a}").unwrap();
assert_eq!(spans(&flow)[0].1.font.as_deref(), Some("Georgia"));
let html =
format!("<p><span style=\"font-family:'{long}'\"><a href=\"https://e.com/{long}\">kept</a></span> <font face=\"{long}\">too</font></p>");
let sp = spans(&crate::html::parse(&html));
assert!(sp.iter().any(|(t, _)| t.contains("kept")) && sp.iter().any(|(t, _)| t.contains("too")));
assert!(sp.iter().all(|(_, f)| f.font.is_none() && f.link.is_none()), "{:?}", sp.iter().map(|s| &s.0).collect::<Vec<_>>());
let sp = spans(&crate::markdown::parse(&format!("[kept](https://e.com/{long})")));
assert_eq!(sp.len(), 1);
assert_eq!((sp[0].0.as_str(), sp[0].1.link.is_none()), ("kept", true));
}
/// Spans of one link share its target, and so do references to one definition: a long
/// definition and many references no longer copy the destination into each (#3713).
#[test]
fn spans_share_their_link_target() {
let dest = format!("https://e.com/{}", "x".repeat(4000));
let md = format!("{}\n\n[r]: {dest}\n", "[a][r] **[b][r]** ".repeat(10_000));
let sp = spans(&crate::markdown::parse(&md));
let links: Vec<_> = sp.iter().filter_map(|(_, f)| f.link.as_ref()).collect();
assert_eq!(links.len(), 20_000);
assert!(links.iter().all(|l| Arc::ptr_eq(l, links[0])), "one copy of the destination");
let html = format!("<a href=\"{dest}\">{}</a>", "<b>a</b>a".repeat(1000));
let sp = spans(&crate::html::parse(&html));
assert!(sp.len() >= 2000);
let first = sp[0].1.link.clone().unwrap();
assert!(sp.iter().all(|(_, f)| f.link.as_ref().is_some_and(|l| Arc::ptr_eq(l, &first))));
}
/// The document's runs own their link and font strings; past a budget further runs keep their
/// text but lose them, so a flow of shared names can't expand into gigabytes (#3713).
#[test]
fn run_link_and_font_copies_are_budgeted() {
let link: Arc<str> = format!("https://e.com/{}", "x".repeat(model::MAX_LINK - 20)).into();
let mut p = model::Para::default();
for i in 0..20_000 {
let f = model::Fmt { bold: i % 2 == 0, link: Some(link.clone()), ..Default::default() };
p.push_text("a", &f);
}
model::begin_import();
let (doc, warned) = crate::test_log::warnings(|| model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() }));
let paras: Vec<_> = doc.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p) } else { None }).collect();
assert_eq!(paras.iter().map(|p| p.text.len()).sum::<usize>(), 20_000, "all text kept");
let runs: Vec<_> = paras.iter().flat_map(|p| p.runs.iter()).collect();
let kept: usize = runs.iter().filter_map(|r| r.props.link.as_ref()).map(String::len).sum();
assert!(kept > 0 && kept <= 64 << 20, "{kept}");
// The runs that lose theirs are said to, once for the document (review of #3873).
assert_eq!(warned.iter().filter(|w| w.contains("later runs lose them")).count(), 1, "{warned:?}");
}
/// Every limit that leaves a font name, link or style name out says so, once per kind per
/// document rather than once per item (review of #3873; the tracker complains of silent loss).
#[test]
fn dropped_names_and_links_are_logged_once_per_document() {
let long = "x".repeat(model::MAX_LINK + 1);
let html = format!(
"<p><span style=\"font-family:'{long}'\"><a href=\"https://e.com/{long}\">a</a></span> <font face=\"{long}\">b</font> <a href=\"https://e.com/{long}\">c</a></p>"
);
let count = |w: &[String], what: &str| w.iter().filter(|l| l.contains(what)).count();
for _ in 0..2 {
// Each import is its own document: the second is reported too.
let (doc, warned) = crate::test_log::warnings(|| crate::html::import(&html));
assert!(doc.body.len() == 1, "the text is kept");
assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}");
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
}
let (_, warned) = crate::test_log::warnings(|| crate::html::import("<p><a href=\"https://e.com\" style=\"font-family:Georgia\">fine</a></p>"));
assert!(warned.is_empty(), "nothing left out, nothing said: {warned:?}");
let md = format!("[a](https://e.com/{long}) [b](https://e.com/{long})");
let (_, warned) = crate::test_log::warnings(|| crate::markdown::import(&md));
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
let rtf =
format!("{{\\rtf1{{\\fonttbl{{\\f0 Arial;}}{{\\f1 {long};}}{{\\f2 {long};}}}}{{\\stylesheet{{\\s1 {long};}}{{\\s2 {long};}}}}\\f1 a\\f2 b}}");
let (_, warned) = crate::test_log::warnings(|| crate::rtf::import(rtf.as_bytes()).unwrap());
assert_eq!(count(&warned, "font names longer"), 1, "{warned:?}");
assert_eq!(count(&warned, "style names longer"), 1, "{warned:?}");
let tex = format!("\\href{{https://e.com/{long}}}{{a}} \\url{{https://e.com/{long}}}");
let (_, warned) = crate::test_log::warnings(|| crate::latex::import(tex.as_bytes()));
assert_eq!(count(&warned, "links longer"), 1, "{warned:?}");
}
crates/geom/src/lib.rs
+1 -1
View File
@@ -13,7 +13,7 @@ pub mod picture;
mod region;
pub use region::Region;
mod pixels;
pub use pixels::{MAX_DECODED_BYTES, MAX_RASTER_PIXELS, decode_fits, fit_pixels, rgba_bytes, rgba_fits};
pub use pixels::{MAX_DECODED_BYTES, MAX_RASTER_PIXELS, decode_fits, decode_peak_fits, fit_pixels, rgba_bytes, rgba_fits};
/// Points per inch.
pub const PT_PER_IN: f32 = 72.0;
crates/geom/src/pixels.rs
+23 -2
View File
@@ -18,9 +18,18 @@ pub fn rgba_fits(w: u32, h: u32, budget: u64) -> bool {
}
/// Whether decoding a `w`×`h` picture fits `budget` bytes at its peak: the decoder's buffer and
/// the RGBA conversion (or its premultiplied copy) are alive together, so two RGBA buffers.
/// the RGBA conversion (or its premultiplied copy) are alive together, so two RGBA buffers. For a
/// decoder whose buffer isn't four bytes a pixel, see [`decode_peak_fits`].
pub fn decode_fits(w: u32, h: u32, budget: u64) -> bool {
w > 0 && h > 0 && rgba_bytes(w, h).and_then(|b| b.checked_mul(2)).is_some_and(|b| b <= budget)
decode_peak_fits(w, h, 4, budget)
}
/// [`decode_fits`] for a decoder that hands back `native` bytes a pixel (8 for 16-bit RGBA, 16 for
/// float RGBA): its buffer lives alongside the RGBA8 conversion, then the conversion alongside its
/// premultiplied copy, so the peak is the larger of `native + 4` and `8` bytes a pixel.
pub fn decode_peak_fits(w: u32, h: u32, native: u64, budget: u64) -> bool {
let per_pixel = native.saturating_add(4).max(8);
w > 0 && h > 0 && (w as u64).checked_mul(h as u64).and_then(|p| p.checked_mul(per_pixel)).is_some_and(|b| b <= budget)
}
/// `w`×`h` shrunk (aspect kept, each side at least 1) until it holds at most `max_pixels`.
@@ -53,6 +62,18 @@ mod tests {
assert!(!decode_fits(0, 8, u64::MAX));
}
/// A 16-bit RGBA decode (8 bytes a pixel) lives alongside its 4-byte RGBA8 conversion: 12 bytes
/// a pixel at the peak, not the 8 two RGBA8 buffers take.
#[test]
fn wide_decoder_buffers_are_charged_at_their_size() {
assert!(decode_peak_fits(16, 8, 4, 1024), "RGBA8: two RGBA8 buffers");
assert!(decode_peak_fits(16, 8, 1, 1024), "grey: the premultiplied copy is the peak");
assert!(!decode_peak_fits(16, 8, 8, 1024), "RGBA16 needs 1536 bytes");
assert!(decode_peak_fits(16, 8, 8, 1536));
assert!(!decode_peak_fits(16, 8, 16, 2559), "RGBA32F: 20 bytes a pixel");
assert!(!decode_peak_fits(u32::MAX, u32::MAX, u64::MAX, u64::MAX), "overflow is refused");
}
#[test]
fn fit_pixels_keeps_normal_sizes_and_shrinks_huge_ones() {
assert_eq!(fit_pixels(612, 792, MAX_RASTER_PIXELS), (612, 792));
crates/layout/src/balloons.rs
+4 -13
View File
@@ -20,7 +20,7 @@ use wordcraft_doc::props::{Border, BorderStyle, CharProps, Rgb, TextColor};
use wordcraft_doc::{Document, Paragraph, Pos, StoryRef};
use wordcraft_geom::Rect;
use crate::display::revision_color;
use crate::display::{author_order, revision_color};
use crate::{BoxContext, BoxLayout, Ctx, DocLayout, Edges, Placed, fit_box, layout_box, push_para};
/// Width of the markup area beside each page (points): the editor's.
@@ -103,17 +103,6 @@ pub fn replies(doc: &Document) -> BTreeMap<u32, Vec<u32>> {
m
}
/// The comment authors in order of first appearance (for their colours).
fn author_index(doc: &Document, author: &str) -> u32 {
let mut seen: Vec<&str> = Vec::new();
for c in doc.comments.values() {
if !seen.contains(&c.author.as_str()) {
seen.push(&c.author);
}
}
seen.iter().position(|a| *a == author).unwrap_or(0) as u32
}
fn author_name(a: &str) -> &str {
if a.trim().is_empty() { "Author" } else { a }
}
@@ -132,10 +121,12 @@ pub(crate) fn add(ctx: &mut Ctx, lay: &mut DocLayout) {
let card_w = MARKUP_AREA - 2.0 * SIDE;
let inner = card_w - STRIPE - 2.0 * PAD;
let mut cards: Vec<Card> = Vec::new();
// The comment authors in order of first appearance (for their colours), once for every card.
let authors = author_order(doc.comments.values().map(|c| c.author.as_str()));
for (id, pos) in anchors {
let Some(caret) = lay.caret_on(&pos, 0) else { continue };
let Some(c) = doc.comments.get(&id) else { continue };
let color = revision_color(author_index(doc, &c.author));
let color = revision_color(authors.get(c.author.as_str()).copied().unwrap_or(0));
let mut thread = vec![id];
thread.extend(threads.get(&id).into_iter().flatten().copied());
let x = STRIPE + PAD;
crates/layout/src/display.rs
+86 -50
View File
@@ -1,5 +1,7 @@
//! Page → draw items, shared by the raster renderer, the PDF exporter and thumbnails.
use std::collections::HashMap;
use wordcraft_doc::graphic::{Graphic, GraphicItem, GraphicKind, PathSeg, TextAlign};
use wordcraft_doc::para::{InlineObject, ShapeKind};
use wordcraft_doc::props::{Border, BorderStyle, Rgb, TextColor, TextDirection, Underline};
@@ -530,8 +532,9 @@ fn stroke_of(u: Underline) -> Stroke {
pub fn page_display(doc: &Document, page: &Page, opts: &DisplayOptions) -> Vec<Draw> {
let mut out = Vec::new();
let (ha, ba) = dimming(opts);
stories_display(doc, &[&page.header, &page.footer], opts, ha, &mut out);
stories_display(doc, &[&page.items], opts, ba, &mut out);
let authors = Authors::new(doc);
stories_display(doc, &[&page.header, &page.footer], opts, &authors, ha, &mut out);
stories_display(doc, &[&page.items], opts, &authors, ba, &mut out);
body_change_bars(doc, page, opts, ba, &mut out);
out
}
@@ -542,8 +545,9 @@ pub fn page_display(doc: &Document, page: &Page, opts: &DisplayOptions) -> Vec<D
pub fn page_layers(doc: &Document, page: &Page, opts: &DisplayOptions) -> [(f32, Vec<Draw>); 2] {
let (ha, ba) = dimming(opts);
let (mut hf, mut body) = (Vec::new(), Vec::new());
stories_display(doc, &[&page.header, &page.footer], opts, 1.0, &mut hf);
stories_display(doc, &[&page.items], opts, 1.0, &mut body);
let authors = Authors::new(doc);
stories_display(doc, &[&page.header, &page.footer], opts, &authors, 1.0, &mut hf);
stories_display(doc, &[&page.items], opts, &authors, 1.0, &mut body);
body_change_bars(doc, page, opts, 1.0, &mut body);
[(ha, hf), (ba, body)]
}
@@ -561,7 +565,7 @@ fn body_change_bars(doc: &Document, page: &Page, opts: &DisplayOptions, alpha: f
}
/// Draw items for the placed items of `stories`, faded by `a`.
fn stories_display(doc: &Document, stories: &[&Vec<Placed>], opts: &DisplayOptions, a: f32, out: &mut Vec<Draw>) {
fn stories_display(doc: &Document, stories: &[&Vec<Placed>], opts: &DisplayOptions, authors: &Authors, a: f32, out: &mut Vec<Draw>) {
let hidden = opts.hide_object.as_ref().map(|p| Hidden::new(doc, p));
for items in stories {
let warps = warped_boxes(doc, items);
@@ -576,7 +580,7 @@ fn stories_display(doc: &Document, stories: &[&Vec<Placed>], opts: &DisplayOptio
&& let Some(rect) = cell_clip(&clips, *story, &path.0)
{
let mut inner = Vec::new();
item(doc, it, opts, a, &mut inner);
item(doc, it, opts, authors, a, &mut inner);
out.push(Draw::Clip { rect, items: inner });
continue;
}
@@ -590,10 +594,10 @@ fn stories_display(doc: &Document, stories: &[&Vec<Placed>], opts: &DisplayOptio
if let Some((_, rect, warp)) = warps.iter().find(|(w, ..)| w == id) {
let lines: Vec<&Placed> =
items.iter().filter(|i| matches!(i, Placed::Lines { story: StoryRef::Part(s), .. } if s == id)).collect();
warped_text(doc, &lines, *rect, warp, opts, a, out);
warped_text(doc, &lines, *rect, warp, opts, authors, a, out);
}
}
_ => item(doc, it, opts, a, out),
_ => item(doc, it, opts, authors, a, out),
}
}
}
@@ -606,35 +610,50 @@ pub fn object_display(doc: &Document, page: &Page, o: &InlineObject, rect: Rect,
let mut out = Vec::new();
object_draws(o, rect, Spin::default(), 1.0, opts, &mut out);
let Some(id) = text_box else { return out };
let authors = Authors::new(doc);
for items in [&page.items, &page.header, &page.footer] {
let lines: Vec<&Placed> = items.iter().filter(|i| matches!(i, Placed::Lines { story: StoryRef::Part(s), .. } if *s == id)).collect();
match warped_boxes(doc, items).into_iter().find(|(w, ..)| *w == id) {
Some((_, rect, warp)) if !lines.is_empty() => warped_text(doc, &lines, rect, &warp, opts, 1.0, &mut out),
_ => lines.into_iter().for_each(|it| item(doc, it, opts, 1.0, &mut out)),
Some((_, rect, warp)) if !lines.is_empty() => warped_text(doc, &lines, rect, &warp, opts, &authors, 1.0, &mut out),
_ => lines.into_iter().for_each(|it| item(doc, it, opts, &authors, 1.0, &mut out)),
}
}
out
}
/// The cells among `items` whose content is cut off at their edge (see [`Placed::Cell`]'s
/// `clip`): their story, the path of their content (table, row, cell) and their area.
fn clipped_cells(items: &[Placed]) -> Vec<(StoryRef, Vec<u32>, Rect)> {
items
.iter()
.filter_map(|it| match it {
Placed::Cell { rect, table, row, cell, story, clip: true, .. } => {
let mut prefix = table.0.clone();
prefix.extend([u32::try_from(*row).ok()?, u32::try_from(*cell).ok()?]);
Some((*story, prefix, *rect))
}
_ => None,
})
.collect()
/// `clip`), by story and the path of their content (table, row, cell): their area, and the
/// order they come in (the first of two that hold a line clips it).
type Clips = HashMap<StoryRef, HashMap<Vec<u32>, (usize, Rect)>>;
fn clipped_cells(items: &[Placed]) -> Clips {
let mut clips = Clips::new();
let cells = items.iter().filter_map(|it| match it {
Placed::Cell { rect, table, row, cell, story, clip: true, .. } => {
let mut prefix = table.0.clone();
prefix.extend([u32::try_from(*row).ok()?, u32::try_from(*cell).ok()?]);
Some((*story, prefix, *rect))
}
_ => None,
});
for (i, (story, prefix, rect)) in cells.enumerate() {
crate::work::steps(1);
clips.entry(story).or_default().entry(prefix).or_insert((i, rect));
}
clips
}
/// The area to clip a line at `path` of `story` to: its cell's, when that cell is clipped.
fn cell_clip(clips: &[(StoryRef, Vec<u32>, Rect)], story: StoryRef, path: &[u32]) -> Option<Rect> {
clips.iter().find(|(s, prefix, _)| *s == story && path.starts_with(prefix)).map(|(_, _, r)| *r)
/// The area to clip a line at `path` of `story` to: its cell's, when that cell is clipped. Only
/// the cells holding the line are looked up, not every clipped cell on the page.
fn cell_clip(clips: &Clips, story: StoryRef, path: &[u32]) -> Option<Rect> {
let cells = clips.get(&story)?;
(0..=path.len())
.filter_map(|n| {
crate::work::steps(1);
cells.get(path.get(..n)?)
})
.min_by_key(|(i, _)| *i)
.map(|(_, r)| *r)
}
/// The text boxes among `items` whose text is warped (Text Effects › Transform, drawn bent): their
@@ -661,12 +680,13 @@ fn warped_text(
rect: Rect,
warp: &wordcraft_doc::wordart::TextWarp,
opts: &DisplayOptions,
authors: &Authors,
alpha: f32,
out: &mut Vec<Draw>,
) {
let mut raw = Vec::new();
for it in lines {
item(doc, it, opts, alpha, &mut raw);
item(doc, it, opts, authors, alpha, &mut raw);
}
// The letters: runs with effects as they are, plain runs from their glyphs.
let mut arts: Vec<(BezPath, Box<wordcraft_doc::wordart::TextEffects>, Rgb, f32)> = Vec::new();
@@ -788,13 +808,13 @@ fn char_box(b: &Border, x0: f32, x1: f32, top: f32, bottom: f32, alpha: f32, out
out.push(rule(r, top, r, bottom, b, alpha));
}
fn item(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, out: &mut Vec<Draw>) {
item_in(doc, it, opts, alpha, Spin::default(), out)
fn item(doc: &Document, it: &Placed, opts: &DisplayOptions, authors: &Authors, alpha: f32, out: &mut Vec<Draw>) {
item_in(doc, it, opts, authors, alpha, Spin::default(), out)
}
/// [`item`] drawn inside the turn `outer` (a turned Drawing Canvas's members): a shape's
/// effects follow all the turns it is drawn inside.
fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer: Spin, out: &mut Vec<Draw>) {
fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, authors: &Authors, alpha: f32, outer: Spin, out: &mut Vec<Draw>) {
match it {
Placed::Fill { rect, color } => out.push(Draw::Fill { rect: *rect, color: *color, alpha, text_bg: false }),
Placed::Rule { x0, y0, x1, y1, border } => out.push(rule(*x0, *y0, *x1, *y1, border, alpha)),
@@ -814,14 +834,14 @@ fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer
Placed::Clip { rect, spin, items } => {
let mut inner = Vec::new();
for it in items {
item_in(doc, it, opts, alpha, spin.within(outer), &mut inner);
item_in(doc, it, opts, authors, alpha, spin.within(outer), &mut inner);
}
spun(*spin, *rect, vec![Draw::Clip { rect: *rect, items: inner }], out)
}
Placed::Cell { .. } | Placed::Object { .. } => {}
Placed::Lines { story, path, para, l0, l1, x, y, turn } if turn.is_turned() => {
let mut items = Vec::new();
lines(doc, *story, path, para, *l0, *l1, 0.0, 0.0, opts, alpha, *turn == TextDirection::Down, &mut items);
lines(doc, *story, path, para, *l0, *l1, 0.0, 0.0, opts, authors, alpha, *turn == TextDirection::Down, &mut items);
// Link areas are axis-aligned page rectangles: turned text gets none.
for d in &mut items {
if let Draw::Glyphs { link, .. } = d {
@@ -833,7 +853,7 @@ fn item_in(doc: &Document, it: &Placed, opts: &DisplayOptions, alpha: f32, outer
}
out.push(Draw::Turned { x: *x, y: *y, turn: *turn, items });
}
Placed::Lines { story, path, para, l0, l1, x, y, .. } => lines(doc, *story, path, para, *l0, *l1, *x, *y, opts, alpha, false, out),
Placed::Lines { story, path, para, l0, l1, x, y, .. } => lines(doc, *story, path, para, *l0, *l1, *x, *y, opts, authors, alpha, false, out),
}
}
@@ -851,6 +871,7 @@ fn lines(
x: f32,
y: f32,
opts: &DisplayOptions,
authors: &Authors,
alpha: f32,
down: bool,
out: &mut Vec<Draw>,
@@ -1078,7 +1099,7 @@ fn lines(
let shown = opts.markup && m.insertions_deletions;
let del = rc.del.filter(|_| shown);
let ins = rc.ins.filter(|_| shown && del.is_none());
let author = |r: u32| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0));
let author = |r: u32| authors.color(r);
// Moved text: the Moves marks and colours (by default green, double strikethrough
// where it was, double underline where it went).
let moved_to = doc.move_name(ins).is_some();
@@ -1411,18 +1432,12 @@ fn lines(
let mx = if line.rtl { ex - 1.0 - size * 0.6 } else { ex + 1.0 };
// A tracked (inserted or deleted) paragraph mark is drawn in its author's colour.
let m = &opts.revisions;
let by_author = |r: u32| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0));
let by_author = |r: u32| authors.color(r);
let color = para.filter(|_| opts.markup && m.insertions_deletions).and_then(|p| match (p.mark.ins, p.mark.del) {
(_, Some(r)) if doc.move_name(Some(r)).is_some() => Some(m.move_from_color.unwrap_or_else(|| by_author(r))),
(Some(r), None) if doc.move_name(Some(r)).is_some() => Some(m.move_to_color.unwrap_or_else(|| by_author(r))),
(_, Some(r)) => Some(
m.delete_color
.unwrap_or_else(|| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0))),
),
(Some(r), None) => Some(
m.insert_color
.unwrap_or_else(|| revision_color(doc.revisions.get(r as usize).map(|v| author_index(doc, &v.author)).unwrap_or(0))),
),
(_, Some(r)) => Some(m.delete_color.unwrap_or_else(|| by_author(r))),
(Some(r), None) => Some(m.insert_color.unwrap_or_else(|| by_author(r))),
_ => None,
});
// The last paragraph of a section ends in a section break instead of a plain ¶.
@@ -1851,14 +1866,35 @@ fn text_glyphs(doc: &Document, r: Rect, it: &GraphicItem, alpha: f32, k: f32) ->
})
}
fn author_index(doc: &Document, author: &str) -> u32 {
let mut seen: Vec<&str> = Vec::new();
for r in &doc.revisions {
if !seen.contains(&r.author.as_str()) {
seen.push(&r.author);
}
/// Each author's place among `authors` in order of first appearance (which colour marks their
/// changes and comments), found in one pass.
pub(crate) fn author_order<'a>(authors: impl Iterator<Item = &'a str>) -> HashMap<&'a str, u32> {
let mut seen = HashMap::new();
for a in authors {
crate::work::steps(1);
let n = u32::try_from(seen.len()).unwrap_or(u32::MAX);
seen.entry(a).or_insert(n);
}
seen
}
/// The colours of the document's revision authors ([`author_order`]), worked out once for a
/// drawing, when its first tracked change is drawn, not once for every change.
struct Authors<'a> {
doc: &'a Document,
order: std::cell::OnceCell<HashMap<&'a str, u32>>,
}
impl<'a> Authors<'a> {
fn new(doc: &'a Document) -> Self {
Authors { doc, order: std::cell::OnceCell::new() }
}
/// The colour of revision `r`'s author.
fn color(&self, r: u32) -> Rgb {
let order = self.order.get_or_init(|| author_order(self.doc.revisions.iter().map(|v| v.author.as_str())));
revision_color(self.doc.revisions.get(r as usize).and_then(|v| order.get(v.author.as_str()).copied()).unwrap_or(0))
}
seen.iter().position(|a| *a == author).unwrap_or(0) as u32
}
/// How far a legacy check box's square sits inside the room it takes on its line, on every side.
crates/layout/src/hit.rs
+68 -7
View File
@@ -1,5 +1,7 @@
//! Hit testing and caret geometry.
use std::collections::HashMap;
use wordcraft_doc::para::Wrap;
use wordcraft_doc::props::TextDirection;
use wordcraft_doc::{Document, Path, Pos, StoryRef};
@@ -140,7 +142,48 @@ fn text_at(p: &Page, index: usize, x: f32, y: f32) -> Option<StoryRef> {
})
}
/// The pieces of story `story`'s paragraphs on each page, by paragraph, for the pages looked at
/// so far ([`DocLayout::pieces_in`]).
struct PartPages<'a> {
story: StoryRef,
pages: HashMap<usize, HashMap<&'a Path, Vec<&'a Placed>>>,
}
impl<'a> PartPages<'a> {
fn new(story: StoryRef) -> Self {
PartPages { story, pages: HashMap::new() }
}
/// The pieces of the paragraph at `path` on page `pi` of `lay`, as [`DocLayout::pieces_on`]
/// gives them.
fn on(&mut self, lay: &'a DocLayout, pi: usize, path: &Path) -> Vec<(usize, &'a Placed)> {
let Some(p) = lay.pages.get(pi) else { return Vec::new() };
let story = self.story;
let index = self.pages.entry(pi).or_insert_with(|| {
let mut index: HashMap<&'a Path, Vec<&'a Placed>> = HashMap::new();
for it in items_of(p, story) {
if let Placed::Lines { story: s, path, .. } = it
&& *s == story
{
index.entry(path).or_default().push(it);
}
}
index
});
index.get(path).map(|v| v.iter().map(|it| (pi, *it)).collect()).unwrap_or_default()
}
}
/// The items of `page` that `story`'s pieces are among (each one looked at counted as a step of
/// [`crate::work`]).
fn items_of(page: &Page, story: StoryRef) -> Box<dyn Iterator<Item = &Placed> + '_> {
let items = story_items(page, story);
#[cfg(test)]
let items: Box<dyn Iterator<Item = &Placed> + '_> = Box::new(items.inspect(|_| crate::work::steps(1)));
items
}
fn story_items(page: &Page, story: StoryRef) -> Box<dyn Iterator<Item = &Placed> + '_> {
match story {
StoryRef::Body => Box::new(page.items.iter()),
// Headers and footers, and the text boxes in them, are drawn with the page's header and
@@ -417,6 +460,11 @@ impl DocLayout {
.map(|v| v.iter().filter_map(|(pi, ii)| self.pages.get(*pi).and_then(|p| p.items.get(*ii)).map(|it| (*pi, it))).collect())
.unwrap_or_default();
}
self.part_pieces(page_hint, &mut |pi| self.pieces_on(pi, story, path))
}
/// Pieces of a paragraph of a story part (see [`Self::pieces`]), `on` giving those on a page.
fn part_pieces<'a>(&'a self, page_hint: usize, on: &mut dyn FnMut(usize) -> Vec<(usize, &'a Placed)>) -> Vec<(usize, &'a Placed)> {
// Headers/footers repeat: prefer the hinted page, else the first that shows the part.
let mut order: Vec<usize> = (0..self.pages.len()).collect();
if page_hint < order.len() {
@@ -424,7 +472,7 @@ impl DocLayout {
order.insert(0, page_hint);
}
for pi in order {
let mut found = self.pieces_on(pi, story, path);
let mut found = on(pi);
if found.is_empty() {
continue;
}
@@ -439,7 +487,7 @@ impl DocLayout {
let mut k = pi;
while lo > 0 && k > 0 {
k -= 1;
let more = self.pieces_on(k, story, path);
let more = on(k);
if more.is_empty() {
break;
}
@@ -449,7 +497,7 @@ impl DocLayout {
let mut k = pi;
while hi < n && k + 1 < self.pages.len() {
k += 1;
let more = self.pieces_on(k, story, path);
let more = on(k);
if more.is_empty() {
break;
}
@@ -461,6 +509,17 @@ impl DocLayout {
Vec::new()
}
/// [`Self::pieces`] for each of many paragraphs of `pages.story`: a part's pieces on a page
/// are looked up in that page's index, made the first time it is looked at, so a selection
/// over many header, footer or text box paragraphs walks each page's items once rather than
/// once for every paragraph.
fn pieces_in<'a>(&'a self, pages: &mut PartPages<'a>, path: &Path, page_hint: usize) -> Vec<(usize, &'a Placed)> {
if pages.story == StoryRef::Body {
return self.pieces(StoryRef::Body, path, page_hint);
}
self.part_pieces(page_hint, &mut |pi| pages.on(self, pi, path))
}
/// Pieces of a paragraph of story part `story` on page `pi`.
fn pieces_on(&self, pi: usize, story: StoryRef, path: &Path) -> Vec<(usize, &Placed)> {
let Some(p) = self.pages.get(pi) else { return Vec::new() };
@@ -629,8 +688,9 @@ impl DocLayout {
if a.story != b.story || !left.is_finite() || !right.is_finite() {
return out;
}
let mut pages = PartPages::new(a.story);
for path in doc.paths_between(a, b) {
for (_, it) in self.pieces(a.story, &path, page_hint) {
for (_, it) in self.pieces_in(&mut pages, &path, page_hint) {
let Placed::Lines { para, l0, l1, x, .. } = it else { continue };
let first = if path == a.path { para.line_of(a.off) } else { 0 };
let last = if path == b.path { para.line_of(b.off) } else { usize::MAX };
@@ -656,8 +716,9 @@ impl DocLayout {
pub fn selection_rects(&self, doc: &Document, a: &Pos, b: &Pos, page_hint: usize) -> Vec<(usize, Rect)> {
let (a, b) = if a <= b { (a, b) } else { (b, a) };
let mut out = Vec::new();
let mut pages = PartPages::new(a.story);
for path in doc.paths_between(a, b) {
for (pi, it) in self.pieces(a.story, &path, page_hint) {
for (pi, it) in self.pieces_in(&mut pages, &path, page_hint) {
let Placed::Lines { para, l0, l1, x, y, turn, .. } = it else { continue };
let Some(first) = para.lines.get(*l0) else { continue };
// Rectangles in the lines' own frame, turned onto the page.
@@ -722,8 +783,8 @@ impl DocLayout {
let l = para.lines.get(li)?;
let here = para.x_of(li, pos.off)?;
let mut best: Option<(usize, f32)> = None;
for off in para.line_offsets(li) {
let Some(x) = para.x_of(li, off) else { continue };
for (off, x) in para.line_carets(li) {
let Some(x) = x else { continue };
let ahead = if left { x < here - 0.01 } else { x > here + 0.01 };
let d = (x - here).abs();
if ahead && best.is_none_or(|b| d < b.1) {
crates/layout/src/lib.rs
+100 -58
View File
@@ -29,6 +29,7 @@ mod table;
mod textbox;
mod vertical;
pub mod wordart;
mod work;
use std::collections::{HashMap, HashSet};
use std::hash::{Hash, Hasher};
@@ -701,11 +702,6 @@ enum Pass {
/// Walk state while paginating.
struct Ctx<'a> {
doc: &'a Document,
/// The body's sections ([`Document::sections`]) for one pass, with the block index each
/// starts after: header and footer wrap work asks for them per page, and rebuilding the
/// list walked the whole body every time, so a long document paid a body-length scan per
/// page. The document does not change during a pass.
sections: Vec<(usize, &'a SectionProps)>,
cache: &'a mut LayoutCache,
opts: &'a LayoutOptions,
counters: Counters,
@@ -750,6 +746,8 @@ struct Ctx<'a> {
/// Each body float's latest placement: on which page and with what wrap area, when it is
/// above its anchor paragraph with text before it ([`EarlyFloats`]).
floats_above: HashMap<(usize, usize), Option<(usize, WrapArea)>>,
/// The document's sections with their headers and footers linked ([`HfLinks`]).
hf: Arc<HfLinks<'a>>,
}
/// A header or footer (`story`) laid out for page `number` of a section (`sect`, `mirror`): its
@@ -2313,7 +2311,6 @@ fn layout_pass(
});
let mut ctx = Ctx {
doc,
sections: doc.sections(),
cache,
opts,
hide_deleted: opts.hides_deleted(StoryRef::Body),
@@ -2347,8 +2344,10 @@ fn layout_pass(
footer_wraps: HashMap::new(),
pin: None,
floats_above: HashMap::new(),
hf: Arc::new(HfLinks::new(doc.sections())),
};
let sections = doc.sections();
let hf = Arc::clone(&ctx.hf);
let sections = &hf.sections;
let web = matches!(opts.view, ViewMode::Web | ViewMode::Draft | ViewMode::Outline);
let read = opts.view == ViewMode::Read;
let reflow = web || read;
@@ -2478,9 +2477,9 @@ fn layout_pass(
ctx.fields.section = u32::try_from(si).unwrap_or(u32::MAX).saturating_add(1);
ctx.fields.section_pages = totals.section(si);
// Headers and footers left out are linked to the previous section's, and take its room.
let (heads, feet) = (linked(&sections, si, |s| s.headers), linked(&sections, si, |s| s.footers));
let hb = |i: Option<u32>| defining_width(&sections, si, |s| s.headers, i);
let fb = |i: Option<u32>| defining_width(&sections, si, |s| s.footers, i);
let (heads, feet) = (hf.heads.set(si), hf.feet.set(si));
let hb = |i: Option<u32>| hf.defining_width(false, si, i);
let fb = |i: Option<u32>| hf.defining_width(true, si, i);
let body_top = if reflow || turn.is_turned() {
sect.margin_top
} else {
@@ -2520,7 +2519,7 @@ fn layout_pass(
{
pb.sect = prev;
pb.sect_idx = si - 1;
let (ph, pf) = (linked(&sections, si - 1, |s| s.headers), linked(&sections, si - 1, |s| s.footers));
let (ph, pf) = (hf.heads.set(si - 1), hf.feet.set(si - 1));
let top = body_top_for(&mut ctx, prev, ph.default);
pb.foot_top = foot_top_for(&mut ctx, prev, pf.default);
pb.new_page(block, top);
@@ -2634,7 +2633,7 @@ fn layout_pass(
}
}
if !reflow {
let found = Totals::of(&pages, &sections);
let found = Totals::of(&pages, sections);
if ctx.uses_totals && may_repeat && found != *totals {
return Pass::Again(found);
}
@@ -2642,13 +2641,13 @@ fn layout_pass(
// Page references in headers and footers are filled in after pagination, like their page
// totals: they don't make the body lay out again.
if headers_footers_refer_to_pages(doc) {
let marks = bookmark_pages(doc, &pages, &sections);
let marks = bookmark_pages(doc, &pages, sections);
ctx.fields.bookmarks_key = marks_key(&marks);
ctx.fields.bookmark_pages = Arc::new(marks);
}
headers_footers(&mut ctx, &mut pages, &sections, &found);
headers_footers(&mut ctx, &mut pages, sections, &found);
if doc.settings.mirror_margins {
mirror_even_pages(&mut pages, &sections);
mirror_even_pages(&mut pages, sections);
}
}
// Evict paragraph layouts that weren't used this pass if the cache grew large.
@@ -2931,23 +2930,77 @@ fn section_side(start: SectionStart, restart: Option<u32>, last: u32, pages: usi
}
}
/// The text width of the section that defines header or footer `id`, when section `si` inherits
/// it rather than naming it (`None` when it names it). Measured in Word: an inherited part's
/// percentage-width tables are sized by that section, though it's laid out on `si`'s pages.
fn defining_width(sections: &[(usize, &SectionProps)], si: usize, get: fn(&SectionProps) -> HeaderSet, id: Option<u32>) -> Option<f32> {
let id = id?;
let names = |s: &SectionProps| {
let h = get(s);
[h.default, h.first, h.even].contains(&Some(id))
};
let own = sections.get(si)?.1;
if names(own) {
return None;
}
sections.iter().take(si).rev().find(|(_, s)| names(s)).map(|(_, s)| s.text_width()).filter(|w| (w - own.text_width()).abs() > 0.01)
/// The document's sections with their headers and footers (Word's "link to previous"), each
/// section's resolved once for the whole layout in document order rather than by looking back
/// over the sections before it whenever one is needed.
struct HfLinks<'a> {
sections: Vec<(usize, &'a SectionProps)>,
heads: Linked,
feet: Linked,
}
/// Lay out with percentage-width tables sized by `base` (see [`defining_width`]).
impl<'a> HfLinks<'a> {
fn new(sections: Vec<(usize, &'a SectionProps)>) -> Self {
let heads = Linked::new(&sections, |s| s.headers);
let feet = Linked::new(&sections, |s| s.footers);
HfLinks { sections, heads, feet }
}
/// The text width of the section that defines header (or, `footer`, footer) `id`, when
/// section `si` inherits it rather than naming it (`None` when it names it). Measured in
/// Word: an inherited part's percentage-width tables are sized by that section, though it's
/// laid out on `si`'s pages.
fn defining_width(&self, footer: bool, si: usize, id: Option<u32>) -> Option<f32> {
let id = id?;
work::steps(1);
let linked = if footer { &self.feet } else { &self.heads };
let own = self.sections.get(si)?.1;
if linked.own.get(si).is_some_and(|h| [h.default, h.first, h.even].contains(&Some(id))) {
return None;
}
// The last section before `si` naming it.
let namers = linked.namers.get(&id)?;
let k = *namers.get(namers.partition_point(|&k| k < si).checked_sub(1)?)?;
self.sections.get(k).map(|(_, s)| s.text_width()).filter(|w| (w - own.text_width()).abs() > 0.01)
}
}
/// The headers (or footers) of every section: those it names, and with those it leaves out
/// inherited from the sections before it.
struct Linked {
own: Vec<HeaderSet>,
linked: Vec<HeaderSet>,
/// The sections naming each part, in order.
namers: HashMap<u32, Vec<usize>>,
}
impl Linked {
fn new(sections: &[(usize, &SectionProps)], get: fn(&SectionProps) -> HeaderSet) -> Self {
let own: Vec<HeaderSet> = sections.iter().map(|(_, s)| get(s)).collect();
let mut linked = Vec::with_capacity(own.len());
let mut namers: HashMap<u32, Vec<usize>> = HashMap::new();
let mut prev = HeaderSet::default();
for (si, o) in own.iter().enumerate() {
work::steps(1);
prev = HeaderSet { default: o.default.or(prev.default), first: o.first.or(prev.first), even: o.even.or(prev.even) };
linked.push(prev);
for id in [o.default, o.first, o.even].into_iter().flatten() {
let v = namers.entry(id).or_default();
if v.last() != Some(&si) {
v.push(si);
}
}
}
Linked { own, linked, namers }
}
/// Section `si`'s headers (or footers), those it leaves out inherited.
fn set(&self, si: usize) -> HeaderSet {
self.linked.get(si).copied().unwrap_or_default()
}
}
/// Lay out with percentage-width tables sized by `base` (see [`HfLinks::defining_width`]).
fn with_pct_base<T>(ctx: &mut Ctx, base: Option<f32>, f: impl FnOnce(&mut Ctx) -> T) -> T {
let before = std::mem::replace(&mut ctx.pct_base, base);
let out = f(ctx);
@@ -3407,11 +3460,12 @@ fn footer_table_wraps(ctx: &mut Ctx, pb: &PageBuilder) -> Vec<WrapArea> {
}
ctx.footer_wraps.insert(key, Vec::new());
let doc = ctx.doc;
let Some((_, sect)) = ctx.sections.get(page.section).copied() else { return Vec::new() };
let hf = Arc::clone(&ctx.hf);
let Some((_, sect)) = hf.sections.get(page.section) else { return Vec::new() };
if sect.text_direction.is_turned() {
return Vec::new();
}
let set = linked(&ctx.sections, page.section, |s| s.footers);
let set = hf.feet.set(page.section);
let id = if sect.title_page && first {
set.first
} else if doc.settings.even_odd_headers && page.number.is_multiple_of(2) {
@@ -3423,7 +3477,7 @@ fn footer_table_wraps(ctx: &mut Ctx, pb: &PageBuilder) -> Vec<WrapArea> {
let Some(part) = doc.parts.get(&id) else { return Vec::new() };
let blocks = part.blocks.clone();
let x = sect.margin_left + sect.gutter;
let base = defining_width(&ctx.sections, page.section, |s| s.footers, Some(id));
let base = hf.defining_width(true, page.section, Some(id));
let counters = std::mem::take(&mut ctx.counters);
let boxes = std::mem::take(&mut ctx.boxes);
let chains = std::mem::take(&mut ctx.chains);
@@ -4541,11 +4595,12 @@ fn header_table_areas(ctx: &mut Ctx, pb: &PageBuilder) -> Vec<Rect> {
if let Some(areas) = ctx.header_tables.get(&key) {
return areas.clone();
}
let Some((_, sect)) = ctx.sections.get(page.section).copied() else { return Vec::new() };
let hf = Arc::clone(&ctx.hf);
let Some((_, sect)) = hf.sections.get(page.section) else { return Vec::new() };
if sect.text_direction.is_turned() {
return Vec::new();
}
let set = linked(&ctx.sections, page.section, |s| s.headers);
let set = hf.heads.set(page.section);
let id = if sect.title_page && first {
set.first
} else if ctx.doc.settings.even_odd_headers && page.number.is_multiple_of(2) {
@@ -4558,7 +4613,7 @@ fn header_table_areas(ctx: &mut Ctx, pb: &PageBuilder) -> Vec<Rect> {
let blocks = part.blocks.clone();
let x = sect.margin_left + sect.gutter;
let frame = PageFrame { sect, origin: (x, sect.header), number: page.number, mirror: ctx.mirror };
let base = defining_width(&ctx.sections, page.section, |s| s.headers, Some(id));
let base = hf.defining_width(false, page.section, Some(id));
let counters = std::mem::take(&mut ctx.counters);
let boxes = std::mem::take(&mut ctx.boxes);
let chains = std::mem::take(&mut ctx.chains);
@@ -4711,22 +4766,6 @@ fn below(excl: &[WrapArea], mut y: f32, x0: f32, x1: f32) -> f32 {
y
}
/// Section `si`'s headers (or footers, as `get` picks), with those it leaves out inherited from
/// the sections before it (Word's "link to previous").
fn linked(sections: &[(usize, &SectionProps)], si: usize, get: fn(&SectionProps) -> HeaderSet) -> HeaderSet {
let mut set = sections.get(si).map(|(_, s)| get(s)).unwrap_or_default();
for (_, s) in sections.iter().take(si).rev() {
if set.default.is_some() && set.first.is_some() && set.even.is_some() {
break;
}
let o = get(s);
set.default = set.default.or(o.default);
set.first = set.first.or(o.first);
set.even = set.even.or(o.even);
}
set
}
/// Whether a header, a footer or a text box shows a `PAGEREF` field, which they resolve from the
/// paginated body (see [`bookmark_pages`]).
fn headers_footers_refer_to_pages(doc: &Document) -> bool {
@@ -4805,6 +4844,7 @@ fn marks_key(marks: &fields::BookmarkPages) -> u64 {
fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &SectionProps)], totals: &Totals) {
let total = pages.len() as u32;
let hf = Arc::clone(&ctx.hf);
let even_odd = ctx.doc.settings.even_odd_headers;
let styles = styleref::headers_use_styleref(ctx.doc).then(|| ctx.cache.style_index(ctx.doc));
let mut first_of_section: HashSet<usize> = HashSet::new();
@@ -4817,8 +4857,8 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti
for (i, page) in pages.iter_mut().enumerate() {
let Some((_, sect)) = sections.get(page.section) else { continue };
// Inherit header/footer references from earlier sections (Word's "link to previous").
let pick = |get: fn(&SectionProps) -> HeaderSet| -> Option<u32> {
let set = linked(sections, page.section, get);
let pick = |linked: &Linked| -> Option<u32> {
let set = linked.set(page.section);
if sect.title_page && first_of_section.contains(&i) {
return set.first;
}
@@ -4827,8 +4867,8 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti
}
set.default
};
let hid = pick(|s| s.headers);
let fid = pick(|s| s.footers);
let hid = pick(&hf.heads);
let fid = pick(&hf.feet);
page.hf_slot = if sect.title_page && first_of_section.contains(&i) {
HfSlot::First
} else if even_odd && page.number % 2 == 0 {
@@ -4856,7 +4896,7 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti
{
let blocks = part.blocks.clone();
let frame = PageFrame { sect, origin: (x, sect.header), number: page.number, mirror: ctx.mirror };
let base = defining_width(sections, page.section, |s| s.headers, hid);
let base = hf.defining_width(false, page.section, hid);
let (mut items, behind, front) = with_pin(ctx, id, sect, page.number, |ctx| {
with_pct_base(ctx, base, |c| layout_box(c, StoryRef::Part(id), &blocks, &[], w, BoxContext::Other, 0, Some(frame)).items)
});
@@ -4870,7 +4910,7 @@ fn headers_footers(ctx: &mut Ctx, pages: &mut [Page], sections: &[(usize, &Secti
&& let Some(part) = ctx.doc.parts.get(&id)
{
let blocks = part.blocks.clone();
let base = defining_width(sections, page.section, |s| s.footers, fid);
let base = hf.defining_width(true, page.section, fid);
// The footer's height fixes where it sits, which page-relative floats need.
let (h, ..) = with_pin(ctx, id, sect, page.number, |ctx| {
with_pct_base(ctx, base, |c| layout_box(c, StoryRef::Part(id), &blocks, &[], w, BoxContext::Other, 0, None).height)
@@ -4940,6 +4980,8 @@ mod tests_grid;
#[cfg(test)]
mod tests_ligature;
#[cfg(test)]
mod tests_linear;
#[cfg(test)]
mod tests_nbsp;
#[cfg(test)]
mod tests_numeral;
crates/layout/src/math.rs
+20 -9
View File
@@ -631,19 +631,19 @@ impl Ctx {
last_gid = Some(g.gid);
}
b.w = x;
// Character boundaries: where the first glyph of each character starts.
// Character boundaries: where the first glyph of each character starts. The glyph for a
// boundary is never before the previous boundary's, so one walk of the glyphs finds all.
let mut cuts = Vec::with_capacity(text.len() + 1);
let (mut j, mut gx) = (0usize, 0.0f32);
for (cb, _) in text.char_indices() {
let mut gx = 0.0f32;
let mut found = None;
for g in &shaped {
if g.cluster >= cb {
found = Some(gx);
break;
}
while let Some(g) = shaped.get(j)
&& g.cluster < cb
{
crate::work::steps(1);
gx += g.x_advance as f32 * k;
j += 1;
}
cuts.push(found.unwrap_or(x));
cuts.push(if j < shaped.len() { gx } else { x });
}
cuts.push(x);
b.cuts = cuts;
@@ -2389,4 +2389,15 @@ mod tests {
}
}
}
/// Finding 23: each character's boundary in a run of text was found by walking the run's
/// glyphs from the first.
#[test]
fn a_long_text_run_lays_out_in_linear_time() {
use crate::tests_linear::assert_linear_steps;
use wordcraft_doc::math::*;
let eq = |n: usize| Math { nodes: vec![MNode::Run(MRun { nor: true, ..MRun::new("a".repeat(n)) })], ..Default::default() };
let lay = |m: &Math| drop(std::hint::black_box(lay_in(m, "", false, 1e6, &MathProps::default())));
assert_linear_steps("math text run", 400, eq, lay);
}
}
crates/layout/src/para.rs
+48 -11
View File
@@ -3107,20 +3107,43 @@ impl ParaLayout {
let k = self.clusters.partition_point(|c| c.start < off);
self.clusters.get(..k).into_iter().flatten().rev().find(|c| self.is_edit_boundary(c.start)).map_or(0, |c| c.start)
};
let mut from = l.c0;
self.x_from(l, off, &mut from)
}
/// [`Self::x_of`] on line `l`, looking for the cluster at `off` from cluster `*from` on (the
/// clusters before it must all end at or before `off`: so they do when `from` was left there
/// by an offset no greater), and leaving `from` at the cluster found. Walks a line's carets
/// in order in one pass over its clusters.
fn x_from(&self, l: &Line, off: usize, from: &mut usize) -> Option<f32> {
if !l.vis.is_empty() {
return self.x_of_bidi(l, off);
return self.x_of_bidi(l, off, from);
}
for k in l.c0..l.c1 {
for k in *from..l.c1 {
crate::work::steps(1);
let c = self.clusters.get(k)?;
if off <= c.start {
return l.xs.get(k - l.c0).copied();
}
if off < c.end {
if off <= c.start || off < c.end {
*from = k;
return l.xs.get(k - l.c0).copied();
}
}
*from = l.c1;
l.xs.last().copied()
}
/// Each caret offset on line `li` ([`Self::line_offsets`]) with its x ([`Self::x_of`]).
pub fn line_carets(&self, li: usize) -> Vec<(usize, Option<f32>)> {
let Some(l) = self.lines.get(li) else { return Vec::new() };
let (mut from, mut last) = (l.c0, 0);
self.line_offsets(li)
.into_iter()
.map(|off| {
if off < last {
from = l.c0;
}
last = off;
(off, self.x_from(l, off, &mut from))
})
.collect()
}
/// The farthest right a caret goes on left-to-right line `li`: spaces at the end of a line
/// hang past its right edge, but the caret after them stays at the edge, as in Word (#2146);
/// text that overflows the line still carries it. `None` on bidirectional lines.
@@ -3155,8 +3178,12 @@ impl ParaLayout {
/// edge of right-to-left text), except where `off` leaves an embedded run (the cluster before
/// it has the higher level), which keeps the caret at that run's trailing edge, so each
/// visual boundary of the common mixed cases has its own offset.
fn x_of_bidi(&self, l: &Line, off: usize) -> Option<f32> {
let k = (l.c0..l.c1).find(|&k| self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start));
fn x_of_bidi(&self, l: &Line, off: usize, from: &mut usize) -> Option<f32> {
let k = (*from..l.c1).find(|&k| {
crate::work::steps(1);
self.clusters.get(k).is_some_and(|c| off < c.end || off <= c.start)
});
*from = k.unwrap_or(l.c1);
let Some(k) = k else { return Some(l.end_x()) };
let c = self.clusters.get(k)?;
let here = l.vis.get(k - l.c0)?;
@@ -3204,8 +3231,8 @@ impl ParaLayout {
let Some(l) = self.lines.get(li) else { return self.text_len };
if !l.vis.is_empty() {
let mut best = (l.start, f32::MAX);
for off in self.line_offsets(li) {
let d = self.x_of(li, off).map_or(f32::MAX, |cx| (cx - x).abs());
for (off, cx) in self.line_carets(li) {
let d = cx.map_or(f32::MAX, |cx| (cx - x).abs());
if d < best.1 {
best = (off, d);
}
@@ -3319,8 +3346,18 @@ fn proof_issues(
/// U+0001 bytes (neither words nor spaces).
pub fn proof_text(p: &wordcraft_doc::Paragraph) -> String {
let mut out = String::with_capacity(p.text.len());
// The run holding each character ([`Paragraph::props_of_char`]), found by walking the runs
// along with the text rather than from the first run for every character.
let mut runs = p.runs.iter();
let mut run = runs.next();
let mut end = run.map_or(0, |r| r.len);
for (i, c) in p.text.char_indices() {
let deleted = c == wordcraft_doc::para::OBJ || p.props_of_char(i).del.is_some();
while run.is_some() && i >= end {
run = runs.next();
end = end.saturating_add(run.map_or(0, |r| r.len));
}
let props = run.map_or(&p.mark, |r| &r.props);
let deleted = c == wordcraft_doc::para::OBJ || props.del.is_some();
if deleted {
for _ in 0..c.len_utf8() {
out.push('\u{1}');
crates/layout/src/table.rs
+13 -4
View File
@@ -71,6 +71,13 @@ struct Turned {
top: f32,
}
/// The number of grid columns of `t` ([`Table::cols`]), which walks every row (each row counted
/// as a step of [`crate::work`]).
fn grid_cols(t: &Table) -> usize {
crate::work::steps(t.rows.len());
t.cols()
}
/// How long the lines laid out in `items` are without the room they didn't use: the length
/// turned text needs so that nothing wraps.
fn natural_length(items: &[Placed]) -> f32 {
@@ -178,7 +185,7 @@ pub fn measure_table_columns(doc: &wordcraft_doc::Document, t: &Table) -> Vec<(f
use wordcraft_doc::Block;
let style = t.props.style.as_deref().and_then(|id| doc.styles.table_style(id));
let margins_def = default_margins(t, style.as_ref());
let ncols = t.cols().clamp(1, wordcraft_doc::table::MAX_COLS);
let ncols = grid_cols(t).clamp(1, wordcraft_doc::table::MAX_COLS);
let fields = crate::FieldCtx::default();
let mut cols = vec![(0.0f32, 0.0f32); ncols];
// Cells spanning several columns: (first column, span, min, max), applied after the others.
@@ -360,7 +367,7 @@ pub fn layout_table(ctx: &mut Ctx, story: StoryRef, t: &Table, path: &[u32], ava
// Cell text formatting and looks per set of style parts, built once per table.
let mut cell_text: Vec<(Region, CellText)> = Vec::new();
let mut looks: Vec<(Region, RegionLook)> = Vec::new();
let ncols = t.cols().max(1);
let ncols = grid_cols(t).max(1);
// Column widths.
let mut grid: Vec<f32> = if t.grid.len() == ncols {
// Grid units can be narrower than a text box, including omitted row-end columns.
@@ -1122,7 +1129,7 @@ fn region_text(st: &TableStyleProps, region: Region) -> CellText {
/// with their own. `None` when every cell is empty.
fn column_content(ctx: &mut Ctx, t: &Table, style: Option<&TableStyleProps>, depth: usize) -> Option<Vec<(f32, f32)>> {
let margins_def = default_margins(t, style);
let mut cols = vec![(0.0f32, 0.0f32); t.cols().max(1)];
let mut cols = vec![(0.0f32, 0.0f32); grid_cols(t).max(1)];
let mut spanning = Vec::new();
let mut any = false;
for (ri, row) in t.rows.iter().enumerate() {
@@ -1144,8 +1151,10 @@ fn column_content(ctx: &mut Ctx, t: &Table, style: Option<&TableStyleProps>, dep
}
spanning.extend(row_gaps(row, g).map(|(g, span, w)| (g, span, (w, w))));
}
// The grid's width once: counting it walks every row, and there can be a spanning cell in each.
let ncols = grid_cols(t);
for (g, span, need) in spanning {
let Some(cs) = cols.get_mut(g..(g + span).min(t.cols())) else { continue };
let Some(cs) = cols.get_mut(g..g.saturating_add(span).min(ncols)) else { continue };
let n = cs.len().max(1) as f32;
let (have_lo, have_hi) = cs.iter().fold((0.0, 0.0), |(l, h), c| (l + c.0, h + c.1));
for c in cs {
crates/layout/src/tests.rs
+3 -3
View File
@@ -756,7 +756,7 @@ fn hidden_text_adds_no_line_break_opportunity_when_not_shown() {
/// How much bigger the second document is in the linear-time tests. At 8x a linear layout costs
/// about 8x as much and a quadratic one about 64x, so a limit of 24x stays far from both.
const SCALE: usize = 8;
pub(crate) const SCALE: usize = 8;
/// The CPU time `f` takes on this thread, in seconds. On Linux it is the scheduler's count of the
/// time the thread actually ran, so the time it waits while other work on a busy machine has the
@@ -779,7 +779,7 @@ fn cpu_cost(f: &mut impl FnMut()) -> f64 {
/// cheapest of several runs after a warm-up, the runs alternating so a busy spell slows both sizes
/// alike; `small` is timed `scale` times over per run, so both timings are long against the clock's
/// resolution.
fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) {
pub(crate) fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (f64, f64, f64) {
small();
big();
let mut small_scaled = || (0..scale).for_each(|_| small());
@@ -792,7 +792,7 @@ fn cost_ratio(scale: usize, mut small: impl FnMut(), mut big: impl FnMut()) -> (
}
/// [`cost_ratio`] of laying out `doc(SCALE * n)` against `doc(n)`.
fn layout_time_ratio(doc: impl Fn(usize) -> Document, n: usize, opts: &LayoutOptions) -> (f64, f64, f64) {
pub(crate) fn layout_time_ratio(doc: impl Fn(usize) -> Document, n: usize, opts: &LayoutOptions) -> (f64, f64, f64) {
let (small, big) = (doc(n), doc(SCALE * n));
cost_ratio(
SCALE,
crates/layout/src/tests_linear.rs
+296
View File
@@ -0,0 +1,296 @@
//! Lookups that once rescanned the page, the paragraph or the document for every item they
//! looked up (#3692, #3706): an input 8x the size must cost about 8x as much, not 64x.
//!
//! The cost is the steps the lookups' walks count ([`crate::work`]), not a timing, so a busy
//! machine can't make a linear lookup look quadratic or the other way round. Only proofing text,
//! whose walk is the document model's, is timed.
use super::tests::{SCALE, cost_ratio};
use super::*;
use wordcraft_doc::para::Run;
use wordcraft_doc::{Pos, Table};
fn lay(d: &Document, opts: &LayoutOptions) -> DocLayout {
layout(d, &mut LayoutCache::new(), opts)
}
/// The most steps 8x the input may take, against the steps of the input: linear walks take
/// about 8x (a little more where a fixed cost is shared out), quadratic ones about 64x.
const MAX_STEP_RATIO: f64 = 1.5 * SCALE as f64;
/// Asserts `f` takes about [`SCALE`] times the steps ([`crate::work`]) on `make(SCALE * n)` as on
/// `make(n)`, not [`SCALE`]² times. Making the inputs isn't counted.
pub(crate) fn assert_linear_steps<T>(what: &str, n: usize, make: impl Fn(usize) -> T, f: impl Fn(&T)) {
let (small, big) = (make(n), make(SCALE * n));
let (s, b) = (work::count(|| f(&small)), work::count(|| f(&big)));
let ratio = b as f64 / s.max(1) as f64;
eprintln!("{what}: {s} steps for n, {b} for {SCALE}n, ratio {ratio:.1}");
assert!(s > 0, "{what}: no steps counted, so the walk this test watches no longer counts them (crate::work)");
assert!(ratio <= MAX_STEP_RATIO, "{what}: {SCALE}x the input took {ratio:.1}x the steps ({s} → {b}): not linear");
}
/// [`assert_linear_steps`] for laying out `doc(n)`.
fn assert_layout_linear(what: &str, n: usize, doc: impl Fn(usize) -> Document, opts: &LayoutOptions) {
assert_linear_steps(what, n, doc, |d| drop(std::hint::black_box(lay(d, opts))));
}
/// The first laid-out body paragraph on page 0.
fn first_para(l: &DocLayout) -> Option<&ParaLayout> {
l.pages.first()?.items.iter().find_map(|it| match it {
Placed::Lines { para, story: StoryRef::Body, .. } => Some(para.as_ref()),
_ => None,
})
}
/// Finding 29: proofing text found each character's run from the paragraph's first run.
#[test]
fn proof_text_walks_the_runs_once() {
// One run per character, every other one deleted.
let para = |n: usize| {
let mut p = Paragraph::with_text(&"a".repeat(n), CharProps::default());
p.runs = (0..n).map(|i| Run { len: 1, props: CharProps { del: (i % 2 == 1).then_some(0), ..Default::default() } }).collect();
p
};
assert_eq!(para::proof_text(&para(5)), "a\u{1}a\u{1}a");
// The runs are the document model's, whose walk counts no steps, so this one is timed: the
// CPU time this thread ran (`cost_ratio`). At 50,000 runs the old walk took 64x as long for
// 8x the runs, and the new one about 8x; a limit of 24x stays far from both, and a ratio over
// it is measured again, twice, before the test fails, so a busy spell can't fail it alone.
let (small, big) = (para(50_000), para(SCALE * 50_000));
let proof_8 = |p: &Paragraph| (0..8).for_each(|_| drop(std::hint::black_box(para::proof_text(p))));
let mut ratios = Vec::new();
for _ in 0..3 {
let (t1, tk, ratio) = cost_ratio(SCALE, || proof_8(&small), || proof_8(&big));
eprintln!("proof_text: t(n) {t1:.4} s, t(8n) {tk:.4} s, ratio {ratio:.1}");
if ratio < 24.0 {
return;
}
ratios.push(ratio);
}
panic!("proof_text: 8x the runs took {ratios:.1?}x the time, three times over: not linear");
}
/// A right-to-left paragraph whose one line holds `n` hidden letters: caret offsets with no width.
fn bidi_hidden(n: usize) -> Document {
let head = "سلام ";
let mut d = Document::from_text(&format!("{head}{} world", "x".repeat(n)));
let at = Pos::body(0, 0);
d.format_paragraphs(&at, &at, &|p| p.bidi = Some(true)).unwrap();
d.format_range(&Pos::body(0, head.len()), &Pos::body(0, head.len() + n), &|c| c.hidden = Some(true)).unwrap();
d
}
/// Finding 5: a click on a bidirectional line and a visual caret step measured every caret
/// offset from the line's first cluster.
#[test]
fn bidi_clicks_and_caret_steps_take_linear_time() {
let opts = LayoutOptions { show_hidden: false, ..Default::default() };
let laid = |n: usize| lay(&bidi_hidden(n), &opts);
let Some(pb) = first_para(&laid(SCALE * 5000)).map(|p| p.lines.len()) else { panic!("no paragraph") };
assert_eq!(pb, 1, "the hidden letters stay on one line");
assert_linear_steps("bidi hit test", 5000, laid, |l| {
let Some(p) = first_para(l) else { panic!("no paragraph") };
std::hint::black_box(p.off_at_x(0, 1.0));
std::hint::black_box(l.visual_step(&Pos::body(0, 0), true, 0));
});
}
/// The carets of a line walked in one pass are where `x_of` puts each of them, on bidirectional
/// and left-to-right lines alike.
#[test]
fn line_carets_match_x_of() {
let docs = [bidi_hidden(20), Document::from_text("Plain left-to-right text, long enough to wrap. ".repeat(6).as_str()), {
let mut d = Document::from_text("سلام world 123 دنیا (مرحبا) end");
let at = Pos::body(0, 0);
d.format_paragraphs(&at, &at, &|p| p.bidi = Some(true)).unwrap();
d
}];
for d in &docs {
for show_hidden in [false, true] {
let l = lay(d, &LayoutOptions { show_hidden, ..Default::default() });
let Some(p) = first_para(&l) else { panic!("no paragraph") };
for li in 0..p.lines.len() {
let want: Vec<_> = p.line_offsets(li).into_iter().map(|off| (off, p.x_of(li, off))).collect();
assert!(!want.is_empty());
assert_eq!(p.line_carets(li), want, "line {li}");
}
}
}
}
/// Finding 3: autofitting a table counted its grid's columns again for every spanning cell.
#[test]
fn autofit_tables_with_a_spanning_cell_in_every_row_lay_out_in_linear_time() {
// Rows with no cells, only a column skipped before and after them with a width: two
// spanning entries a row for next to no layout, so counting the grid per entry dominates.
let doc = |n: usize| {
let mut t = Table::new(n, 1, 468.0);
for row in t.rows.iter_mut().skip(1) {
row.cells.clear();
row.props.grid_before = 1;
row.props.w_before = Some(10.0);
row.props.grid_after = 1;
row.props.w_after = Some(10.0);
}
let mut d = Document::from_text("x");
d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap();
d
};
assert_layout_linear("spanning cells", 2000, doc, &LayoutOptions::default());
}
/// `n` continuous sections of an empty paragraph each, each with a different first page. The
/// first section names every header and footer; the others name them too, or (`inherit`) name
/// none and link to the previous section's.
fn sections(n: usize, inherit: bool) -> Document {
use wordcraft_doc::PartKind::{Footer, Header};
let mut d = Document::from_text(&"\n".repeat(n.saturating_sub(1)));
let mut part = |k| d.add_part(k, vec![wordcraft_doc::para_block(Paragraph::with_text("Head", CharProps::default()))]);
let ids = [part(Header), part(Header), part(Header), part(Footer), part(Footer), part(Footer)];
let named = SectionProps {
start: SectionStart::Continuous,
title_page: true,
headers: HeaderSet { default: Some(ids[0]), first: Some(ids[1]), even: Some(ids[2]) },
footers: HeaderSet { default: Some(ids[3]), first: Some(ids[4]), even: Some(ids[5]) },
..Default::default()
};
let linked = SectionProps { headers: HeaderSet::default(), footers: HeaderSet::default(), ..named.clone() };
for i in 0..n.saturating_sub(1) {
let s = if i == 0 || !inherit { named.clone() } else { linked.clone() };
d.para_mut(StoryRef::Body, &Path::top(i)).unwrap().section = Some(Box::new(s));
}
d.last_section = if inherit { linked } else { named };
d
}
/// Finding 13: each section's headers and footers were found by walking back through the
/// sections before it, for every section and again for every page. Linked to the previous
/// section's, they must cost no more to lay out than named in each section.
#[test]
fn headers_inherited_through_many_sections_lay_out_in_linear_time() {
let l = lay(&sections(200, true), &LayoutOptions::default());
assert!(l.pages.iter().all(|p| !p.header.is_empty() && !p.footer.is_empty()), "the header and footer are inherited");
let opts = LayoutOptions::default();
assert_layout_linear("inherited headers", 1000, |n| sections(n, true), &opts);
let steps = |inherit: bool| work::count(|| drop(std::hint::black_box(lay(&sections(1000, inherit), &opts))));
let (named, inherited) = (steps(false), steps(true));
assert!(inherited <= 2 * named, "1000 inheriting sections took {inherited} steps, named ones {named}");
}
/// A header of `n` short paragraphs.
fn tall_header(n: usize) -> (Document, u32) {
let mut d = Document::from_text("Body");
let paras = (0..n).map(|i| wordcraft_doc::para_block(Paragraph::with_text(&format!("h{i}"), CharProps::default()))).collect();
let id = d.add_part(wordcraft_doc::PartKind::Header, paras);
d.last_section.headers.default = Some(id);
(d, id)
}
/// Finding 28: selecting across a header looked through the whole page for each paragraph.
#[test]
fn selecting_a_long_header_takes_linear_time() {
// The header laid out, and the selection from its first paragraph to its last.
let laid = |n: usize| {
let (d, id) = tall_header(n);
let l = lay(&d, &LayoutOptions::default());
let ends = (Pos::new(StoryRef::Part(id), Path::top(0), 0), Pos::new(StoryRef::Part(id), Path::top(n - 1), 2));
(d, l, ends)
};
let (d, l, (a, b)) = laid(SCALE * 300);
assert!(l.selection_rects(&d, &a, &b, 0).len() >= SCALE * 300, "a rectangle per paragraph");
assert_linear_steps("header selection", 300, laid, |(d, l, (a, b))| drop(std::hint::black_box(l.selection_rects(d, a, b, 0))));
}
/// A table of `n` rows of 60 cells, each row exactly half a point high, so every cell clips its
/// (empty) paragraph and the whole table fits on the first page.
fn clipped_table(n: usize) -> Document {
let mut t = Table::new(n, 60, 468.0);
for row in &mut t.rows {
row.props.height = Some(0.5);
row.props.height_rule = HeightRule::Exact;
}
let mut d = Document::from_text("x");
d.insert_block(StoryRef::Body, &Path::top(0), Block::Table(t)).unwrap();
d
}
/// Finding 1: drawing a line in a clipped cell looked through every clipped cell on the page.
#[test]
fn drawing_many_clipped_cells_takes_linear_time() {
let laid = |n: usize| {
let d = clipped_table(n);
let l = lay(&d, &LayoutOptions::default());
(d, l)
};
let opts = display::DisplayOptions::default();
let draw = |(d, l): &(Document, DocLayout)| l.pages.first().map(|p| display::page_display(d, p, &opts)).unwrap_or_default();
let clips = draw(&laid(SCALE * 16)).iter().filter(|i| matches!(i, display::Draw::Clip { .. })).count();
assert_eq!(clips, SCALE * 16 * 60, "every cell's line is clipped, all on the first page");
assert_linear_steps("clipped cells", 16, laid, |dl| drop(std::hint::black_box(draw(dl))));
}
/// `n` words, each inserted by its own author, at a size that fits them all on the first page.
fn tracked_words(n: usize) -> Document {
let mut d = Document::from_text("x");
let p = Paragraph {
text: "word ".repeat(n),
runs: (0..n).map(|i| Run { len: 5, props: CharProps { ins: u32::try_from(i).ok(), size: Some(1.0), ..Default::default() } }).collect(),
..Paragraph::with_text("", CharProps::default())
};
d.body = vec![wordcraft_doc::para_block(p)];
d.revisions = (0..n).map(|i| wordcraft_doc::Revision::new(wordcraft_doc::RevisionKind::Insert, format!("Author {i}"), "")).collect();
d
}
/// Finding 30 (#3706): every tracked run ranked its author among all the document's revisions.
#[test]
fn drawing_tracked_changes_by_many_authors_takes_linear_time() {
let laid = |n: usize| {
let d = tracked_words(n);
let l = lay(&d, &LayoutOptions::default());
(d, l)
};
assert_eq!(laid(SCALE * 1000).1.pages.len(), 1);
let opts = display::DisplayOptions { markup: true, ..Default::default() };
let draw = |(d, l): &(Document, DocLayout)| l.pages.first().map(|p| display::page_display(d, p, &opts)).unwrap_or_default();
// Every run is drawn in its author's colour, so the authors are ranked at least once.
assert_linear_steps("tracked runs", 1000, laid, |dl| drop(std::hint::black_box(draw(dl))));
}
/// Each author keeps the colour of their first revision's place among the authors.
#[test]
fn tracked_changes_colour_by_author_in_order_of_appearance() {
let mut d = tracked_words(3);
// Words 0 and 2 by Ana, word 1 by Ben: Ana first, Ben second.
for (r, a) in d.revisions.iter_mut().zip(["Ana", "Ben", "Ana"]) {
r.author = a.into();
}
let l = lay(&d, &LayoutOptions::default());
let opts = display::DisplayOptions { markup: true, ..Default::default() };
let colours: Vec<_> = display::page_display(&d, &l.pages[0], &opts)
.into_iter()
.filter_map(|i| if let display::Draw::Glyphs { color, .. } = i { Some(color) } else { None })
.collect();
let (ana, ben) = (display::revision_color(0), display::revision_color(1));
assert_ne!(ana, ben);
assert_eq!(colours, [ana, ben, ana]);
}
/// Finding 30 (#3706): every comment balloon ranked its author among all the comments.
#[test]
fn balloons_for_many_authors_lay_out_in_linear_time() {
let doc = |n: usize| {
let mut d = Document::from_text(&"word ".repeat(n));
for i in (0..n).rev() {
let part = d.add_part(wordcraft_doc::PartKind::Comment, vec![wordcraft_doc::para_block(Paragraph::with_text("c", CharProps::default()))]);
let id = u32::try_from(i).unwrap();
d.comments.insert(id, wordcraft_doc::Comment { author: format!("Author {i}"), part, ..Default::default() });
let p = CharProps::default();
d.insert_object(&Pos::body(0, 5 * i + 4), InlineObject::CommentEnd { id }, &p).unwrap();
d.insert_object(&Pos::body(0, 5 * i), InlineObject::CommentStart { id }, &p).unwrap();
}
d
};
let opts = LayoutOptions { comment_balloons: true, ..Default::default() };
assert_layout_linear("comment balloons", 120, doc, &opts);
}
crates/layout/src/work.rs
+29
View File
@@ -0,0 +1,29 @@
//! A count of the steps the layout's lookups take, for the linear-time tests (#3692): they check
//! that an input 8x the size costs about 8x the steps, which a busy machine can't skew the way it
//! skews a timing. Each walk that once ran again for every item it served (a line's clusters, a
//! page's items, the revision authors, the sections, a table's rows) counts the elements it
//! looks at, so a caller that walks it once per item again shows up as a quadratic count. Outside
//! tests the count is nothing.
#[cfg(test)]
thread_local! {
/// The steps counted on this thread (layout runs on the thread that calls it).
static STEPS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
}
/// Count `n` steps of a walk (nothing outside tests).
#[inline(always)]
pub(crate) fn steps(n: usize) {
#[cfg(test)]
STEPS.with(|s| s.set(s.get().saturating_add(n as u64)));
#[cfg(not(test))]
let _ = n;
}
/// The steps `f` takes.
#[cfg(test)]
pub(crate) fn count(f: impl FnOnce()) -> u64 {
let before = STEPS.with(std::cell::Cell::get);
f();
STEPS.with(std::cell::Cell::get).saturating_sub(before)
}
crates/metafile/src/canvas.rs
+6 -3
View File
@@ -284,9 +284,12 @@ impl Canvas {
fn push(&mut self, raw: Raw) {
let cost = match &raw {
Raw::Path { ops, .. } => ops.len() * std::mem::size_of::<Op>(),
Raw::Bitmap { img, .. } => img.px.len(),
Raw::Text(t) => t.text.len() + t.font.len() + t.dx.len() * std::mem::size_of::<f64>(),
// What is held, capacity and all.
Raw::Path { ops, .. } => ops.capacity().saturating_mul(std::mem::size_of::<Op>()),
Raw::Bitmap { img, .. } => img.px.capacity(),
Raw::Text(t) => {
t.text.capacity().saturating_add(t.font.capacity()).saturating_add(t.dx.capacity().saturating_mul(std::mem::size_of::<f64>()))
}
};
if self.items.len() >= MAX_ITEMS || self.used.saturating_add(cost) > MAX_BYTES {
self.full = true;
crates/metafile/src/geom.rs
+11 -1
View File
@@ -99,7 +99,17 @@ fn arc(cx: f64, cy: f64, rx: f64, ry: f64, a0: f64, sweep: f64) -> Vec<Op> {
/// Consecutive subpaths: `counts[i]` points each, taken in order from `pts`. Counts that run past the
/// point list are cut short rather than failing the whole record.
pub(crate) fn subpaths(pts: &[P], counts: &[usize], close: bool) -> Vec<Op> {
let mut ops = Vec::with_capacity(pts.len() + counts.len());
// The exact size first: empty groups add no ops, so a long count list of zeros (file data)
// mustn't reserve room the canvas budget never sees.
let (mut n, mut at) = (0usize, 0usize);
for &c in counts {
let end = at.saturating_add(c).min(pts.len());
if end > at {
n = n.saturating_add(end - at + usize::from(close));
}
at = end;
}
let mut ops = Vec::with_capacity(n);
let mut at = 0usize;
for &c in counts {
let end = at.saturating_add(c).min(pts.len());
crates/metafile/src/tests.rs
+15
View File
@@ -1053,3 +1053,18 @@ fn plain_wmf_target_comes_from_a_window_that_was_read() {
let (x0, y0, x1, y1) = rect_extent(&p.items[0]);
assert!(close(x0, 0.0) && close(y0, 0.0) && close(x1, 200.0) && close(y1, 100.0), "{x0} {y0} {x1} {y1}");
}
/// A poly-polygon's group counts are file data: a long list of empty groups adds no ops, so it
/// mustn't reserve room for them either (the canvas charges what a path holds, capacity included).
#[test]
fn empty_poly_groups_reserve_nothing() {
let pts = [(0.0, 0.0), (10.0, 0.0), (10.0, 10.0)];
let mut counts = vec![0usize; 100_000];
counts.push(3);
let ops = crate::geom::subpaths(&pts, &counts, true);
assert_eq!(ops.len(), 4, "M L L Z");
assert!(ops.capacity() <= 8, "capacity {}", ops.capacity());
// Counts past the points are cut short, and the size still matches.
let ops = crate::geom::subpaths(&pts, &[2, 5, 4], false);
assert_eq!((ops.len(), ops.capacity()), (3, 3));
}
crates/pdf/src/lib.rs
+82 -24
View File
@@ -747,7 +747,9 @@ struct Exporter<'a> {
/// Bookmark positions by name (the first of a name wins), gathered once: looking each internal
/// link up by walking the whole document made a long document with a linked table of contents
/// take seconds to export.
bookmarks: std::cell::OnceCell<HashMap<String, wordcraft_doc::Pos>>,
/// Bookmarks by exact name, and by ASCII-lowercased name for the case-blind fallback; the
/// first in document order wins either way.
bookmarks: std::cell::OnceCell<(HashMap<String, wordcraft_doc::Pos>, HashMap<String, wordcraft_doc::Pos>)>,
}
/// What the content being drawn is, for tagging.
@@ -918,8 +920,9 @@ impl Exporter<'_> {
placeholders: false,
..Default::default()
};
let clips = clipped_cells(page);
for it in page.header.iter().chain(page.footer.iter()) {
let draws = self.draws(page, it, &dopts);
let draws = self.draws(page, it, &clips, &dopts);
self.tagged(s, Role::Artifact(ArtifactType::Other), None, |me, s| {
for d in &draws {
me.draw(s, d);
@@ -927,7 +930,7 @@ impl Exporter<'_> {
});
}
for it in &page.items {
let draws = self.draws(page, it, &dopts);
let draws = self.draws(page, it, &clips, &dopts);
match it {
Placed::Lines { story, path, para, .. } => {
let idx = self.entry(*story, path, para);
@@ -990,13 +993,14 @@ impl Exporter<'_> {
}
/// Display items for one placed item.
fn draws(&self, page: &Page, it: &Placed, dopts: &DisplayOptions) -> Vec<Draw> {
fn draws(&self, page: &Page, it: &Placed, clips: &CellClips, dopts: &DisplayOptions) -> Vec<Draw> {
let mut items = vec![it.clone()];
// Lines in a cell cut off at its edge are drawn inside its clip: `page_display` finds the
// cell among the items (cells themselves draw nothing).
if matches!(it, Placed::Lines { .. }) {
let clipped = page.header.iter().chain(&page.footer).chain(&page.items).filter(|c| matches!(c, Placed::Cell { clip: true, .. }));
items.extend(clipped.cloned());
// cell among the items (cells themselves draw nothing). Only that line's cell goes along.
if let Placed::Lines { story, path, .. } = it
&& let Some(cell) = clip_cell(clips, *story, &path.0)
{
items.push(cell.clone());
}
let one = Page { w: page.w, h: page.h, section: page.section, number: page.number, items, body: page.body, ..Default::default() };
page_display(self.doc, &one, dopts)
@@ -1544,14 +1548,15 @@ impl Exporter<'_> {
/// Where a bookmark is: (output page, x, y) — the page and line its start lands on, which
/// in a paragraph split across pages need not be the paragraph's first page.
fn bookmark(&self, name: &str) -> Option<(usize, f32, f32)> {
let marks = self.bookmarks.get_or_init(|| {
let mut m = HashMap::new();
let (exact, folded) = self.bookmarks.get_or_init(|| {
let (mut exact, mut folded) = (HashMap::new(), HashMap::new());
for (n, pos) in self.doc.bookmarks() {
m.entry(n).or_insert(pos);
folded.entry(n.to_ascii_lowercase()).or_insert_with(|| pos.clone());
exact.entry(n).or_insert(pos);
}
m
(exact, folded)
});
let pos = marks.get(name).or_else(|| marks.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v))?.clone();
let pos = exact.get(name).or_else(|| folded.get(&name.to_ascii_lowercase()))?.clone();
if let Some(c) = self.lay.caret(&pos)
&& let Some(out) = self.out_index.get(&c.page)
{
@@ -1703,11 +1708,12 @@ impl Exporter<'_> {
match font {
Some(font) => {
let upem = face.upem.max(1.0) as f32;
let ends = cluster_ends(&shaped.iter().map(|g| g.cluster).collect::<Vec<_>>(), t.text.len());
let kg: Vec<KrillaGlyph> = shaped
.iter()
.enumerate()
.map(|(i, g)| {
let end = shaped.iter().skip(i + 1).map(|n| n.cluster).find(|cl| *cl > g.cluster).unwrap_or(t.text.len());
let end = ends.get(i).copied().unwrap_or(t.text.len());
let start = g.cluster.min(end);
let x = lay.glyphs.get(i).map_or(0.0, |p| p.0);
let adv = match lay.glyphs.get(i + 1) {
@@ -1758,11 +1764,12 @@ impl Exporter<'_> {
if font.is_none() && !self.outlined.contains(&face.id()) {
return;
}
let ends = cluster_ends(&shaped.iter().map(|g| g.cluster).collect::<Vec<_>>(), text.len());
let glyphs: Vec<KrillaGlyph> = shaped
.iter()
.enumerate()
.map(|(i, g)| {
let end = shaped.iter().skip(i + 1).map(|n| n.cluster).find(|cl| *cl > g.cluster).unwrap_or(text.len());
let end = ends.get(i).copied().unwrap_or(text.len());
let start = g.cluster.min(end);
KrillaGlyph::new(
GlyphId::new(g.gid),
@@ -1891,6 +1898,48 @@ fn path_from(b: &kurbo::BezPath) -> Option<Path> {
pb.finish()
}
/// A page's cells whose content is cut off at their edge, by story and content path (table,
/// row, cell), with their place in header, footer, items order (the first wins, as in layout).
type CellClips<'a> = HashMap<StoryRef, HashMap<Vec<u32>, (usize, &'a Placed)>>;
fn clipped_cells(page: &Page) -> CellClips<'_> {
let mut clips: CellClips = HashMap::new();
for (order, it) in page.header.iter().chain(&page.footer).chain(&page.items).enumerate() {
if let Placed::Cell { table, row, cell, story, clip: true, .. } = it
&& let (Ok(row), Ok(cell)) = (u32::try_from(*row), u32::try_from(*cell))
{
let mut prefix = table.0.clone();
prefix.extend([row, cell]);
clips.entry(*story).or_default().entry(prefix).or_insert((order, it));
}
}
clips
}
/// The clipped cell a line at `path` of `story` is in: the first one whose path starts it. A
/// lookup per prefix of the (short) path instead of passing every clipped cell with every line.
fn clip_cell<'a>(clips: &CellClips<'a>, story: StoryRef, path: &[u32]) -> Option<&'a Placed> {
let cells = clips.get(&story)?;
(0..=path.len()).filter_map(|k| cells.get(path.get(..k)?)).min_by_key(|(order, _)| *order).map(|(_, it)| *it)
}
/// Where each glyph's text ends: the cluster (byte offset) of the first later glyph whose cluster
/// is past the glyph's own, else `len`. One backward pass keeping the later clusters that can
/// still be an answer, so a long run of glyphs sharing a cluster (combining marks) stays linear.
fn cluster_ends(clusters: &[usize], len: usize) -> Vec<usize> {
let mut ends = vec![len; clusters.len()];
// Later clusters, nearest last, each greater than every one nearer.
let mut later: Vec<usize> = Vec::new();
for (end, &c) in ends.iter_mut().zip(clusters).rev() {
while later.last().is_some_and(|&n| n <= c) {
later.pop();
}
*end = later.last().copied().unwrap_or(len);
later.push(c);
}
ends
}
/// Decode a picture for embedding (validated with the `image` crate first, so a broken file
/// can't fail the whole export).
fn load_image(data: &Arc<Vec<u8>>) -> Option<Image> {
@@ -1903,19 +1952,28 @@ fn load_image(data: &Arc<Vec<u8>>) -> Option<Image> {
/// [`load_image`] with an explicit RGBA byte budget (a small one in tests).
fn load_image_within(data: &Arc<Vec<u8>>, budget: u64) -> Option<Image> {
use image::ImageDecoder;
let fmt = image::guess_format(data).ok()?;
// Header-only size check before anything is allocated (decode buffer + RGBA conversion are
// charged); the side limits and the default 512 MB allocation cap stay as a second guard.
let (hw, hh) = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?.into_dimensions().ok()?;
if !wordcraft_geom::decode_fits(hw, hh, budget) {
return None;
}
let mut reader = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?;
let mut limits = image::Limits::default();
limits.max_image_width = Some(MAX_IMAGE_SIDE);
limits.max_image_height = Some(MAX_IMAGE_SIDE);
reader.limits(limits);
let dec = reader.decode().ok()?;
let reader = || {
let mut r = image::ImageReader::new(std::io::Cursor::new(data.as_slice())).with_guessed_format().ok()?;
r.limits(limits.clone());
Some(r)
};
// Header-only size check before anything is allocated (the decoder's buffer at its own pixel
// size, 8 bytes for 16-bit RGBA, plus the RGBA conversion are charged); the side limits and
// the default 512 MB allocation cap apply to the header read and the decode alike.
let (hw, hh, native) = {
let d = reader()?.into_decoder().ok()?;
let (w, h) = d.dimensions();
(w, h, u64::from(d.color_type().bytes_per_pixel()))
};
if !wordcraft_geom::decode_peak_fits(hw, hh, native, budget) {
return None;
}
let dec = reader()?.decode().ok()?;
let (w, h) = (dec.width(), dec.height());
if w == 0 || h == 0 || w > MAX_IMAGE_SIDE || h > MAX_IMAGE_SIDE {
return None;
crates/pdf/src/tests.rs
+79
View File
@@ -1323,6 +1323,7 @@ fn bookmark_destination_is_the_page_of_its_start() {
let mut p = Paragraph::with_text("Paragraph start\u{c}Bookmark target", CharProps::default());
let at = "Paragraph start\u{c}".len();
p.insert_object(at, InlineObject::BookmarkStart { name: "target".into() }, &CharProps::default()).unwrap();
p.insert_object(at, InlineObject::BookmarkStart { name: "First".into() }, &CharProps::default()).unwrap();
p.insert_object(0, InlineObject::BookmarkStart { name: "first".into() }, &CharProps::default()).unwrap();
d.body = vec![para_block(p)];
let lay = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default());
@@ -1354,6 +1355,13 @@ fn bookmark_destination_is_the_page_of_its_start() {
let (page_ci, x_ci, y_ci) = ex.bookmark("TARGET").unwrap();
assert_eq!(page_ci, 1);
assert_eq!((x_ci, y_ci), (x, y));
// An exact name wins; names that differ only in case fall back to the first in the document
// (a folded index, not a scan of a hash map in arbitrary order for every link).
assert_eq!(ex.bookmark("First").map(|b| b.0), Some(1));
assert_eq!(ex.bookmark("FIRST").map(|b| b.0), Some(0));
assert!(ex.bookmark("absent").is_none());
let (exact, folded) = ex.bookmarks.get().unwrap();
assert_eq!((exact.len(), folded.len()), (3, 2));
// Whitespace in ISO date string is parsed successfully
assert!(parse_iso(" 2024-05-06T07:08:09Z \n").is_some());
@@ -2140,6 +2148,13 @@ fn image_budget_is_checked_from_the_header() {
// The decoded image and its RGBA conversion are charged: two buffers at peak.
assert!(load_image_within(&data, 1024).is_some());
assert!(load_image_within(&data, 1023).is_none());
// 16-bit RGBA decodes to 8 bytes a pixel, alive with its RGBA8 conversion: 12 bytes a pixel.
let mut wide = Vec::new();
let img16 = image::ImageBuffer::<image::Rgba<u16>, _>::from_fn(16, 8, |x, y| image::Rgba([x as u16 * 999, y as u16, 9, 65535]));
image::DynamicImage::ImageRgba16(img16).write_to(&mut std::io::Cursor::new(&mut wide), image::ImageFormat::Png).unwrap();
let wide = Arc::new(wide);
assert!(load_image_within(&wide, 1024).is_none(), "two RGBA8 buffers' worth is not enough for RGBA16");
assert!(load_image_within(&wide, 1536).is_some());
// A header (valid CRC) claiming 30000x30000 with no pixel data reads fine, then the budget refuses it.
let hdr = ihdr_only_png(30_000, 30_000);
let dims = image::ImageReader::new(std::io::Cursor::new(hdr.as_slice())).with_guessed_format().unwrap().into_dimensions().unwrap();
@@ -2411,3 +2426,67 @@ fn only_safe_link_targets_become_annotations() {
.collect();
assert_eq!(uris, ["https://example.com/", "mailto:ada@example.com"]);
}
/// A glyph's text runs to the first later glyph with a cluster past its own (the end of the text
/// without one), found in one pass: a long run of glyphs sharing a cluster used to rescan the rest
/// of the run for every glyph.
#[test]
fn cluster_ends_match_the_next_larger_cluster_in_linear_time() {
let brute = |cl: &[usize], len: usize| -> Vec<usize> {
(0..cl.len()).map(|i| cl.iter().skip(i + 1).copied().find(|n| *n > cl[i]).unwrap_or(len)).collect::<Vec<_>>()
};
for cl in [vec![], vec![0], vec![0, 0, 3, 3, 5], vec![5, 3, 3, 0], vec![2, 0, 4, 1, 1, 6, 3], vec![0, 7, 2, 2, 9, 1]] {
assert_eq!(cluster_ends(&cl, 10), brute(&cl, 10), "{cl:?}");
}
// One base letter and a million combining marks in its cluster, then one more letter.
// Milliseconds; the rescan (`brute`) took 2.9 s at 50,000 and grows with the square, so
// about 20 minutes here.
let n = 1_000_000;
let mut cl = vec![0; n + 1];
cl.push(2 * n + 2);
let start = std::time::Instant::now();
let ends = cluster_ends(&cl, 2 * n + 4);
assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed());
assert!(ends[..=n].iter().all(|e| *e == 2 * n + 2));
assert_eq!(ends.last(), Some(&(2 * n + 4)));
}
/// A line in a cell cut off at its edge is drawn with that cell's clip, found by path in an index
/// built once per page: every line used to carry a clone of every clipped cell on the page, so a
/// page of exact-height rows cost lines × cells.
#[test]
fn a_line_finds_its_clipped_cell_by_path() {
let cell = |story, table: Vec<u32>, row, cell, clip| Placed::Cell {
rect: Rect::default(),
covered: false,
table: DocPath(table),
row,
cell,
story,
clip,
};
let mut page = Page { items: vec![cell(StoryRef::Body, vec![0], 0, 1, false), cell(StoryRef::Body, vec![2], 1, 0, true)], ..Default::default() };
// A nested clipped cell inside the first one; the outer one comes first in the page.
page.items.push(cell(StoryRef::Body, vec![2, 1, 0, 3], 0, 0, true));
page.footer.push(cell(StoryRef::Part(7), vec![0], 0, 0, true));
let clips = clipped_cells(&page);
let found = |story, path: &[u32]| {
clip_cell(&clips, story, path).and_then(|c| if let Placed::Cell { table, .. } = c { Some(table.0.clone()) } else { None })
};
assert_eq!(found(StoryRef::Body, &[2, 1, 0, 0]), Some(vec![2]));
assert_eq!(found(StoryRef::Body, &[2, 1, 0, 3, 0, 0, 1]), Some(vec![2]), "the first cell in page order wins, as in layout");
assert_eq!(found(StoryRef::Body, &[0, 0, 1, 0]), None, "that cell isn't clipped");
assert_eq!(found(StoryRef::Body, &[2, 0, 0, 0]), None, "another row");
assert_eq!(found(StoryRef::Part(7), &[0, 0, 0, 2]), Some(vec![0]));
assert_eq!(found(StoryRef::Body, &[0, 0, 0, 2]), None, "another story");
// 100 000 clipped cells and as many lines: a lookup each, not a pass over the cells each
// (10^10 steps, minutes; the lookups take well under a second on a loaded machine).
let n = 100_000;
let big = Page { items: (0..n).map(|r| cell(StoryRef::Body, vec![0], r, 0, true)).collect(), ..Default::default() };
let start = std::time::Instant::now();
let clips = clipped_cells(&big);
let hits = (0..n as u32).filter(|r| clip_cell(&clips, StoryRef::Body, &[0, *r, 0, 0]).is_some()).count();
assert_eq!(hits, n);
assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed());
}
crates/proof/Cargo.toml
+1 -1
View File
@@ -10,7 +10,7 @@ workspace = true
[dependencies]
log = { workspace = true }
miniz_oxide = "0.8"
miniz_oxide = "0.9"
unicode-normalization = "0.1"
[dev-dependencies]
crates/proof/src/affix.rs
+152 -22
View File
@@ -120,6 +120,11 @@ fn parse_cond(text: &str) -> Result<Box<[Cond]>, String> {
const MAX_CHAIN: u8 = 2;
/// How many suffixes may sit under the outermost one: the depth `form_has_flag` walks.
const INNER_CHAIN: u8 = MAX_CHAIN - 1;
/// Most forms [`AffixDict::forms`] lists for one stem (real stems have a few hundred at most).
pub const MAX_FORMS: usize = 4096;
/// Most rules [`AffixDict::forms`] tries along continuation chains and prefix–suffix cross
/// products for one stem: real dictionaries need a few thousand, hostile ones are cut off.
const MAX_FORMS_WORK: usize = 1 << 18;
/// One prefix or suffix rule.
#[derive(Clone, Debug)]
@@ -182,6 +187,8 @@ pub struct AffixDict {
suffixes: Vec<Rule>,
/// Appended string → groups of suffix rules by stripped string.
sfx_index: FxMap<Box<str>, Vec<SfxGroup>>,
/// Per flag bit: the suffix rules of that flag, in rule order.
sfx_by_flag: Vec<Vec<u32>>,
/// Longest appended suffix, in chars.
max_sfx_chars: usize,
try_chars: String,
@@ -484,14 +491,27 @@ impl AffixDict {
/// Build the suffix index and the stem hash table.
fn index(&mut self) {
let mut idx: FxMap<Box<str>, Vec<SfxGroup>> = FxMap::default();
// Each (appended, stripped) pair's group, found by hash rather than by scanning the
// appended string's groups: a dictionary with many strips per suffix stays linear.
let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();
let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];
for (i, r) in self.suffixes.iter().enumerate() {
let groups = idx.entry(r.add.clone()).or_default();
let Ok(i) = u32::try_from(i) else { break };
match groups.iter_mut().find(|g| g.strip == r.strip) {
match group_of.get(&(&*r.add, &*r.strip)).and_then(|&g| groups.get_mut(g)) {
Some(g) => g.rules.push(i),
None => groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }),
None => {
group_of.insert((&r.add, &r.strip), groups.len());
groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] });
}
}
if r.bit != 0
&& let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)
{
list.push(i);
}
}
self.sfx_by_flag = by_flag;
self.max_sfx_chars = self.suffixes.iter().map(|r| r.add.chars().count()).max().unwrap_or(0);
self.sfx_index = idx;
let n = self.len();
@@ -596,10 +616,14 @@ impl AffixDict {
None
}
/// Does `w` carry flag `bit` — as a stem, or as the form of a suffix rule whose
/// continuation classes include `bit` (chained suffixation, depth-bounded)?
fn form_has_flag(&self, w: &str, bit: u128, depth: u8, excluded: u128) -> bool {
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & bit != 0) {
/// Does `w` carry any flag of `flags` — as a stem, or as the form of a suffix rule whose
/// continuation classes include one (chained suffixation, depth-bounded)?
///
/// The rules of a group that could lead to `w` are merged into one mask of their flags and
/// followed once, not once per rule: the answer is the same, but thousands of rules of one
/// flag cost one walk, not one each (#3613).
fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {
if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {
return true;
}
if depth == 0 {
@@ -614,10 +638,14 @@ impl AffixDict {
root.clear();
root.push_str(base);
root.push_str(&g.strip);
for r in g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)) {
if r.cont & bit != 0 && r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, depth - 1, excluded) {
return true;
}
let inner = g
.rules
.iter()
.filter_map(|&i| self.suffixes.get(i as usize))
.filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))
.fold(0u128, |m, r| m | r.bit);
if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {
return true;
}
}
}
@@ -690,9 +718,13 @@ impl AffixDict {
return true;
}
// Continuation classes: `root` need not be a stem carrying this rule's flag —
// it may be a form that gained the flag from another rule's continuation.
if need == 0 && rules().any(|r| r.cond_matches(&root, true) && self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded)) {
return true;
// it may be a form that gained the flag from another rule's continuation. The
// matching rules' flags are looked for together, in one walk.
if need == 0 {
let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);
if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {
return true;
}
}
}
}
@@ -725,13 +757,19 @@ impl AffixDict {
}
}
/// Every form of the stem `stem` (itself included), unsorted, possibly with repeats.
/// Every form of the stem `stem` (itself included), unsorted, possibly with repeats; at most
/// [`MAX_FORMS`], and at most [`MAX_FORMS_WORK`] rules tried along the continuation chains and
/// cross products, so a dictionary with thousands of self-continuing rules can't make one stem
/// cost millions of strings (#3613).
pub fn forms(&self, stem: &str) -> Vec<String> {
let Some(f) = self.stem_flags(stem) else { return Vec::new() };
let mut out = vec![stem.to_string()];
let mut cross_sfx = Vec::new();
// Suffix forms queued with the continuation classes they carry, for chained expansion.
// Suffix forms queued with the continuation classes they carry, for chained expansion;
// each (form, classes) pair is expanded once.
let mut chain: Vec<(String, u128)> = Vec::new();
let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();
let mut work = 0usize;
for r in &self.suffixes {
if f & r.bit == 0 || !r.cond_matches(stem, true) {
continue;
@@ -741,29 +779,39 @@ impl AffixDict {
if r.cross {
cross_sfx.push(form.clone());
}
if r.cont != 0 {
if r.cont != 0 && queued.insert((form.clone(), r.cont)) {
chain.push((form.clone(), r.cont));
}
out.push(form);
if out.len() >= MAX_FORMS {
return out;
}
}
// Continuation classes chain suffixes (`workable/B` → `workables`), up to [`MAX_CHAIN`]
// suffixes in all — the same bound the checker walks.
for _ in 0..INNER_CHAIN {
'chain: for _ in 0..INNER_CHAIN {
let level = std::mem::take(&mut chain);
if level.is_empty() {
break;
}
for (form, flags) in level {
for r in &self.suffixes {
if flags & r.bit == 0 || !r.cond_matches(&form, true) {
for r in self.suffixes_with(flags).into_iter().filter_map(|i| self.suffixes.get(i as usize)) {
work += 1;
if work > MAX_FORMS_WORK {
break 'chain;
}
if !r.cond_matches(&form, true) {
continue;
}
let Some(base) = form.strip_suffix(&*r.strip) else { continue };
let next = format!("{base}{}", r.add);
if r.cont != 0 {
if r.cont != 0 && queued.insert((next.clone(), r.cont)) {
chain.push((next.clone(), r.cont));
}
out.push(next);
if out.len() >= MAX_FORMS {
return out;
}
}
}
}
@@ -775,15 +823,38 @@ impl AffixDict {
out.push(format!("{}{rest}", r.add));
if r.cross {
for s in &cross_sfx {
work += 1;
if out.len() >= MAX_FORMS || work > MAX_FORMS_WORK {
return out;
}
if let Some(rest) = s.strip_prefix(&*r.strip) {
out.push(format!("{}{rest}", r.add));
}
}
}
if out.len() >= MAX_FORMS {
return out;
}
}
out
}
/// The suffix rules carrying any flag of `mask`, in rule order.
fn suffixes_with(&self, mask: u128) -> Vec<u32> {
let mut rules: Vec<u32> = Vec::new();
let mut lists = 0;
for (bit, list) in self.sfx_by_flag.iter().enumerate() {
if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {
rules.extend_from_slice(list);
lists += 1;
}
}
if lists > 1 {
rules.sort_unstable();
}
rules
}
/// The `TRY` letters, most frequent first (suggestions try these).
pub fn try_chars(&self) -> &str {
&self.try_chars
@@ -877,8 +948,8 @@ impl AffixDict {
/// Read data written by [`AffixDict::to_bytes`].
pub fn from_bytes(data: &[u8]) -> Result<AffixDict, String> {
let raw =
miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20).map_err(|e| format!("affix dictionary: inflate failed: {e:?}"))?;
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, 64 << 20)
.map_err(|e| format!("affix dictionary: inflate failed: {:?}", e.status))?;
let words_at = raw.windows(7).position(|w| w == b"\nWORDS\n").ok_or("affix dictionary: no WORDS section")?;
let head = std::str::from_utf8(raw.get(..words_at).unwrap_or_default()).map_err(|_| "affix dictionary: bad utf-8 in the rules")?;
let mut lines = head.lines();
@@ -1216,6 +1287,65 @@ pub(crate) mod tests {
}
}
#[test]
fn continuation_classes_merge_the_flags_of_a_group() {
// `s` under B and under C share one group (same append, same strip): a form carrying
// either flag takes it, and the stem carrying neither doesn't.
let aff = "SET UTF-8\nSFX A N 1\nSFX A 0 able/C .\nSFX D N 1\nSFX D 0 er/B .\nSFX B N 1\nSFX B 0 s .\nSFX C N 1\nSFX C 0 s [e]\n";
let d = AffixDict::from_hunspell(aff, "2\nwork/A\nplay/D\n", &[]).unwrap();
assert!(d.contains("workables") && d.contains("players"));
assert!(!d.contains("works") && !d.contains("plays") && !d.contains("workabless"));
assert!(d.forms("work").iter().any(|f| f == "workables") && d.forms("play").iter().any(|f| f == "players"));
}
/// `n` suffix rules of flag A on one stem, `rule(i)` giving each rule's strip, append and condition.
fn many_rules(n: usize, rule: impl Fn(usize) -> String) -> AffixDict {
let mut aff = format!("SET UTF-8\nSFX A Y {n}\n");
for i in 0..n {
aff.push_str(&format!("SFX A {}\n", rule(i)));
}
AffixDict::from_hunspell(&aff, "1\nkot/A\n", &[]).unwrap()
}
#[test]
fn self_continuing_rules_check_in_linear_time() {
// Thousands of empty-append rules that continue with their own flag: each check used to
// follow every rule from every rule, millions of steps a word (#3613). The checks take
// about a millisecond; the old checker took 14 s for two of them at 16,000 rules, so at
// 40,000 (quadratic: minutes) the 10 s bound leaves a busy machine a wide margin.
let d = many_rules(40_000, |_| "0 0/A .".into());
let t = std::time::Instant::now();
assert!(d.check("kot") && !d.check("pies") && !d.check("kotx"));
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
}
#[test]
fn forms_of_self_continuing_rules_are_bounded() {
// Every rule's form continues with every rule: millions of strings for one stem (#3613).
let same = many_rules(3000, |_| "0 0/A .".into());
let distinct = many_rules(3000, |i| format!("0 x{i}/A ."));
let t = std::time::Instant::now();
let (a, b) = (same.forms("kot"), distinct.forms("kot"));
// The cap on the count is what tells the old code (9 million forms each) apart; the
// generous time bound (tens of ms here) only backs up the rule-visit cap.
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
assert!(a.len() <= MAX_FORMS && b.len() <= MAX_FORMS, "{} {}", a.len(), b.len());
assert!(b.iter().any(|f| f == "kotx0x1"), "chained forms are still listed");
}
#[test]
fn many_strips_per_suffix_index_in_linear_time() {
// One append with thousands of strips: the index used to scan the append's groups for
// each rule. Building 200,000 rules takes well under a second; the old index took 3 s at
// 30,000 and over 10 minutes at 300,000.
let n = 200_000;
let t = std::time::Instant::now();
let d = many_rules(n, |i| format!("k{i} y ."));
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
assert_eq!(d.sfx_index.get("y").map(Vec::len), Some(n));
assert!(!d.contains("koty"));
}
#[test]
fn compact_format_round_trips_continuations_and_forbidden() {
let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nSFX A N 1\nSFX A 0 able/B .\nSFX B N 1\nSFX B 0 s .\n";
crates/proof/src/dict.rs
+46 -2
View File
@@ -14,6 +14,11 @@ pub struct Dictionary {
}
const MAGIC: &[u8] = b"DCHYPH1\n";
/// Largest inflated dictionary, and most bytes of words kept from one (the bundled US English one
/// is about 1 MB).
const MAX_BYTES: usize = 64 << 20;
/// Most words read from one dictionary (the bundled one has about 160k).
const MAX_ENTRIES: usize = 4_000_000;
impl Dictionary {
/// Build from `(lowercase word, break mask)` pairs (sorted and de-duplicated here; the first
@@ -76,9 +81,16 @@ impl Dictionary {
miniz_oxide::deflate::compress_to_vec(&raw, 10)
}
/// Deserialize data written by [`Dictionary::to_bytes`].
/// Deserialize data written by [`Dictionary::to_bytes`]. Data that inflates past
/// [`MAX_BYTES`], or holds more than [`MAX_ENTRIES`] words or [`MAX_BYTES`] of them, is refused.
pub fn from_bytes(data: &[u8]) -> Result<Dictionary, String> {
let raw = miniz_oxide::inflate::decompress_to_vec(data).map_err(|e| format!("hyphenation dictionary: inflate failed: {e:?}"))?;
Dictionary::from_bytes_within(data, MAX_BYTES, MAX_ENTRIES)
}
/// [`Dictionary::from_bytes`] with the limits given.
fn from_bytes_within(data: &[u8], max_bytes: usize, max_entries: usize) -> Result<Dictionary, String> {
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(data, max_bytes)
.map_err(|e| format!("hyphenation dictionary: inflate failed: {:?}", e.status))?;
let body = raw.strip_prefix(MAGIC).ok_or("hyphenation dictionary: bad header")?;
let mut d = Dictionary { words: String::with_capacity(body.len()), offs: vec![0], masks: Vec::new() };
let mut cur: Vec<u8> = Vec::new();
@@ -94,6 +106,11 @@ impl Dictionary {
i = end + 1;
let s = std::str::from_utf8(&cur).map_err(|_| "hyphenation dictionary: bad utf-8")?;
let (w, m) = parse_hyphenated(s);
// Front coding lets a short line repeat a long prefix: bound what is kept, not only
// what is read.
if d.masks.len() >= max_entries || d.words.len() + w.len() > max_bytes {
return Err("hyphenation dictionary: too large".into());
}
d.words.push_str(&w);
d.offs.push(d.words.len() as u32);
d.masks.push(m);
@@ -147,4 +164,31 @@ mod tests {
assert_eq!(back.get("naïveté").map(|m| hyphenated("naïveté", m)), Some("naïve-té".to_string()));
assert!(back.get("dog").is_none());
}
#[test]
fn oversized_data_is_refused() {
const LIMIT: usize = 1 << 16;
let within = |raw: &[u8]| Dictionary::from_bytes_within(&miniz_oxide::deflate::compress_to_vec(raw, 6), LIMIT, 1000);
// Data that inflates past the limit is refused before anything is decoded.
let mut raw = MAGIC.to_vec();
raw.resize(LIMIT + 1, b'a');
assert!(within(&raw).unwrap_err().contains("inflate failed"));
// Short lines that each repeat a long prefix: small to inflate, but the words they spell
// out would be ~100x larger. Kept storage is bounded too.
let mut raw = MAGIC.to_vec();
raw.push(0);
raw.extend_from_slice(&[b'a'; 255]);
raw.push(b'\n');
for _ in 0..(LIMIT / 255 + 2) {
raw.extend_from_slice(&[255, b'\n']);
}
assert!(raw.len() < LIMIT);
assert!(within(&raw).unwrap_err().contains("too large"));
// So is the number of words.
let words: Vec<(String, u64)> = (0..1001).map(|i| (format!("w{i}"), 0)).collect();
let raw = miniz_oxide::inflate::decompress_to_vec(&Dictionary::from_entries(words).to_bytes()).unwrap();
assert!(within(&raw).unwrap_err().contains("too large"));
// The real limits leave the bundled dictionary whole.
assert!(Dictionary::en_us().len() > 100_000);
}
}
crates/proof/src/hunspell.rs
+93 -9
View File
@@ -140,16 +140,31 @@ pub fn find_in(dir: &Path, names: &[&str]) -> Option<(PathBuf, PathBuf)> {
})
}
/// Read a dictionary file: a regular file of at most [`MAX_FILE`] bytes. The read itself is
/// bounded, so a file that grows after its size is checked (or a filesystem that reports a wrong
/// size) can't make it read more (#3661).
pub(crate) fn read_capped(p: &Path) -> Result<Vec<u8>, String> {
use std::io::Read;
let err = |e: std::io::Error| format!("{}: {e}", p.display());
let f = std::fs::File::open(p).map_err(err)?;
let meta = f.metadata().map_err(err)?;
if !meta.is_file() {
return Err(format!("{}: not a regular file", p.display()));
}
if meta.len() > MAX_FILE {
return Err(format!("{}: too large ({} bytes)", p.display(), meta.len()));
}
let mut buf = Vec::new();
f.take(MAX_FILE + 1).read_to_end(&mut buf).map_err(err)?;
if buf.len() as u64 > MAX_FILE {
return Err(format!("{}: too large", p.display()));
}
Ok(buf)
}
/// Read a Hunspell pair from disk (size-capped), decoded per the `.aff`'s `SET` line.
pub fn load(aff: &Path, dic: &Path) -> Result<AffixDict, String> {
let read = |p: &Path| -> Result<Vec<u8>, String> {
let len = std::fs::metadata(p).map_err(|e| format!("{}: {e}", p.display()))?.len();
if len > MAX_FILE {
return Err(format!("{}: too large ({len} bytes)", p.display()));
}
std::fs::read(p).map_err(|e| format!("{}: {e}", p.display()))
};
let (aff_text, dic_text) = decode(&read(aff)?, &read(dic)?)?;
let (aff_text, dic_text) = decode(&read_capped(aff)?, &read_capped(dic)?)?;
let source = format!("Hunspell dictionary {}", dic.display());
AffixDict::from_hunspell(&aff_text, &dic_text, &[source.as_str()])
}
@@ -159,6 +174,23 @@ fn decode_latin1(bytes: &[u8]) -> String {
bytes.iter().map(|&b| char::from(b)).collect()
}
/// ISO 8859-9 (Latin-5, Turkish): ISO 8859-1 with Ğ İ Ş ğ ı ş in place of Ð Ý Þ ð ý þ. The
/// encoding of older Turkish dictionaries (`SET ISO8859-9`).
fn decode_latin5(bytes: &[u8]) -> String {
bytes
.iter()
.map(|&b| match b {
0xd0 => 'Ğ',
0xdd => 'İ',
0xde => 'Ş',
0xf0 => 'ğ',
0xfd => 'ı',
0xfe => 'ş',
_ => char::from(b),
})
.collect()
}
/// ISO 8859-2 code points 0xA0..=0xFF (0x00..=0x9F map to themselves).
const ISO_8859_2_HIGH: &str = "\u{a0}Ą˘Ł¤ĽŚ§¨ŠŞŤŹ\u{ad}ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
@@ -184,7 +216,8 @@ fn decode_high(bytes: &[u8], high: &str) -> String {
/// Decode an `.aff`/`.dic` pair to UTF-8 from the encoding the `.aff`'s `SET` line names
/// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the
/// one the original Portuguese VERO uses, KOI8-R and Windows-1251 the ones older Russian dictionaries use).
/// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R
/// and Windows-1251 the ones older Russian dictionaries use).
pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();
let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();
@@ -195,6 +228,7 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {
}
"ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))),
"ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),
"ISO8859-9" | "ISO-8859-9" => Ok((decode_latin5(aff), decode_latin5(dic))),
"KOI8-R" | "KOI8R" => Ok((decode_high(aff, KOI8_R_HIGH), decode_high(dic, KOI8_R_HIGH))),
"CP1251" | "WINDOWS-1251" | "MICROSOFT-CP1251" => Ok((decode_high(aff, CP1251_HIGH), decode_high(dic, CP1251_HIGH))),
other => Err(format!("unsupported encoding {other}")),
@@ -213,6 +247,8 @@ mod tests {
assert_eq!(d, "2\nżółw\n");
let (_, d) = decode("SET UTF-8\n".as_bytes(), "żółw".as_bytes()).unwrap();
assert_eq!(d, "żółw");
let (_, d) = decode(b"SET ISO8859-9\n", b"\xd0\xdd\xde\xf0\xfd\xfe\xd6\xe7").unwrap();
assert_eq!(d, "ĞİŞğışÖç");
assert!(decode(b"SET SHIFT_JIS\n", b"").is_err());
assert!(decode(b"SET UTF-8\n", b"\xff\xfe").is_err());
}
@@ -283,4 +319,52 @@ mod tests {
assert!(load(&dir.join("missing.aff"), &dic).is_err());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn reads_are_bounded_and_regular_files_only() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-cap-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let aff = dir.join("xx.aff");
std::fs::write(&aff, "SET UTF-8\n").unwrap();
// One byte over the cap (sparse: nothing is written) is refused without reading it.
let big = dir.join("big.dic");
std::fs::File::create(&big).unwrap().set_len(MAX_FILE + 1).unwrap();
assert!(load(&aff, &big).unwrap_err().contains("too large"));
// A directory, and on Unix an endless device, are not dictionary files. (No clock: a
// reader that read `/dev/zero` would never return, so the error itself is the proof.)
assert!(load(&aff, &dir).unwrap_err().contains("not a regular file"));
#[cfg(unix)]
assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file"));
// Turkish reads through the same reader.
let (tr_aff, tr_dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&tr_aff, "SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n").unwrap();
std::fs::write(&tr_dic, "1\nkitap/1\n").unwrap();
assert!(crate::tr::load(&tr_aff, &tr_dic).unwrap().contains("kitaplar"));
assert!(crate::tr::load(&tr_aff, &big).unwrap_err().contains("too large"));
let _ = std::fs::remove_dir_all(&dir);
}
/// A Turkish dictionary in ISO 8859-9, the encoding older tr_TR dictionaries use, read from
/// disk through the capped reader and decoded per its `SET` line: its words with Turkish
/// letters (ğ ı ş İ, and ç ö ü from Latin-1) are found, as stems and with a suffix. Before the
/// Turkish dictionary went through this reader it was read as UTF-8, which these bytes are not.
#[test]
fn a_turkish_dictionary_in_iso_8859_9_loads_from_disk() {
let dir = std::env::temp_dir().join(format!("wordcraft-hunspell-latin5-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let (aff, dic) = (dir.join("tr_TR.aff"), dir.join("tr_TR.dic"));
std::fs::write(&aff, b"SET ISO8859-9\nSFX 1 Y 2\nSFX 1 0 lar .\nSFX 1 0 \xfd .\nSFX 2 Y 1\nSFX 2 0 ler .\n").unwrap();
// ağaç/1 (-lar, -ı), ışık, şişe/2 (-ler), İzmir, öğrenci/2, güneş.
let words: &[u8] = b"6\na\xf0a\xe7/1\n\xfd\xfe\xfdk\n\xfei\xfee/2\n\xddzmir\n\xf6\xf0renci/2\ng\xfcne\xfe\n";
std::fs::write(&dic, words).unwrap();
assert!(String::from_utf8(words.to_vec()).is_err(), "the .dic is not UTF-8");
let d = crate::tr::load(&aff, &dic).unwrap();
for w in ["ağaç", "ağaçlar", "ağaçı", "ışık", "şişe", "şişeler", "İzmir", "öğrenci", "öğrenciler", "güneş"] {
assert!(d.contains(w), "{w}");
}
// The six letters ISO 8859-9 puts in place of ISO 8859-1's aren't read as those.
assert!(!d.contains("aðaç") && !d.contains("ýþýk") && !d.contains("Ýzmir"));
assert!(!d.contains("ağaçs") && !d.contains("ışıklar") && !d.contains("şişelar"), "only the rules' suffixes, on the stems that take them");
let _ = std::fs::remove_dir_all(&dir);
}
}
crates/proof/src/hyphen.rs
+27 -3
View File
@@ -320,10 +320,14 @@ pub fn hyphenate_word(word: &str, lim: &Limits) -> String {
/// `word`, a word in `lang`, with `-` at every hyphenation point (tests and diagnostics).
pub fn hyphenate_word_in(word: &str, lim: &Limits, lang: ProofLang) -> String {
let pts = hyphen_points_in(word, lim, lang);
let mut s = String::with_capacity(word.len() + pts.len());
let mut pts = hyphen_points_in(word, lim, lang);
pts.sort_unstable();
// Walk the points alongside the chars rather than searching them for each char.
let mut next = pts.into_iter().peekable();
let mut s = String::with_capacity(word.len() + 8);
for (i, ch) in word.chars().enumerate() {
if pts.contains(&i) {
if next.next_if_eq(&i).is_some() {
while next.next_if_eq(&i).is_some() {}
s.push('-');
}
s.push(ch);
@@ -706,6 +710,26 @@ mod tests {
}
}
#[test]
fn russian_hyphenation_and_hyphenated_output_are_linear() {
// A Russian word breaks at every syllable: each break used to be checked against all the
// earlier ones (#3657), and the hyphenated output searched every break for each char.
let word = |k: usize| "мама".repeat(k);
assert_eq!(word_breaks_in(&word(2), ProofLang::Ru).0, [2, 4, 6]);
assert_eq!(hyphenate_word_in(&word(2), &Limits::default(), ProofLang::Ru), "ма-ма-ма-ма");
// The breaks' rescan is a cheap one per break: it takes longer words to stand out.
for (what, k, repeat, f) in [
("points", 1000, 3, (|w: &str| drop(hyphen_points_in(w, &Limits::default(), ProofLang::Ru))) as fn(&str)),
("hyphenated", 400, 20, |w: &str| drop(hyphenate_word_in(w, &Limits::default(), ProofLang::Ru))),
] {
let (small, big) = (word(k), word(8 * k));
let time = |w: &str| (0..repeat).for_each(|_| f(w));
let (t1, t8, ratio) = cost_ratio(8, || time(&small), || time(&big));
eprintln!("{what}: t(n) {t1:.4} s, t(8n) {t8:.4} s, ratio {ratio:.1}");
assert!(ratio < 24.0, "8x the letters, not linear: {what} {ratio:.1}");
}
}
#[test]
fn heuristic_fallback() {
let c: Vec<char> = "happen".chars().collect();
crates/proof/src/ru/hyphen.rs
+3 -1
View File
@@ -57,7 +57,9 @@ pub fn points(c: &[char]) -> Vec<usize> {
while c.get(cut).is_some_and(|&x| sticks_left(x)) && cut < end {
cut += 1;
}
if cut >= 2 && n - cut >= 2 && !out.contains(&cut) {
// Cuts come in increasing order (each lies after the previous pair's vowel), so a repeat
// can only be the last one: no scan of the whole list per syllable (#3657).
if cut >= 2 && n - cut >= 2 && out.last().is_none_or(|&l| l < cut) {
out.push(cut);
}
}
crates/proof/src/tr/dict.rs
+2 -1
View File
@@ -87,7 +87,8 @@ impl TrDict {
/// Check if word is correct (stem or affixed).
pub fn contains(&self, word: &str) -> bool {
if self.is_empty() || word.is_empty() {
// Longer input is never a dictionary word, and isn't scanned (#3625).
if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES {
return false;
}
crates/proof/src/tr/mod.rs
+11 -12
View File
@@ -44,18 +44,17 @@ fn load_installed() -> TrDict {
log::info!("Turkish spelling: no Hunspell tr_TR dictionary installed");
return TrDict::default();
};
let read = |p: &std::path::Path| std::fs::read_to_string(p);
match (read(&aff), read(&dic)) {
(Ok(aff_text), Ok(dic_text)) => crate::hunspell::note_load(NAMES[0], &dic, TrDict::from_hunspell(&aff_text, &dic_text)).unwrap_or_else(|e| {
log::warn!("Turkish spelling: {e}");
TrDict::default()
}),
(a, d) => {
let reason = a.err().or(d.err()).map_or_else(String::new, |e| e.to_string());
let _ = crate::hunspell::note_load::<()>(NAMES[0], &dic, Err(reason));
TrDict::default()
}
}
crate::hunspell::note_load(NAMES[0], &dic, load(&aff, &dic)).unwrap_or_else(|e| {
log::warn!("Turkish spelling: {e}");
TrDict::default()
})
}
/// Read a Turkish Hunspell pair from disk: size-capped and decoded per the `.aff`'s `SET` line,
/// like the other languages' dictionaries (#3661).
pub(crate) fn load(aff: &std::path::Path, dic: &std::path::Path) -> Result<TrDict, String> {
let (aff_text, dic_text) = crate::hunspell::decode(&crate::hunspell::read_capped(aff)?, &crate::hunspell::read_capped(dic)?)?;
TrDict::from_hunspell(&aff_text, &dic_text)
}
/// Use this Turkish dictionary from now on.
crates/proof/src/tr/suggest.rs
+34 -1
View File
@@ -46,6 +46,12 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[
("deyil", "değil"),
];
/// Longest word (in letters) given suggestions. Every candidate is a fresh copy of the word, so
/// the cost grows with the square of its length (#3625); Turkish words run longer than most
/// (`Çekoslovakyalılaştıramadıklarımızdanmışsınız` is 43 letters), so the cap is above the other
/// languages' 40.
const MAX_LETTERS: usize = 64;
/// Produce spelling suggestions for a misspelled Turkish word.
pub fn suggest(word: &str, max: usize) -> Vec<String> {
let dict = super::dictionary();
@@ -54,7 +60,8 @@ pub fn suggest(word: &str, max: usize) -> Vec<String> {
/// Suggest with a given dictionary.
pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
if word.is_empty() || max == 0 {
let n = word.chars().count();
if n == 0 || n > MAX_LETTERS || max == 0 {
return Vec::new();
}
@@ -72,6 +79,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
}
}
}
// Only the table takes more than one word (`bir kaç`).
if word.contains(char::is_whitespace) {
return suggestions;
}
// 2. Diacritic substitutions (`c` -> `ç`, `g` -> `ğ`, `ı` -> `i`, `o` -> `ö`, `s` -> `ş`, `u` -> `ü`)
if !d.is_empty() {
@@ -277,6 +288,28 @@ mod tests {
assert!(sugs.contains(&"geldin mi".to_string()));
}
#[test]
fn overlong_words_get_no_suggestions_promptly() {
let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();
assert!(suggest_with(&d, "sorna", 3).contains(&"sonra".to_string()));
assert!(suggest_with(&d, "kitaplr", 3).contains(&"kitaplar".to_string()));
let t = std::time::Instant::now();
let long = "a".repeat(100_000);
assert!(suggest_with(&d, &long, 8).is_empty());
assert!(!d.contains(&long) && !d.contains(&format!("kitap{long}")));
assert!(!super::super::is_correct_with(&d, &long));
// Tens of milliseconds with the caps; without them this ran for over 5 minutes.
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
}
#[test]
fn two_words_get_only_the_tables_replacement() {
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap();
assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]);
assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]);
assert!(suggest_with(&d, "sorna sorna", 3).is_empty());
}
#[test]
fn turkish_to_lower_handles_i() {
assert_eq!(turkish_to_lower("IŞIK"), "ışık");
crates/render/src/color_glyph.rs
+36 -10
View File
@@ -23,7 +23,7 @@ pub(crate) enum Drawable {
impl Drawable {
fn bytes(&self) -> usize {
match self {
Drawable::Layers(ops) => ops.len().saturating_mul(64),
Drawable::Layers(ops) => wordcraft_fonts::color::ops_bytes(ops),
Drawable::Bitmap { pixmap, .. } => pixmap.width() as usize * pixmap.height() as usize * 4,
}
}
@@ -102,18 +102,11 @@ fn decode_png(png: &[u8], w: u32, h: u32) -> Option<Pixmap> {
if w == 0 || h == 0 || w > side || h > side {
return None;
}
let reader = || image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png);
let fits = |(iw, ih): (u32, u32)| iw > 0 && ih > 0 && iw <= w && ih <= h;
if !fits(reader().into_dimensions().ok()?) {
if !fits(glyph_png_reader(png, w, h).into_dimensions().ok()?) {
return None;
}
let mut limits = image::Limits::default();
limits.max_image_width = Some(w);
limits.max_image_height = Some(h);
limits.max_alloc = Some(w as u64 * h as u64 * 4 * 2 + 64 * 1024);
let mut r = reader();
r.limits(limits);
let img = r.decode().ok()?.to_rgba8();
let img = glyph_png_reader(png, w, h).decode().ok()?.to_rgba8();
let (iw, ih) = img.dimensions();
if !fits((iw, ih)) {
return None;
@@ -121,6 +114,19 @@ fn decode_png(png: &[u8], w: u32, h: u32) -> Option<Pixmap> {
crate::premul_pixmap(iw, ih, img.as_raw())
}
/// A reader for a bitmap glyph's PNG with the glyph's budget: at most `w`×`h` and about two RGBA
/// buffers of it. Every read goes through it, the header probe too: reading the header inflates
/// metadata such as an ICC profile, which the default budget would let run to hundreds of MB.
fn glyph_png_reader(png: &[u8], w: u32, h: u32) -> image::ImageReader<std::io::Cursor<&[u8]>> {
let mut limits = image::Limits::default();
limits.max_image_width = Some(w);
limits.max_image_height = Some(h);
limits.max_alloc = Some(u64::from(w).saturating_mul(u64::from(h)).saturating_mul(8).saturating_add(64 * 1024));
let mut r = image::ImageReader::with_format(std::io::Cursor::new(png), image::ImageFormat::Png);
r.limits(limits);
r
}
/// A pixmap from premultiplied BGRA rows.
fn bgra_pixmap(bgra: &[u8], w: u32, h: u32) -> Option<Pixmap> {
if w == 0 || h == 0 || w > u16::MAX as u32 || h > u16::MAX as u32 {
@@ -436,4 +442,24 @@ mod tests {
let side = wordcraft_fonts::color::MAX_BITMAP_SIDE;
assert!(decode_png(&png(4, 4), side + 1, 4).is_none(), "past the cap");
}
/// A tiny bitmap glyph whose PNG carries a big ICC profile (it compresses to almost nothing):
/// the header probe already inflates it, so the probe is held to the glyph's budget too, not
/// the decoder's default hundreds of MB. Past the budget the profile is dropped (the glyph is
/// still drawn, without it).
#[test]
fn bitmap_png_header_probe_keeps_the_glyph_budget() {
use image::ImageDecoder;
let mut out = vec![];
let mut enc = image::codecs::png::PngEncoder::new(&mut out);
image::ImageEncoder::set_icc_profile(&mut enc, vec![0u8; 4 << 20]).unwrap();
image::ImageEncoder::write_image(enc, &[200, 0, 0, 255], 1, 1, image::ExtendedColorType::Rgba8).unwrap();
assert!(out.len() < 64 * 1024, "the profile compresses: {} bytes", out.len());
let profile = |r: image::ImageReader<std::io::Cursor<&[u8]>>| r.into_decoder().unwrap().icc_profile().unwrap().map(|p| p.len());
let unbounded = image::ImageReader::with_format(std::io::Cursor::new(out.as_slice()), image::ImageFormat::Png);
assert_eq!(profile(unbounded), Some(4 << 20), "the default budget inflates the 4 MB profile");
assert_eq!(profile(glyph_png_reader(&out, 1, 1)), None, "the glyph budget stops it at the header");
assert_eq!(decode_png(&out, 1, 1).map(|p| (p.width(), p.height())), Some((1, 1)));
assert_eq!(glyph_png_reader(&png(4, 4), 4, 4).into_dimensions().ok(), Some((4, 4)), "a plain glyph reads");
}
}
crates/render/src/lib.rs
+223 -20
View File
@@ -57,15 +57,21 @@ impl Rendered {
}
out
}
/// Whether the pixels are exactly `width`×`height` RGBA, within a page raster's budget
/// ([`wordcraft_geom::MAX_RASTER_PIXELS`]): the fields are public, so an encoder is never asked
/// to pad a short buffer out to a huge claimed size (or, for JPEG, to assert on a mismatch).
fn encodable(&self) -> bool {
let budget = wordcraft_geom::MAX_RASTER_PIXELS.saturating_mul(4);
wordcraft_geom::rgba_fits(self.width, self.height, budget)
&& wordcraft_geom::rgba_bytes(self.width, self.height).is_some_and(|b| b == self.pixels.len() as u64)
}
/// PNG bytes (empty on failure, logged).
pub fn to_png(&self) -> Vec<u8> {
let mut px = self.to_straight();
let Some(len) = wordcraft_geom::rgba_bytes(self.width, self.height).and_then(|b| usize::try_from(b).ok()) else {
log::error!("PNG encode: size {}×{} overflows", self.width, self.height);
if !self.encodable() {
log::error!("PNG encode: {} bytes for {}×{} pixels", self.pixels.len(), self.width, self.height);
return Vec::new();
};
px.resize(len, 0);
let Some(img) = image::RgbaImage::from_raw(self.width, self.height, px) else {
}
let Some(img) = image::RgbaImage::from_raw(self.width, self.height, self.to_straight()) else {
log::error!("PNG encode: bad size {}×{}", self.width, self.height);
return Vec::new();
};
@@ -76,7 +82,12 @@ impl Rendered {
}
buf
}
/// JPEG bytes on white (empty on failure, logged).
pub fn to_jpeg(&self, quality: u8) -> Vec<u8> {
if !self.encodable() {
log::error!("JPEG encode: {} bytes for {}×{} pixels", self.pixels.len(), self.width, self.height);
return Vec::new();
}
let rgba = self.to_straight();
let rgb: Vec<u8> = rgba
.as_chunks::<4>()
@@ -209,17 +220,30 @@ pub fn decode_pixmap(bytes: &[u8]) -> Option<Pixmap> {
/// [`decode_pixmap`] with an explicit RGBA byte budget (a small one in tests).
fn decode_pixmap_within(bytes: &[u8], budget: u64) -> Option<Pixmap> {
// The header gives the size; refuse before the decoder (and the RGBA copy) allocate anything.
let (hw, hh) = image_size(bytes)?;
if !wordcraft_geom::decode_fits(hw, hh, budget) {
return None;
}
let mut reader = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?;
use image::ImageDecoder;
let mut limits = image::Limits::default();
limits.max_image_width = Some(MAX_DECODE_SIDE);
limits.max_image_height = Some(MAX_DECODE_SIDE);
reader.limits(limits);
let img = reader.decode().ok()?.into_rgba8();
let reader = || {
let mut r = image::ImageReader::new(std::io::Cursor::new(bytes)).with_guessed_format().ok()?;
r.limits(limits.clone());
Some(r)
};
// The header gives the size and the decoder's pixel format (16-bit RGBA takes 8 bytes a pixel
// before its RGBA8 copy); refuse before the decoder (and the RGBA copy) allocate anything.
let (hw, hh, native) = {
let d = reader()?.into_decoder().ok()?;
let (w, h) = d.dimensions();
(w, h, u64::from(d.color_type().bytes_per_pixel()))
};
if !wordcraft_geom::decode_peak_fits(hw, hh, native, budget) {
// Logged once a picture: its refusal is cached with it (`pixmap_for`).
log::warn!(
"a {hw}×{hh} picture of {native} bytes a pixel needs more than {budget} bytes to decode: it is not drawn (in the text, a grey box takes its place)"
);
return None;
}
let img = reader()?.decode().ok()?.into_rgba8();
let (w, h) = img.dimensions();
premul_pixmap(w, h, img.as_raw())
}
@@ -390,11 +414,46 @@ fn draw_vector_text(ctx: &mut RenderContext, t: &wordcraft_metafile::Text, tf: A
let k = lay.em / face.upem.max(1.0);
ctx.set_paint(rgba_color(t.color));
let db = FontDb::global();
let canvas = kurbo::Rect::new(0.0, 0.0, f64::from(ctx.width()), f64::from(ctx.height()));
let mut oversized = 0usize;
for (g, (x, y)) in shaped.iter().zip(&lay.glyphs) {
let o = db.outline(&face, g.gid);
ctx.set_transform(tf * Affine::translate((*x, *y)) * Affine::scale(k));
let m = tf * Affine::translate((*x, *y)) * Affine::scale(k);
// A record's font size is file data: a glyph many canvases tall (or off the canvas) would
// have its outline flattened into millions of segments for nothing, so it is left out.
let b = m.transform_rect_bbox(o.bounding_box());
let sane = [b.x0, b.y0, b.x1, b.y1].iter().all(|v| v.is_finite()) && b.width() <= MAX_GLYPH_PIXELS && b.height() <= MAX_GLYPH_PIXELS;
if !sane {
oversized = oversized.saturating_add(1);
continue;
}
if !b.overlaps(canvas) {
continue;
}
ctx.set_transform(m);
ctx.fill_path(&o);
}
note_oversized_glyphs(&OVERSIZED_GLYPHS_NOTED, oversized);
}
/// Largest metafile glyph drawn, in device pixels per side (four times the widest canvas).
const MAX_GLYPH_PIXELS: f64 = 65_536.0;
/// Whether metafile glyphs left out for their size have been logged ([`note_oversized_glyphs`]).
static OVERSIZED_GLYPHS_NOTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
/// Logs that `n` glyphs of a metafile text run were left out for being larger than
/// [`MAX_GLYPH_PIXELS`] (or of no finite size), the first time any are this run (`noted`): a
/// page is drawn again on every repaint, so a warning each time would flood the log. True when
/// this call logged it.
fn note_oversized_glyphs(noted: &std::sync::atomic::AtomicBool, n: usize) -> bool {
if n == 0 || noted.swap(true, std::sync::atomic::Ordering::Relaxed) {
return false;
}
log::warn!(
"a picture's text has {n} letter(s) larger than {MAX_GLYPH_PIXELS} pixels drawn, left out (logged once a run; later ones are left out too)"
);
true
}
/// One device pixel in page points under `view` (its mean axis scale); 1 for a degenerate transform.
@@ -868,6 +927,44 @@ fn wave_path(x0: f64, y0: f64, x1: f64, y1: f64, amp: f64) -> BezPath {
p
}
/// Most pieces one dashed stroke may be cut into; past it the stroke is drawn solid. kurbo's dash
/// iterator makes no progress along a segment when a dash is too small next to it (it never ends),
/// and every piece is more stroke to build.
const MAX_DASH_PIECES: f64 = 100_000.0;
/// `dashes` to stroke a path at most `len` long with, or none (solid) when they can't be drawn: a
/// negative or non-finite length, a pattern that adds up to nothing, or more than
/// [`MAX_DASH_PIECES`] pieces along the path.
fn safe_dashes(dashes: impl IntoIterator<Item = f64>, len: f64) -> Vec<f64> {
let d: Vec<f64> = dashes.into_iter().take(MAX_DASH_PIECES as usize + 1).collect();
let period: f64 = d.iter().sum();
let ok = d.iter().all(|v| v.is_finite() && *v >= 0.0) && period > 0.0 && period.is_finite() && len.is_finite();
// Each repeat of the pattern is up to `d.len()` pieces; a partial repeat at each end counts too.
if ok && (len / period + 2.0) * d.len() as f64 <= MAX_DASH_PIECES { d } else { Vec::new() }
}
/// An upper bound on `path`'s length: its control polygon's (a curve is no longer than it).
fn path_len_bound(path: &BezPath) -> f64 {
let (mut len, mut at, mut start) = (0.0, kurbo::Point::ZERO, kurbo::Point::ZERO);
for el in path.elements() {
let pts: &[kurbo::Point] = match el {
kurbo::PathEl::MoveTo(p) => {
(at, start) = (*p, *p);
continue;
}
kurbo::PathEl::LineTo(p) => std::slice::from_ref(p),
kurbo::PathEl::QuadTo(a, b) => &[*a, *b],
kurbo::PathEl::CurveTo(a, b, c) => &[*a, *b, *c],
kurbo::PathEl::ClosePath => std::slice::from_ref(&start),
};
for p in pts {
len += at.distance(*p);
at = *p;
}
}
len
}
/// Rasterize one display item with `view` mapping its current point coordinates to pixels.
/// `visible` is a culling area in those point coordinates; nested clips intersect it and turns
/// transform it into the child's coordinates. Shape and path strokes apply their stored caps,
@@ -896,10 +993,10 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl
// every end, blobbing over the neighbouring border at a table's corners (#2892). Wavy
// underlines keep round ends, as the PDF draws them.
let mut st = kurbo::Stroke::new(w).with_caps(kurbo::Cap::Butt);
if let Some(d) = stroke.dashes() {
st = st.with_dashes(0.0, d.iter().map(|k| f64::from(*k) * w));
}
let (x0, y0, x1, y1) = (*x0 as f64, *y0 as f64, *x1 as f64, *y1 as f64);
if let Some(d) = stroke.dashes() {
st = st.with_dashes(0.0, safe_dashes(d.iter().map(|k| f64::from(*k) * w), (x1 - x0).hypot(y1 - y0)));
}
let line = |ctx: &mut RenderContext, ox: f64, oy: f64| {
let mut p = BezPath::new();
p.move_to((x0 + ox, y0 + oy));
@@ -1102,7 +1199,11 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl
// Width 0 (or none) is a hairline.
let w = if stroke_width.is_finite() && *stroke_width > 0.0 { stroke_width.clamp(0.25, 200.0) } else { 0.75 };
ctx.set_paint(color(opts.ink(*s), 1.0));
ctx.set_stroke(kurbo::Stroke::new(w as f64).with_caps(screen_cap(*cap)).with_dashes(0.0, dash.iter().map(|v| f64::from(*v))));
ctx.set_stroke(
kurbo::Stroke::new(w as f64)
.with_caps(screen_cap(*cap))
.with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path))),
);
ctx.stroke_path(&path);
}
}
@@ -1193,7 +1294,7 @@ fn draw(ctx: &mut RenderContext, doc: &Document, it: &Draw, view: Affine, visibl
ctx.set_stroke(
kurbo::Stroke::new(if stroke_width.is_nan() { 0.25 } else { stroke_width.clamp(0.25, 200.0) } as f64)
.with_caps(screen_cap(*cap))
.with_dashes(0.0, dash.iter().map(|v| f64::from(*v))),
.with_dashes(0.0, safe_dashes(dash.iter().map(|v| f64::from(*v)), path_len_bound(&path))),
);
ctx.stroke_path(&path);
}
@@ -1815,12 +1916,36 @@ mod tests {
// Decode, the RGBA conversion and the premultiplied copy are charged: two buffers at peak.
assert!(decode_pixmap_within(&buf, 1024).is_some());
assert!(decode_pixmap_within(&buf, 1023).is_none());
// 16-bit RGBA decodes to 8 bytes a pixel, alive with its RGBA8 copy: 12 bytes a pixel.
let mut wide = Vec::new();
let img16 = image::ImageBuffer::<image::Rgba<u16>, _>::from_fn(16, 8, |x, y| image::Rgba([x as u16 * 999, y as u16, 9, 65535]));
image::DynamicImage::ImageRgba16(img16).write_to(&mut std::io::Cursor::new(&mut wide), image::ImageFormat::Png).unwrap();
assert!(decode_pixmap_within(&wide, 1024).is_none(), "two RGBA8 buffers' worth is not enough for RGBA16");
assert!(decode_pixmap_within(&wide, 1536).is_some());
// A header (valid CRC) claiming 60000x60000 with no pixel data reads fine, then the budget refuses it.
let hdr = ihdr_only_png(60_000, 60_000);
assert_eq!(image_size(&hdr), Some((60_000, 60_000)));
assert!(decode_pixmap(&hdr).is_none());
}
/// A `Rendered` whose pixels don't match its claimed size isn't padded out to that size: a
/// 60000×60000 claim over no pixels would allocate 14 GB. It, and any mismatch, encodes to
/// nothing (JPEG used to assert on the length).
#[test]
fn encoders_refuse_pixels_that_dont_match_the_size() {
let huge = Rendered { width: 60_000, height: 60_000, pixels: Vec::new() };
assert!(huge.to_png().is_empty());
assert!(huge.to_jpeg(80).is_empty());
let short = Rendered { width: 2, height: 2, pixels: vec![255; 4] };
assert!(short.to_png().is_empty());
assert!(short.to_jpeg(80).is_empty());
let long = Rendered { width: 1, height: 1, pixels: vec![255; 8] };
assert!(long.to_png().is_empty());
let ok = Rendered { width: 2, height: 2, pixels: vec![255; 16] };
assert!(!ok.to_png().is_empty());
assert!(!ok.to_jpeg(80).is_empty());
}
#[test]
fn page_raster_is_capped_by_total_pixels() {
let d = Document::new();
@@ -2431,6 +2556,56 @@ mod tests {
assert!(dark > 50, "the text is painted inside the picture: {dark} dark pixels");
}
/// An EMF text run whose anchor sits inside the picture but whose font is billions of units
/// tall: each glyph's curves would span billions of device pixels, which the rasteriser
/// subdivides for ever. Glyphs that big (or off the raster) are left out; the page renders.
#[test]
fn metafile_text_with_a_huge_font_is_left_out() {
let mut font = emf_rec(82, &le32(&[1, -2_000_000_000, 0, 0, 0, 400, 0, 0]));
let mut name: Vec<u8> = "Arial".encode_utf16().flat_map(|u| u.to_le_bytes()).collect();
name.resize(64, 0);
font.extend(name);
font[4..8].copy_from_slice(&104u32.to_le_bytes());
let mut text = emf_rec(84, &le32(&[0, 0, 100, 100, 1, 0, 0, 5, 40, 4, 76, 0, 0, 0, 0, 0, 0]));
text.extend("WOWS".encode_utf16().flat_map(|u| u.to_le_bytes()));
text[4..8].copy_from_slice(&84u32.to_le_bytes());
let mut emf = red_right_half_emf()[..88].to_vec();
emf.extend([font, emf_rec(37, &le32(&[1])), emf_rec(24, &le32(&[0])), text, emf_rec(14, &[0; 12])].concat());
let mut d = Document::new();
let media = d.add_media(emf, "emf");
let mut p = wordcraft_doc::Paragraph::new();
let obj = wordcraft_doc::para::InlineObject::Image {
media,
w: 70.0,
h: 70.0,
alt: String::new(),
float: Default::default(),
crop: [0.0; 4],
ole: None,
};
p.insert_object(0, obj, &Default::default()).unwrap();
d.body = vec![wordcraft_doc::para_block(p)];
let l = layout(&d, &mut LayoutCache::new(), &LayoutOptions::default());
let start = std::time::Instant::now();
let img = render_page(&d, &l.pages[0], 1.0, &RenderOptions::default());
// About 20 ms without the huge glyphs; drawing them ran past 4 minutes without finishing.
assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed());
assert_eq!(img.width, 612);
// Leaving them out is logged (once a run, so it may have been by another test already).
assert!(OVERSIZED_GLYPHS_NOTED.load(std::sync::atomic::Ordering::Relaxed), "the left-out glyphs are logged");
}
/// Glyphs left out for their size are logged the first time only: a page is drawn on every
/// repaint. A run with none left out logs nothing and doesn't use up the warning.
#[test]
fn oversized_glyphs_are_logged_once_a_run() {
let noted = std::sync::atomic::AtomicBool::new(false);
assert!(!note_oversized_glyphs(&noted, 0), "nothing left out, nothing logged");
assert!(!noted.load(std::sync::atomic::Ordering::Relaxed));
assert!(note_oversized_glyphs(&noted, 4), "the first glyphs left out are logged");
assert!(!note_oversized_glyphs(&noted, 2), "later ones aren't");
}
#[test]
fn unreadable_metafile_keeps_the_grey_box() {
let mut d = Document::new();
@@ -2583,6 +2758,34 @@ mod tests {
assert_eq!(image.pixel(832, 800), [255, 255, 255, 255], "inside the first gap");
}
/// Dash lengths too small to cut the path make kurbo's dash iterator spin without moving along
/// it (and every piece is more stroke to build): such a pattern is drawn solid, as are negative
/// and non-finite ones. A normal dash pattern still dashes.
#[test]
fn hostile_dash_patterns_draw_solid_and_finish() {
use wordcraft_doc::graphic::PathSeg;
let line = |dash: Vec<f32>| Draw::Path {
segs: vec![PathSeg::Move(10.0, 20.0), PathSeg::Line(90.0, 20.0)],
fill: None,
stroke: Some(Rgb(0, 0, 0)),
stroke_width: 2.0,
dash,
cap: Default::default(),
};
let start = std::time::Instant::now();
for dash in [vec![1e-30, 1e-30], vec![1e-30], vec![-1.0, 2.0], vec![f32::NAN, 1.0], vec![f32::INFINITY, 1.0], vec![0.0, 0.0]] {
let img = render_items(&Document::new(), &[line(dash.clone())], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default());
assert!(img.pixel(50, 20)[3] > 200, "{dash:?} draws solid");
}
// Milliseconds drawn solid; drawn as dashes they ran past 4 minutes without finishing.
assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed());
let dashed = render_items(&Document::new(), &[line(vec![10.0, 10.0])], 0.0, 0.0, 100.0, 40.0, 1.0, &RenderOptions::default());
assert!(dashed.pixel(15, 20)[3] > 200 && dashed.pixel(25, 20)[3] == 0, "a normal pattern dashes");
assert!(safe_dashes([3.0, 1.0], 1000.0).len() == 2);
assert!(safe_dashes([1e-3, 1e-3], 1000.0).is_empty(), "a million pieces");
assert!(safe_dashes(std::iter::repeat_n(1.0, 200_000), 1.0).is_empty(), "too many entries");
}
#[test]
fn flowchart_or_spokes() {
let r = kurbo::Rect::new(0.0, 0.0, 10.0, 10.0);
crates/zotero/src/bridge.rs
+62 -15
View File
@@ -5,6 +5,8 @@
//! Field ids handed to Zotero stay valid for the transaction: they are kept in a list parallel
//! to the document-order list of Zotero fields, updated as fields are inserted and removed.
use std::collections::HashMap;
use serde_json::{Value, json};
use wordcraft_doc::edit::Fragment;
use wordcraft_doc::para::OBJ;
@@ -192,6 +194,7 @@ impl Bridge {
}
}
/// Answer one call by its method name; [`Bridge::handle`] wraps it so a panic becomes an error.
fn dispatch(&mut self, s: &mut Session, host: &mut dyn Host, c: &Call) -> R {
match c.method.as_str() {
"Application_getActiveDocument" => {
@@ -274,21 +277,8 @@ impl Bridge {
// citation moves between the text, footnotes and endnotes (switching styles).
"Document_convert" => {
let Value::Array(ids) = arg(c, 1)? else { return Err("field ids must be an array".into()) };
let ids: Vec<Value> = ids.iter().take(100_000).cloned().collect();
let notes = c.args.get(3).cloned().unwrap_or(Value::Null);
for (i, idv) in ids.iter().enumerate() {
let id = match idv {
Value::Number(n) => n.as_u64(),
Value::String(x) => x.trim().parse().ok(),
_ => None,
}
.ok_or("bad field id")?;
let note = match &notes {
Value::Array(a) => a.get(i).and_then(Value::as_i64).unwrap_or(0),
v => v.as_i64().unwrap_or(0),
};
self.convert_field(s, id, note)?;
}
let notes = c.args.get(3).unwrap_or(&Value::Null);
self.convert_fields(s, ids, notes)?;
Ok(Value::Null)
}
"Field_convert" => {
@@ -484,6 +474,49 @@ impl Bridge {
Ok(())
}
/// `Document_convert`: each id once (listed again, its last note type wins), every id
/// checked against one indexed list of the fields before anything moves. Only a citation
/// that actually moves costs a fresh list.
fn convert_fields(&mut self, s: &mut Session, ids: &[Value], notes: &Value) -> Result<(), String> {
let mut todo: Vec<(u64, i64)> = Vec::new();
let mut slot: HashMap<u64, usize> = HashMap::new();
for (i, idv) in ids.iter().take(fields::MAX_FIELDS).enumerate() {
let id = match idv {
Value::Number(n) => n.as_u64(),
Value::String(x) => x.trim().parse().ok(),
_ => None,
}
.ok_or("bad field id")?;
let note = match notes {
Value::Array(a) => a.get(i).and_then(Value::as_i64).unwrap_or(0),
v => v.as_i64().unwrap_or(0),
};
match slot.get(&id).and_then(|k| todo.get_mut(*k)) {
Some(t) => t.1 = note,
None => {
slot.insert(id, todo.len());
todo.push((id, note));
}
}
}
let index = |ids: &[u64]| -> HashMap<u64, usize> { ids.iter().enumerate().map(|(k, id)| (*id, k)).collect() };
let mut list = self.fields(s);
let mut at = index(&self.ids);
if let Some((id, _)) = todo.iter().find(|(id, _)| !at.contains_key(id)) {
return Err(format!("no field with id {id}"));
}
for (id, note) in todo {
let f = at.get(&id).and_then(|k| list.get(*k)).ok_or_else(|| format!("no field with id {id}"))?;
if !moves(s, f, note) {
continue;
}
self.convert_field(s, id, note)?;
list = self.fields(s);
at = index(&self.ids);
}
Ok(())
}
fn convert_placeholders(&mut self, s: &mut Session, ph: &[Value], note: i64) -> R {
let mut ids = Vec::new();
for v in ph {
@@ -519,6 +552,20 @@ impl Bridge {
}
}
/// Whether moving `f` to the text (`note` 0), a footnote (1) or an endnote (2) changes anything,
/// as [`Bridge::convert_field`] decides it.
fn moves(s: &Session, f: &ZField, note: i64) -> bool {
let current = match f.start.story {
StoryRef::Body => 0,
StoryRef::Part(pid) => match s.doc.parts.get(&pid).map(|p| p.kind) {
Some(PartKind::Footnote) => 1,
Some(PartKind::Endnote) => 2,
_ => return false,
},
};
current != if matches!(note, 1 | 2) { note } else { 0 }
}
/// Delete the selection; returns where it collapsed.
fn delete_selection(s: &mut Session) -> Result<Pos, String> {
let (a, b) = s.sel.ordered();
crates/zotero/src/client.rs
+4
View File
@@ -88,7 +88,11 @@ pub struct Outcome {
/// Run `command`, answering every call with `answer`.
pub fn run_command(opts: &Options, command: Command, answer: &mut dyn FnMut(&Call) -> Result<Value, String>) -> Result<Outcome, ZoteroError> {
crate::peer::check_addr(&opts.addr)?;
let mut stream = TcpStream::connect_timeout(&opts.addr, opts.connect_timeout).map_err(|e| ZoteroError::NotRunning(e.to_string()))?;
// Before Zotero sees anything: the command, and then the document, go only to this computer
// and, on Linux only, not to a listener the socket table shows belongs to another user.
crate::peer::check(&stream)?;
let _ = stream.set_nodelay(true);
stream.set_read_timeout(Some(opts.idle_timeout)).map_err(|e| ZoteroError::Io(e.to_string()))?;
wire::write_frame(&mut stream, 0, &wire::command_payload(command.wire_name()))?;
crates/zotero/src/fields.rs
+30 -6
View File
@@ -63,7 +63,14 @@ fn story_fields(doc: &Document, story: StoryRef, note_index: u32) -> Vec<ZField>
.collect()
}
/// Most fields [`list`] hands Zotero (no real document comes near it).
pub const MAX_FIELDS: usize = 100_000;
/// Most bytes of field codes [`list`] hands Zotero, all fields together.
pub const MAX_CODE_BYTES: usize = 64 * 1024 * 1024;
/// Every Zotero field in document order: the body's, with each note's fields at its reference.
/// A note the body references more than once (a crafted file) is listed at its first reference
/// only. Past [`MAX_FIELDS`] fields or [`MAX_CODE_BYTES`] of codes the rest are left out.
pub fn list(doc: &Document) -> Vec<ZField> {
enum Ev {
Field(ZField),
@@ -71,10 +78,12 @@ pub fn list(doc: &Document) -> Vec<ZField> {
}
let mut evs: Vec<(Pos, Ev)> = story_fields(doc, StoryRef::Body, 0).into_iter().map(|f| (f.start.clone(), Ev::Field(f))).collect();
let (mut foot, mut end) = (0u32, 0u32);
let mut seen = std::collections::HashSet::new();
for path in doc.para_paths(StoryRef::Body) {
let Some(p) = doc.para(StoryRef::Body, &path) else { continue };
for off in p.object_offsets() {
if let Some(InlineObject::NoteRef { kind, id, custom }) = p.object_at(off) {
// The k-th object marker holds the k-th object.
for (off, obj) in p.object_offsets().into_iter().zip(&p.objects) {
if let InlineObject::NoteRef { kind, id, custom } = obj {
let n = match kind {
NoteKind::Footnote => &mut foot,
NoteKind::Endnote => &mut end,
@@ -82,16 +91,31 @@ pub fn list(doc: &Document) -> Vec<ZField> {
if custom.is_empty() {
*n = n.saturating_add(1);
}
evs.push((Pos::new(StoryRef::Body, path.clone(), off), Ev::Note(*id, (*n).max(1))));
if seen.insert(*id) {
evs.push((Pos::new(StoryRef::Body, path.clone(), off), Ev::Note(*id, (*n).max(1))));
}
}
}
}
evs.sort_by(|a, b| a.0.cmp(&b.0));
let mut out = Vec::new();
let mut bytes = 0usize;
let mut take = |f: ZField, out: &mut Vec<ZField>| {
bytes = bytes.saturating_add(f.code.len());
let fits = out.len() < MAX_FIELDS && bytes <= MAX_CODE_BYTES;
if fits {
out.push(f);
}
fits
};
for (_, ev) in evs {
match ev {
Ev::Field(f) => out.push(f),
Ev::Note(id, n) => out.extend(story_fields(doc, StoryRef::Part(id), n)),
let fits = match ev {
Ev::Field(f) => take(f, &mut out),
Ev::Note(id, n) => story_fields(doc, StoryRef::Part(id), n).into_iter().all(|f| take(f, &mut out)),
};
if !fits {
log::warn!("Zotero: the document has more citation fields than WordCraft hands Zotero; the rest are left out");
break;
}
}
out
crates/zotero/src/lib.rs
+7
View File
@@ -18,6 +18,7 @@
mod bridge;
pub mod client;
pub mod fields;
mod peer;
mod rich;
pub mod wire;
@@ -37,7 +38,13 @@ pub enum ZoteroError {
/// Zotero sent something that isn't the protocol.
#[error("unexpected data from Zotero: {0}")]
Protocol(String),
/// What answered on Zotero's port may not be Zotero (another computer, on any system; on
/// Linux only, another user's program); it gets no access to the document.
#[error("WordCraft won't let this program edit the document: {0}")]
Refused(String),
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_limits;
crates/zotero/src/peer.rs
+234
View File
@@ -0,0 +1,234 @@
//! Who answered on Zotero's port. Zotero's protocol has no handshake that proves the other end
//! is Zotero, and whatever answers gets to read and rewrite the open document. Two checks:
//!
//! - **Every system:** WordCraft only connects to this computer (a loopback address).
//! - **Linux only:** the program listening there must not run as another user, from the
//! kernel's socket table (`/proc/net/tcp`): another account on a shared machine that took the
//! port while Zotero wasn't running is refused. A listener the table doesn't show (WSL2
//! mirrored networking with Zotero on Windows, a local proxy, a sandbox) leaves only the
//! loopback rule.
//!
//! On Windows, macOS and the BSDs there is **no same-user check**: the loopback rule is the
//! only one, so any local account's listener on the port is taken for Zotero. Those systems have
//! no such table to read without OS calls this crate doesn't make (see `docs/zotero.md`, #3831).
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream};
use crate::ZoteroError;
/// Whether `addr` is on this computer; anything else is refused before connecting.
pub(crate) fn check_addr(addr: &SocketAddr) -> Result<(), ZoteroError> {
if addr.ip().to_canonical().is_loopback() {
Ok(())
} else {
Err(ZoteroError::Refused(format!("{addr} is not on this computer; WordCraft only talks to Zotero on 127.0.0.1")))
}
}
/// Refuse the program at the other end of `stream` when it may not be ours (see the module
/// documentation).
pub(crate) fn check(stream: &TcpStream) -> Result<(), ZoteroError> {
let io = |e: std::io::Error| ZoteroError::Io(e.to_string());
let (local, peer) = (stream.local_addr().map_err(io)?, stream.peer_addr().map_err(io)?);
check_addr(&peer)?;
owner_check(local, peer)
}
/// Refuse a listener at `peer` that the kernel's socket tables say belongs to another user.
#[cfg(any(target_os = "linux", target_os = "android"))]
fn owner_check(local: SocketAddr, peer: SocketAddr) -> Result<(), ZoteroError> {
use std::io::Read;
/// Longest socket table read (a busy server's table is a few MB).
const MAX_TABLE: u64 = 64 * 1024 * 1024;
let mut table = String::new();
for path in ["/proc/net/tcp", "/proc/net/tcp6"] {
if let Ok(f) = std::fs::File::open(path) {
let mut part = String::new();
if f.take(MAX_TABLE).read_to_string(&mut part).is_ok() {
table.push_str(&part);
}
}
}
// Our uid when our own socket's row is missing: `/proc/self` belongs to this process's user.
let me = std::fs::metadata("/proc/self").ok().map(|m| std::os::unix::fs::MetadataExt::uid(&m));
same_owner(&table, local, peer, me)
}
/// Windows, macOS, the BSDs: no same-user check (no socket table to read without OS calls this
/// crate doesn't make, #3831); the loopback rule alone decides.
#[cfg(not(any(target_os = "linux", target_os = "android")))]
fn owner_check(_local: SocketAddr, _peer: SocketAddr) -> Result<(), ZoteroError> {
Ok(())
}
/// From a `/proc/net/tcp`-format socket table: refuse when the socket at `peer` connected to
/// `local` (the listener's side) belongs to another user than ours. Our uid is our own socket's
/// row, else `me` (this process's). When the listener isn't listed (or the table couldn't be
/// read, or our uid is unknown) its owner is unknown and the loopback rule decides.
#[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))]
pub(crate) fn same_owner(table: &str, local: SocketAddr, peer: SocketAddr, me: Option<u32>) -> Result<(), ZoteroError> {
let key = |a: SocketAddr| (a.ip().to_canonical(), a.port());
let (local, peer) = (key(local), key(peer));
let (mut ours, mut theirs) = (None, None);
for line in table.lines() {
let f: Vec<&str> = line.split_whitespace().take(8).collect();
let (Some(l), Some(r), Some(uid)) = (f.get(1).and_then(|a| parse_addr(a)), f.get(2).and_then(|a| parse_addr(a)), f.get(7)) else {
continue;
};
let Ok(uid) = uid.parse::<u32>() else { continue };
let (l, r) = (key(l), key(r));
if l == local && r == peer {
ours = Some(uid);
} else if l == peer && r == local {
theirs = Some(uid);
}
}
match (ours.or(me), theirs) {
(Some(a), Some(b)) if a == b => Ok(()),
(Some(_), Some(b)) => {
Err(ZoteroError::Refused(format!("the program on port {} runs as another user (uid {b}), so it isn't your Zotero", peer.1)))
}
_ => {
log::debug!(
"Zotero: couldn't tell which user runs the program on port {} (not in the socket table); trusting the loopback address alone",
peer.1
);
Ok(())
}
}
}
/// `0100007F:5A5C` (IPv4) or 32 hex digits and a port (IPv6): each 32-bit word as the kernel
/// prints it, in this machine's byte order.
#[cfg_attr(not(any(target_os = "linux", target_os = "android")), allow(dead_code))]
fn parse_addr(s: &str) -> Option<SocketAddr> {
let (ip, port) = s.split_once(':')?;
let port = u16::from_str_radix(port, 16).ok()?;
let word = |k: usize| ip.get(k * 8..k * 8 + 8).and_then(|w| u32::from_str_radix(w, 16).ok()).map(u32::to_ne_bytes);
let ip = match ip.len() {
8 => IpAddr::V4(Ipv4Addr::from(word(0)?)),
32 => {
let mut b = [0u8; 16];
for (k, w) in b.as_chunks_mut::<4>().0.iter_mut().enumerate() {
*w = word(k)?;
}
IpAddr::V6(Ipv6Addr::from(b))
}
_ => return None,
};
Some(SocketAddr::new(ip, port))
}
#[cfg(test)]
mod tests {
use super::*;
/// `a` the way the kernel's socket table prints it.
fn hex(a: SocketAddr) -> String {
let ip = match a.ip() {
IpAddr::V4(v4) => format!("{:08X}", u32::from_ne_bytes(v4.octets())),
IpAddr::V6(v6) => v6.octets().as_chunks::<4>().0.iter().map(|w| format!("{:08X}", u32::from_ne_bytes(*w))).collect(),
};
format!("{ip}:{:04X}", a.port())
}
/// A socket table row for the connection from `l` to `r`, owned by `uid`.
fn row(l: SocketAddr, r: SocketAddr, uid: u32) -> String {
format!(" 0: {} {} 01 00000000:00000000 00:00000000 00000000 {uid:>4} 0 12345 1 0000000000000000 20 4 30 10 -1\n", hex(l), hex(r))
}
const HEADER: &str = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n";
/// Our side and Zotero's listening side of a loopback connection.
fn addrs() -> (SocketAddr, SocketAddr) {
("127.0.0.1:41234".parse().unwrap(), "127.0.0.1:23116".parse().unwrap())
}
#[test]
fn addresses_parse_as_the_kernel_prints_them() {
for a in ["127.0.0.1:23116", "[::1]:23116", "[::ffff:127.0.0.1]:80"] {
let a: SocketAddr = a.parse().unwrap();
assert_eq!(parse_addr(&hex(a)), Some(a));
}
for bad in ["", "0100007F", "0100007F:", "XYZ00000:0050", "0100007F:12345", "0100:0050"] {
assert_eq!(parse_addr(bad), None, "{bad}");
}
}
#[test]
fn a_listener_of_another_user_is_refused() {
let (local, peer) = addrs();
let same = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1000));
assert_eq!(same_owner(&same, local, peer, None), Ok(()));
let other = format!("{HEADER}{}{}", row(local, peer, 1000), row(peer, local, 1001));
assert!(matches!(same_owner(&other, local, peer, None), Err(ZoteroError::Refused(e)) if e.contains("another user")));
// Junk lines are skipped.
let junk = format!("{HEADER}garbage\n 1: zz:zz yy:yy 01 x x x x\n{}{}", row(local, peer, 7), row(peer, local, 7));
assert_eq!(same_owner(&junk, local, peer, None), Ok(()));
}
/// WSL2 mirrored networking (Zotero on Windows), a local proxy or a sandbox: the listener
/// isn't in our table, so the loopback rule decides, as on other systems.
#[test]
fn an_unlisted_listener_falls_back_to_the_loopback_rule() {
let (local, peer) = addrs();
let missing = format!("{HEADER}{}", row(local, peer, 1000));
assert_eq!(same_owner(&missing, local, peer, None), Ok(()));
// The table couldn't be read.
assert_eq!(same_owner("", local, peer, None), Ok(()));
assert_eq!(same_owner("", local, peer, Some(1000)), Ok(()));
}
/// Our own socket's row missing doesn't hide another user's listener: our uid comes from
/// the process itself.
#[test]
fn a_listed_listener_is_checked_against_our_process_uid() {
let (local, peer) = addrs();
let only_theirs = format!("{HEADER}{}", row(peer, local, 1001));
assert!(matches!(same_owner(&only_theirs, local, peer, Some(1000)), Err(ZoteroError::Refused(e)) if e.contains("another user")));
assert_eq!(same_owner(&only_theirs, local, peer, Some(1001)), Ok(()));
// Neither our row nor our process uid: unknown, so the loopback rule decides.
assert_eq!(same_owner(&only_theirs, local, peer, None), Ok(()));
// Our socket's row is ours by definition and wins over the process uid.
let both = format!("{HEADER}{}{}", row(local, peer, 1001), row(peer, local, 1001));
assert_eq!(same_owner(&both, local, peer, Some(1000)), Ok(()));
}
#[test]
fn a_dual_stack_listener_is_matched_through_its_mapped_address() {
let (local, peer) = addrs();
let mapped = |a: SocketAddr| match a.ip() {
IpAddr::V4(v4) => SocketAddr::new(IpAddr::V6(v4.to_ipv6_mapped()), a.port()),
IpAddr::V6(_) => a,
};
let table = format!("{HEADER}{}{}", row(local, peer, 1000), row(mapped(peer), mapped(local), 1001));
assert!(matches!(same_owner(&table, local, peer, None), Err(ZoteroError::Refused(e)) if e.contains("another user")));
}
#[test]
fn only_this_computer_is_contacted() {
for ok in ["127.0.0.1:23116", "127.9.9.9:1", "[::1]:23116", "[::ffff:127.0.0.1]:23116"] {
assert_eq!(check_addr(&ok.parse().unwrap()), Ok(()), "{ok}");
}
for bad in ["192.0.2.1:23116", "0.0.0.0:23116", "[2001:db8::1]:23116", "[::]:23116"] {
assert!(matches!(check_addr(&bad.parse().unwrap()), Err(ZoteroError::Refused(_))), "{bad}");
}
}
#[test]
fn another_computer_is_refused_before_connecting() {
let opts = crate::client::Options { addr: "192.0.2.1:23116".parse().unwrap(), ..Default::default() };
let r = crate::client::run_command(&opts, crate::Command::Refresh, &mut |_| Ok(serde_json::Value::Null));
assert!(matches!(r, Err(ZoteroError::Refused(_))), "{r:?}");
}
/// A listener this process opened runs as us: the real socket table agrees.
#[cfg(any(target_os = "linux", target_os = "android"))]
#[test]
fn our_own_listener_passes_the_owner_check() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let stream = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
let _accepted = listener.accept().unwrap();
assert_eq!(check(&stream), Ok(()));
}
}
crates/zotero/src/rich.rs
+22 -3
View File
@@ -4,10 +4,12 @@
use wordcraft_doc::edit::Fragment;
use wordcraft_doc::para::OBJ;
use wordcraft_doc::{Block, CharProps, Document, Paragraph};
use wordcraft_doc::{Block, CharProps, Document, Paragraph, Run};
/// Most paragraphs taken from one text (a bibliography of thousands of entries is fine).
const MAX_PARAS: usize = 100_000;
/// Most formatting runs taken from one text; past it, text keeps the formatting before it.
pub(crate) const MAX_RUNS: usize = 1_000_000;
fn style_props(base: &CharProps, src: &CharProps) -> CharProps {
let mut p = base.clone();
@@ -38,18 +40,35 @@ fn style_props(base: &CharProps, src: &CharProps) -> CharProps {
p
}
/// The body paragraphs of `d` (Zotero's RTF or HTML, read), at most [`MAX_PARAS`], formatted on
/// top of `base` with the run styling they carry (past [`MAX_RUNS`] runs, text keeps the
/// formatting before it). Objects are dropped.
fn from_doc(d: &Document, base: &CharProps) -> Vec<Paragraph> {
let mut out = Vec::new();
let mut runs = 0usize;
for b in d.body.iter().take(MAX_PARAS) {
let Some(src) = b.as_para() else { continue };
let mut p = Paragraph::new();
p.mark = base.clone();
// Appended run by run, merging only with the run before; normalized once at the end.
for (range, props) in src.run_ranges() {
let Some(t) = src.text.get(range) else { continue };
let t: String = t.chars().filter(|c| *c != OBJ).collect();
let at = p.len();
let _ = p.insert_text(at, &t, &style_props(base, props));
if t.is_empty() {
continue;
}
let props = style_props(base, props);
p.text.push_str(&t);
match p.runs.last_mut() {
Some(last) if last.props == props || runs >= MAX_RUNS => last.len += t.len(),
_ => {
p.runs.push(Run { len: t.len(), props });
runs += 1;
}
}
}
p.normalize();
p.touch();
out.push(p);
}
out
crates/zotero/src/tests_limits.rs
+122
View File
@@ -0,0 +1,122 @@
//! Hostile documents and calls stay cheap: work grows with the document, not with how often a
//! crafted file or call repeats something (#3692).
use std::time::{Duration, Instant};
use serde_json::{Value, json};
use wordcraft_doc::para::{NoteKind, OBJ};
use wordcraft_doc::{CharProps, Document, InlineObject, Paragraph, PartKind, Run, StoryRef, para_block};
use wordcraft_engine::Session;
use crate::wire::Call;
use crate::{Bridge, Headless, fields};
const CITE: &str = r#"ADDIN ZOTERO_ITEM CSL_CITATION {"citationID":"k1","citationItems":[{"id":7}]}"#;
/// A paragraph of `objects`, one marker each, in one plain run.
fn objects_para(objects: Vec<InlineObject>) -> Paragraph {
let mut p = Paragraph::new();
p.text = std::iter::repeat_n(OBJ, objects.len()).collect();
p.runs = vec![Run { len: p.text.len(), props: CharProps::default() }];
p.objects = objects;
p
}
/// `n` empty Zotero citation fields in a row.
fn citations(n: usize) -> Vec<InlineObject> {
(0..n).flat_map(|_| [InlineObject::FieldStart { instr: CITE.into(), locked: false, dirty: false }, InlineObject::FieldEnd]).collect()
}
// Finding 12: one endnote referenced many times is listed once, not once per reference.
#[test]
fn a_note_referenced_many_times_is_listed_once() {
let mut doc = Document::from_text("");
let note = doc.add_part(PartKind::Endnote, vec![para_block(objects_para(citations(200)))]);
let refs = (0..5_000).map(|_| InlineObject::NoteRef { kind: NoteKind::Endnote, id: note, custom: String::new() }).collect();
doc.body = vec![para_block(objects_para(refs))];
// No clock: listing the note once per reference gave a million fields, which the count catches.
let list = fields::list(&doc);
assert_eq!(list.len(), 200, "each field once, at the note's first reference");
assert!(list.iter().all(|f| f.start.story == StoryRef::Part(note) && f.note_index == 1));
}
#[test]
fn listing_stops_at_the_field_budget() {
let mut doc = Document::from_text("");
// Many short paragraphs: the per-paragraph object scan is another finding's (#3614).
doc.body = (0..fields::MAX_FIELDS / 100 + 1).map(|_| para_block(objects_para(citations(100)))).collect();
assert_eq!(fields::list(&doc).len(), fields::MAX_FIELDS);
// Codes count too: a few huge codes fill the byte budget before the field budget.
let big = format!("{CITE}{}", "x".repeat(fields::MAX_CODE_BYTES / 4 - 1000));
let huge = (0..5).flat_map(|_| [InlineObject::FieldStart { instr: big.clone(), locked: false, dirty: false }, InlineObject::FieldEnd]).collect();
doc.body = vec![para_block(objects_para(huge))];
assert_eq!(fields::list(&doc).len(), 4);
}
// Finding 16: rich text with alternating formatting is built in one pass, not run by run with
// every earlier run merged again.
#[test]
fn rich_text_with_many_runs_is_linear() {
let html: String = "<i>a</i>b".repeat(50_000);
let t = Instant::now();
let frag = crate::rich::fragment(&html, true, &CharProps::default());
let elapsed = t.elapsed();
let Some(wordcraft_doc::Block::Para(p)) = frag.blocks.first() else { panic!("no paragraph") };
assert_eq!(p.text, "ab".repeat(50_000));
assert_eq!(p.runs.len(), 100_000);
assert_eq!(p.runs.iter().map(|r| r.len).sum::<usize>(), p.text.len());
// Under half a second on a loaded machine; merging again after every run ran past 9 minutes.
assert!(elapsed < Duration::from_secs(10), "{elapsed:?}");
// Neighbours with the same formatting are one run.
let frag = crate::rich::fragment("<i>a</i><i>b</i>c", true, &CharProps::default());
let Some(wordcraft_doc::Block::Para(p)) = frag.blocks.first() else { panic!("no paragraph") };
assert_eq!((p.text.as_str(), p.runs.len()), ("abc", 2));
assert_eq!(p.props_of_char(1).italic, Some(true));
}
/// Call `method` on a bridge over `s`.
fn call(b: &mut Bridge, s: &mut Session, method: &str, args: Value) -> Result<Value, String> {
let Value::Array(args) = args else { return Err("args must be an array".into()) };
b.handle(s, &mut Headless::default(), &Call::new(method, args))
}
// Finding 32: Document_convert checks each distinct id once against one list of the fields.
#[test]
fn convert_with_one_id_repeated_is_one_check() {
let mut doc = Document::from_text("");
// Checking each id against a fresh list of the fields took 81 s with 300 of them; with 1,000
// that is minutes, against milliseconds for one indexed list.
doc.body = (0..1000).map(|_| para_block(objects_para(citations(1)))).collect();
let mut s = Session::new(doc);
let mut b = Bridge::new();
call(&mut b, &mut s, "Application_getActiveDocument", json!([3])).unwrap();
let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap();
let first = got[0][0].clone();
let ids = vec![first.clone(); 100_000];
let before = format!("{:?}", s.doc.body);
let t = Instant::now();
call(&mut b, &mut s, "Document_convert", json!([1, ids, 0, 0])).unwrap();
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
assert_eq!(format!("{:?}", s.doc.body), before, "already in the text: nothing to do");
// An unknown id is refused before anything moves.
let r = call(&mut b, &mut s, "Document_convert", json!([1, [first.clone(), 999_999_999u64], 0, [1, 1]]));
assert!(r.is_err());
assert!(s.doc.parts.is_empty(), "nothing converted");
}
#[test]
fn convert_with_a_repeated_id_takes_its_last_note_type() {
let mut doc = Document::from_text("");
doc.body = (0..3).map(|_| para_block(objects_para(citations(1)))).collect();
let mut s = Session::new(doc);
let mut b = Bridge::new();
call(&mut b, &mut s, "Application_getActiveDocument", json!([3])).unwrap();
let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap();
let (a, c) = (got[0][0].clone(), got[0][2].clone());
call(&mut b, &mut s, "Document_convert", json!([1, [a.clone(), c, a], 0, [2, 1, 1]])).unwrap();
let kinds: Vec<PartKind> = s.doc.parts.values().map(|p| p.kind).collect();
assert_eq!(kinds, vec![PartKind::Footnote, PartKind::Footnote]);
// Ids stay valid: the fields are where Zotero now expects them.
let got = call(&mut b, &mut s, "Document_getFields", json!([1, "Field"])).unwrap();
assert_eq!(got[2], json!([1, 0, 2]));
}
docs/file-format-parity.md
+10 -4
View File
@@ -1,6 +1,6 @@
# File-format parity with Microsoft Word
> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `<blockquote>` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructions no longer add blank lines; previously trivial: `flowChartOr` draws its circle and cross spokes, as Line truncated
> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; importers bound hostile expansion: font names, links, ODS repeats, LaTeX wrappers, Markdown delimiters, #3692/#3713; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `<blockquote>` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-tripsLine truncated
Every format Word opens or saves, what WordCraft does with it, and how it's tested. Fidelity is the
share of a typical real-world file's content and formatting that survives (read: shows correctly;
@@ -49,7 +49,7 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅
| `.docx` Word Document (ECMA-376 Transitional) | 70% | R/W | ✅ | ✅ | ~65% | 70–110 | `crates/docx/tests` (roundtrip 34, fixtures 29, malformed 8, bidi 5, bidi_runs 12, citations 5); synthetic fixtures only | Word opens our files. Untested on a real-world corpus. Tracked changes inside a field result stay revisions (the field is kept as a range, #1037). Details below |
| `.docx` Strict Open XML | 1% | R/W | ✅ (namespace mapping in `xml.rs`) | ❌ | ~60% | 2–4 | `strict_namespace_and_bad_numbers`, `strict_line_heights_with_a_unit_are_read_as_point_multiples` | Writing Strict is rare; unit-suffixed measures (`13.90pt`) read as points (#2329) |
| `.docm` / `.dotx` / `.dotm` | 3% | R/W | ✅ | ✅ | ~65% | incl. above | `macro_packages.rs` | Macros and signatures kept on save (#172); macros never run |
| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). |
| `.doc` Word 97-2003 (and `.dot`) | 8% | R/W | 🟡 | ❌ | ~60% | 25–40 | `crates/docbin/src/tests.rs`, `tests/drawing.rs` | Spec-based reader ([MS-DOC]): text, formatting, styles, sections, headers/footers, tables, lists, notes, fields, bookmarks; inline and floating pictures (PNG, JPEG, DIB, TIFF, EMF, WMF; Macintosh PICT as a sized placeholder) with crop, alt text, position and wrap; basic shapes, lines, groups and text boxes ([MS-ODRAW]). Table cell shading from every SHD record (`sprmTDefTableShd`, `sprmTDefTableShdRaw` and their 2nd/3rd ranges, with the Word 97 `Shd80` as fallback; a clear pattern shows its background colour); the row-end mark is not a cell, and each table has its own grid and properties (#1380). Horizontally merged rows stored as fewer, wider cells come with #1123. Freeform shapes, z-order and Word 6/95 files are dropped or refused. Password-protected files (XOR obfuscation, RC4, RC4 CryptoAPI) open after a password prompt, from the public specs; not yet tried on a file Word wrote (AES-encrypted ones are refused). No writer. DIB conversion caps BMP bytes (header included) and encoded PNG at the supplied BLIP byte limit, and uses it as the decoder's allocation limit; a BMP fallback is kept if it fits (related #1334 and #1554). Hostile files are bounded: a sector table longer than the file is refused before parsing (#3642); formatting and link runs are read once per change, not per character (#3692, #3708); copies of alt text, link targets and drawn objects share a 64 MB budget (#3709, #3710), and what it leaves out is logged once a kind per document; picture decoding shares a 512 MB budget and aliased store entries decode once (#3711, #3712). |
| PDF export | 8% | W | — | ✅ | ~80% | 6–10 | `crates/pdf/src/tests.rs` | krilla: embedded subsets, selectable text incl. field results (#133); Hebrew and Arabic copy and search as whole words (right-to-left glyphs written in visual order, #1383, also with fonts that draw lam-alef as two glyphs at one x, #2310); All Caps and Small Caps text extracts as the capitals shown, as from Word's PDFs, and never changes the case of other text (#1473), links, outline, tagging. Outline destinations follow leading page breaks to the heading text. Fonts it can't subset become outlines (#132). Print Markup (#589): comments in balloons beside each page (scaled to its paper) and tracked changes; HTML/Markdown/text copies end with a comment list. Horizontal and vertical lines inside groups are drawn (#1474). No PDF/A option, no "best for printing/online" choice |
| PDF open (PDF Reflow, Word for Windows) | 1% | R | ❌ | — | 0% | 30–50 | — | Converting a PDF into an editable document. A sibling app (PdfCraft) may supply the parser |
| `.rtf` Rich Text Format | 3% | R/W | 🟡 | 🟡 | ~50% | 10–15 | `crates/formats/src/tests.rs` | ~1,500 lines: text, character/paragraph formatting, styles, lists, tables, pictures (`\shppict`, #154), fields, bookmarks. 8-bit text in the document's code page (`\ansicpg`) or the current font's (`\fcharset`, `\cpg`): Cyrillic, Central European, Greek, Turkish, Hebrew, Arabic, Baltic, Vietnamese, Thai (was Windows-1252 only), Mac OS Roman (`\mac`, `\fcharset77`) and DOS code pages (`\pc` 437, `\pca` 850, `\cpg` 437/737/850/852/857/866, `\fcharset254`) (#1653). East Asian multibyte code pages decoded as runs: 932 Shift-JIS (`\fcharset128`, Japanese), 936 GBK (134, Simplified Chinese), 949 UHC/windows-949 (129, Korean) and 950 Big5 (136, Traditional Chinese) through `\ansicpg`, `\cpg` or the font's charset, escaped or raw, a lead byte and its trail byte together; invalid sequences read as U+FFFD, and a run that starts `EF BB BF` is text, not a BOM (#1653). Pasted from the system clipboard where the platform offers it (Windows, Wayland) when there is no HTML (#687). Tracked insertions and deletions read and written (`\revtbl`, `\revised`/`\deleted`, authors, `\revdttm` dates), so Accept/Reject All work after opening or a round trip; tracked paragraph marks (`\par` inside `\revised`/`\deleted` groups, including a `\sect` inside a deletion) and character and paragraph formatting revisions (`\crauth`/`\crdate` with `\oldcprops`, `\prauth`/`\prdate` with `\oldpprops`) read and written; section and table formatting revisions (`\oldsprops`, `\oldtprops`) not yet. Sections on import (#1378): each `\sect` ends a section with its own page size and orientation, margins, header/footer distance, columns, break type, page-number restart, start and format, title page and headers/footers (those it leaves out link to the previous); export writes every section too (break type, page size and orientation, margins, header/footer distance, columns, page numbering, title page; not headers/footers). Thin: comments, notes, RTL |
@@ -111,7 +111,7 @@ Measured by grepping `crates/docx/src/read` and `src/write` for the OOXML elemen
| Native Word ink (`w14:contentPart`, InkML) | 🟡 | 🟡 | Drawing-level ink kept opaque with its InkML part, relationship and content type; direct run-level `w:contentPart` and run-level `mc:AlternateContent` keep the native branch and VML picture fallback. Direct content parts remain direct through repeated save/reopen cycles, without a native rendering. Drawing-level ink shows the preview or a missing-picture box. Ink members of groups (`wpg:wgp`, nested groups too) and drawing canvases (`wpc:wpc`), bare or in `mc:AlternateContent` with a `pic:pic` fallback, keep their markup, part, relationship and place among the members; they are written at the member's current position in the group and drawn from their fallback picture. Such a group can't be ungrouped yet (Ungroup is unavailable, so the ink is never left in a form it can't be saved in). Their part is dropped with the group (no orphan part); ink whose part is missing or isn't InkML gives way to its fallback, else is dropped. No native stroke rendering, eraser, conversion or replay |
| 3D models | ❌ | ❌ | Dropped on read, not preserved on save |
| Equations (OMML `m:oMath`) | ✅ | ✅ | #191; manual breaks (`m:brk`) and the settings' `m:mathPr` wrapping options (#326). Every ECMA-376 §22.1 element is read and laid out (pinned by `read::math::tests::every_omml_element_keeps_its_content`); equations straight in the body or a cell (no `w:p`), behind `mc:AlternateContent Requires="m"`, or with runs inside hyperlinks, simple fields, content controls and tracked insertions inside the equation keep their content; `w:sym` and Symbol-font text in math become Unicode (#1273). Not yet: Equation Editor 3.0 / MathType OLE objects show their WMF preview only (text in #1994), `m:mathFont` (Cambria Math is always asked for), `m:dispDef`/`m:defJc`/`m:lMargin`, tracked deletions shown struck through inside equations |
| Themes, font table, settings, compatibility mode | ✅ | ✅ | `w:allowSpaceOfSameStyleInTable` read and written in schema order; absent/false default off, enabled flag retained in modes 11/12/14/15. Embedded fonts (`w:embedRegular`, `w:embedBold`…) lay out and export the text set in them for the session (#1176), subsets Word embeds ("embed only the characters used", a made-up family such as `___WRD_EMBED_SUB_46` inside) found by the font table's name (#2240); the font table, its relationships and the embedded font parts are written back unchanged (#770); attached template (`w:attachedTemplate` and its external relationship) and `w:linkStyles` round-trip (#377). `w:removePersonalInformation` and `w:removeDateAndTime` round-trip and are honoured on save (#1062): comment and revision authors written as "Author" (initials "A"), no `dc:creator`/`cp:lastModifiedBy`, no annotation dates; the open document keeps its values (`file.privacyOptions` sets them, `file.privacy` reads them; File › Options › Save › Privacy for this document). The template path comes from the file, so opening never reads it: styles update from it only on an explicit Update Styles. Settings the model doesn't hold (other `w:compatSetting`s, rsids, document variables, proofing state, colour scheme mapping…) survive a save: the writer merges into the file's own `settings.xml` ([below](#settings-preservation)); `w:latentStyles`, `w:default` on localized default styles and lists' `w:nsid`/`w:tmpl`/`w:multiLevelType` round-trip |
| Themes, font table, settings, compatibility mode | ✅ | ✅ | `w:allowSpaceOfSameStyleInTable` read and written in schema order; absent/false default off, enabled flag retained in modes 11/12/14/15. Embedded fonts (`w:embedRegular`, `w:embedBold`…) lay out and export the text set in them for the session (#1176), subsets Word embeds ("embed only the characters used", a made-up family such as `___WRD_EMBED_SUB_46` inside) found by the font table's name (#2240); an embedded font file yields at most 256 faces (collection members and named instances, #3716); the font table, its relationships and the embedded font parts are written back unchanged (#770); attached template (`w:attachedTemplate` and its external relationship) and `w:linkStyles` round-trip (#377). `w:removePersonalInformation` and `w:removeDateAndTime` round-trip and are honoured on save (#1062): comment and revision authors written as "Author" (initials "A"), no `dc:creator`/`cp:lastModifiedBy`, no annotation dates; the open document keeps its values (`file.privacyOptions` sets them, `file.privacy` reads them; File › Options › Save › Privacy for this document). The template path comes from the file, so opening never reads it: styles update from it only on an explicit Update Styles. Settings the model doesn't hold (other `w:compatSetting`s, rsids, document variables, proofing state, colour scheme mapping…) survive a save: the writer merges into the file's own `settings.xml` ([below](#settings-preservation)); `w:latentStyles`, `w:default` on localized default styles and lists' `w:nsid`/`w:tmpl`/`w:multiLevelType` round-trip |
| Custom XML parts, document properties, bibliography sources | 🟡 | 🟡 | Custom XML stores reached from the package/main document preserve opaque data, properties and relationships; binding evaluation remains absent. Custom properties and bibliography sources round-trip (#532). Word's own bibliography file (`b:Sources`, `Sources.xml`) imports into Manage Sources, sharing the reader of a document's own bibliography part (#2640) |
| Section text direction (`w:textDirection` in `w:sectPr`, §17.6.20) | ✅ | ✅ | `tbRl`, `btLr` (and `tbRlV`/`tbLrV`, read as `tbRl`); `lrTbV` and unknown values read as horizontal (#491) |
| Section vertical alignment (`w:vAlign` in `w:sectPr`, §17.6.23) | ✅ | ✅ | `top`, `center`, `bottom` and `both` (Justified, read as Center before #1320); unknown values read as top, top writes nothing (`crates/docx/tests/it/section_valign.rs`). ODT/RTF don't carry it |
@@ -121,7 +121,7 @@ Measured by grepping `crates/docx/src/read` and `src/write` for the OOXML elemen
| `w:fitText` | ❌ | ❌ | East Asian layout |
| Glossary document (building blocks) | ✅ | ✅ | `w:docPart` entries (name, style, gallery, category, types, behaviours, description, guid, body) are read into `Document::glossary` and offered by Insert › Quick Parts and AutoText; `save` with `"in": "document"` stores one in the file. An untouched glossary is written back byte for byte with its own parts (styles, settings, fonts…); an edited one, or one whose parts weren't kept, is written from the model as `word/glossary1.xml` with copies of the document's styles and numbering. The glossary's own styles, settings and fonts, text boxes inside entries, and the notes and comments of a selection saved into the document don't survive an edit. Tests: `crates/docx/tests/it/glossary.rs` |
| `w:altChunk`, sub-documents | ❌ | ❌ | |
| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard |
| Hostile input | ✅ | — | `malformed.rs`, capped allocations, never-crash standard. Failed zip entries count against the size limit, case variants of a part name are one part, embedded fonts and charts are written within budgets, and Flat OPC inlines kept XML only when it is one well-formed element (2026-10-22) |
| Document paragraph defaults (`w:pPrDefault`) | ✅ | ✅ | Omission and explicit empty defaults remain distinct across repeated saves; nonempty defaults and later default-spacing edits are written |
## Saving many content controls
@@ -268,6 +268,12 @@ Repeated DOCX style IDs use the last complete definition. The original regressio
| 2026-10-25 | trivial | Repeated DOCX style IDs: repeated save/reopen regression for whole-definition replacement. |
| 2026-10-25 | trivial | Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`) |
| 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged |
> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-25 · **Change:** trivial (the `.doc` reader logs what its copy budget leaves out, once a kind per document; previously trivial: repeated DOCX style IDs: save/reopen regression; previously trivial: Markdown code spans keep line-broken text literal, #3738; previously trivial: Arabic list labels: `arabicAlpha` and `arabicAbjad` draw the 28 Arabic letters (alphabetical and abjad order), doubling past the last letter; Hebrew, Hindi, Thai and Vietnamese still show decimal (`doc::section::tests::arabic_letters_count_in_both_orders`); previously trivial: localized TOC style ids read as the TOC styles, and a TOC in columns keeps its layout through Update Table, #3889; previously trivial: cu20we: outline caps; previously trivial: a relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part; previously trivial: the reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, #3411; previously trivial: a table cell span from a `.json` file stays within 63 columns through Insert Column and Merge Cells, #3637; previously trivial: the file commands that change the document are checked on a protected document, the protection of a protected document needs an explicit stop, the session author can't change through `file.setAuthor` under protection (the CLI's `--author` uses the same protection check), and the post-check compares the settings and the metadata; previously trivial: a document with many content controls saves in the time the controls take, #13; previously trivial: `majorBidi`/`minorBidi` theme fonts follow the settings' theme font language, as measured in Word; previously trivial: Word signature lines are kept on save, still signable; previously trivial: Times New Roman keeps Liberation Serif under the roman-font rule; previously trivial: shape gradients and transparency, #1500; previously trivial: a folder that keeps no Unix modes, such as a NAS share mounted by GVFS, still takes a save, #3003; previously trivial: a text frame in the paragraph defaults frames nothing, #2360; previously trivial: table style band sizes written, a style without them unbanded as in Word, #3415; previously trivial: HTML export writes ruby (furigana, phonetic guides) and HTML import reads it back, #1681; previously trivial: RTF import decodes the East Asian multibyte code pages, #1653; previously trivial: PDF export: lam-alef drawn as two glyphs at one x extracts in reading order, #2310; previously trivial: native .json stores pictures and kept parts as base64, #586 follow-up; previously trivial: ODT headers and footers read from each master page, #863; previously trivial: RTF import compares heading and code formatting with the file's stylesheet; previously trivial: text typed into a content control takes the control's formatting; localized placeholder styles recognised; previously trivial: forms protection keeps its content controls: no inserting, Properties or Remove Content Control; previously trivial: Word XML and web pages saved as `.doc` open by what they hold, #2853; previously trivial: a VML shape in a header cell is always placed on the page, user-checked in Word for Mac, #1775; previously trivial: `w:sym` characters keep their symbol font when the text's font changes and are written back as `w:sym`, #2036; previously trivial: RTF export writes sections (gap 19); previously trivial: ODT export writes sections (gap 19); previously trivial: two lines in one and horizontal in vertical (`w:eastAsianLayout`) kept on save; previously trivial: PDF outline targets follow leading page breaks; previously trivial: positional tabs `w:ptab` read and kept on save, #2367; previously trivial: DOCX: mixed right-to-left runs split and marked `w:rtl` with their language, #1543; previously trivial: `w:doNotTrackMoves` round-trips and turns move tracking off; previously trivial: legacy outline, shadow, emboss and imprint text effects drawn, #2534; previously trivial: DOCX glossary documents (building blocks) read, kept and written; previously trivial: cached field results keep their visible run formatting; previously trivial: cached REF results retain hyperlink destinations; previously trivial: CLI conversions can explicitly refresh fields; previously trivial: a copied equation also goes in the clipboard's MathML formats, #1297; previously trivial: a pasted HTML `<blockquote>` is an indented paragraph, not the Quote style, #1270); previously trivial: every table style part round-trips, #714; previously trivial: VML freeforms read and kept on save; previously trivial: multi-paragraph field instructioLine truncated
| 2026-10-23 | trivial | `.doc` reader logs what its copy budget leaves out (floating objects, link targets, alt text, Prm formatting), once a kind per document (#3692) |
| 2026-10-22 | trivial | `.doc` reader bounds hostile files: oversized sector tables refused, per-character re-reads removed, copy and decode budgets (#3642, #3692, #3708-#3712) |
| 2026-10-22 | trivial | Hostile input: DOCX package, comment, font, chart and Flat OPC hardening (#3692, #3723, #3704, #3742, #3648) |
| 2026-10-22 | trivial | An embedded font file yields at most 256 faces however many collection members or named instances it lists (#3716) |
| 2026-10-22 | trivial | Importer hardening (#3692, #3705, #3713, #3714): spreadsheet header numbering and ODS repeated cells, LaTeX wrapper nesting and verbatim ends, Markdown delimiter runs, blank-span trimming and link destinations, and font/link copies per run are bounded or escaped; normal documents unchanged |
| 2026-10-23 | trivial | A relationship the markup names without the prefix is read as it wrote: an unqualified `id` on an element that carries a relationship names that relationship, and the writer keeps the attribute with the id the new package holds the part under; an ordinary `id` — a VML shape's, a drawing's — names no part (#3411 follow-up) |
| 2026-10-21 | trivial | The reader takes the relationship ids the markup refers to from the relationship attributes of the main document it has already parsed, so a document with many relationships opens in the time its part takes, and an id that only appears as text is not a reference (#3411) |
| 2026-10-23 | trivial | DOCX: TOC styles Word stores under ids in another language (`Sumrio1` named `toc 1`, `CabealhodoSumrio` named `TOC Heading`) read as `TOC1`…`TOC9` and `TOCHeading` unless the file uses those ids, so Update Table builds the entries in the document's own TOC styles (#3891); a TOC whose heading ends a section (entries in columns after a continuous break) is written and read as the engine's TOC, with the page-number tab at the column's edge (#3889) |
docs/gaps.md
+6
View File
@@ -35,6 +35,8 @@ Document renaming: desktop renaming requires a filesystem that supports hard lin
Comment command safety (#1690, finding 3): Delete, Resolve, Reply and Edit reject explicit fractional, negative or wrong-type IDs before editing, rather than truncating floats or using the caret. Synthetic tests cover refusal without document or history changes, valid operations with undo/redo and repeated DOCX saves. This narrows the data-path risks in gap #3; the other audit findings and broader stability gate are outside this change.
Proofing robustness (security audit, #3613 part, #3661, #3657, #3625): installed Hunspell dictionaries with thousands of self-continuing rules check each word in linear time and list a bounded number of forms; dictionary files are read with a bounded read (Turkish included) and the hyphenation dictionary inflates under a 64 MiB limit; Russian hyphenation and Turkish suggestions no longer grow with the square of the word's length. This narrows the freeze risks in gap #3; distinct-strip rule chains and proofing off the UI thread remain (#3613).
Font-menu reliability: clearing a query with no matches now shows the recent section from its first row, including its heading, in a scrollable list. Fixed synthetic-family widget and full-ribbon tests cover this case and matching-query controls (#2068).
Additive paragraph spacing retains the full before-space after a section page break; ordinary overlapping spacing is unchanged. Other pagination gaps remain open.
@@ -43,6 +45,8 @@ Exact line spacing keeps the run font baseline around tall equations; automatic
Legacy Word imports (#2445): interactive Save for `.doc`/`.dot` uses the existing DOCX Save As dialog. Cancel and write failures retain the draft and original path; Save before Close continues only after a successful supported-format save. This narrows save reliability under gap #3; binary `.doc` writing remains unsupported. Late Save As answers are rejected before writing if automation has replaced the document.
Document model hangs (#3614, #3692): paragraph object walks, image deduplication, linear-equation letter runs and text box paste no longer do quadratic work or unbounded copies on hostile files. This narrows gap #3; the engine, DOCX reader and UI loops named in #3614 remain.
Active footnote and endnote text stays readable while editing (#2545). This fixes a display-context defect; the broader fidelity, accessibility and readiness gates below are unchanged.
Russian language-aware sorting now keeps ё with е instead of after я, with the diaeresis as a secondary difference. Composed and decomposed ё and й compare equally. Fixed synthetic collation tests cover order, case and index headings; this does not establish full Cyrillic or CLDR collation parity.
@@ -158,6 +162,8 @@ Small follow-ups from the 2026-10-25 field, table and AutoFormat batch (#4232):
| 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged |
| 2026-10-24 | trivial | Exact-spacing table contact uses the specified line height in row selection and word skipping; large/small mark regressions in modes 11/12/14/15 (#3977); shape estimates unchanged |
| 2026-10-24 | trivial | Floating tables count exact line-bottom contact; shapes keep strict overlap (#3977); original boundary/skip regressions in modes 11/12/14/15, expected Word rule from cyril265; readiness estimates unchanged |
| 2026-10-22 | trivial | Gap 3: proofing robustness — linear affix rule walks, bounded dictionary reads, linear Russian hyphenation, Turkish word cap (#3613 part, #3661, #3657, #3625); readiness estimates unchanged |
| 2026-10-22 | trivial | Gap #3 note: document model quadratic scans and text box paste copies bounded (#3692, part of #3614) |
| 2026-10-22 | trivial | Gap 10: group connector rerouting preserves rotated/flipped free and unresolved endpoints under nonuniform scaling, with safe degenerate-dimension and display-size-cap handling; routing around shapes and readiness estimates unchanged |
| 2026-10-21 | trivial | The ranked gap list held two rows each for #19, #20 and #21; the stale copies are gone, so each gap appears once, and the #20 row carries the measured figure (#3411) |
| 2026-10-21 | trivial | #20 narrowed: a document with many relationships opens in the time its part takes (#3411) |
docs/layout-parity.md
+5 -2
View File
@@ -23,7 +23,7 @@ Exact line spacing keeps the run font baseline around tall equations; automatic
| Feature | State | Evidence / gap | Hours |
|---|---|---|---|
| 2026-10-16 | trivial | Split-cell content retains its resolved top margin and border band on continuation pages, matching original Word Mac16.113.2 controls. Synthetic regressions cover margins, borders, line spacing and image frames. |
| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. | — |
| Hint-reliant CJK glyph outlines (DFKai-SB / 標楷體) | 🟡 | Required TrueType interpreter runs at units-per-em size; local Windows glyph-bounds regression and synthetic DOCX page/window comparison pass. Other platforms and hint-reliant families remain unmeasured; no native Word comparison. A font asking for a twilight zone over 1,024 points or a stack over 4,096 values, or a glyph past 256 KiB of hinting buffer, is drawn unhinted; on desktop, once at least three of a face's hinted draws have taken over 50 ms and those slow draws are at least 1/16 of its hinted draws, its later glyphs are drawn unhinted (#3692). | — |
| Subpoint table grid columns | 🟡 | Preserve positive grid widths rather than imposing a text-box minimum; ECMA-376 §17.4.16. Native Word for Mac 16.113.2 build26092012 confirms narrow trailing grid units. `layout::tests::subpoint_grid_keeps_widths`, `tiny_grid_scales_to_width`. Tiny totals normalize without overflowing the scale factor. Zero and invalid widths retain their fallback. Border painting remains separate. | n/a |
| Line breaking (UAX #14 with Word's exceptions: no break after `/`, a break after `-` before a digit) | ✅ | `layout::tests::no_line_break_right_after_a_slash`, `layout::tests::a_hyphen_before_digits_is_a_break_opportunity_like_word`, `layout::tests::a_bookmark_is_no_line_break_opportunity` | — |
| A break before a slash that starts a word (" /word" wraps whole; not after a no-break space) | 🟡 | `layout::tests::a_word_after_a_slash_wraps_whole`, `layout::tests::a_no_break_space_before_a_slash_still_holds`; behaviour from the #2963 report, not compared with Word yet | — |
@@ -35,7 +35,7 @@ Exact line spacing keeps the run font baseline around tall equations; automatic
| Missing Thai document fonts (TH Sarabun New, Angsana New, Cordia New, Browallia New, the UPC families, Leelawadee): one Thai substitute face for the whole run, Latin letters, digits and ฿ included, so the run keeps one font at one size as Word's Cordia New substitution does | ✅ | `fonts::word::tests::a_missing_thai_font_resolves_to_an_installed_thai_face`, `fonts::fontdb::tests::a_thai_run_with_latin_falls_back_to_one_face`, `layout::tests_complex::a_missing_thai_font_draws_the_whole_run_in_one_thai_face`, `layout::tests_complex::a_thai_substitute_without_latin_hands_the_run_to_one_face_covering_it`, `fonts::fontdb::tests_sysfonts::a_run_no_preferred_face_covers_whole_loads_an_installed_font_that_does` (#1549). Windows: Cordia New, then Leelawadee UI, Tahoma, Noto Sans Thai; macOS: Thonburi (Sathu for the serif families); Linux: Noto Sans Thai, then the TLWG fonts. The first listed face with Latin letters and digits as well as Thai wins. Other Thai names (`TH …`, `…UPC`, names with `Thai`) use the same lists. When no listed face is installed, or the one found lacks the run's Thai or Latin letters, the run moves to one installed face covering its Thai and Latin text if there is one, even when a preferred Thai family without Latin letters (Noto Sans Thai) is installed; that face's whole family is loaded, so regular text keeps the Regular face. Only Thai fonts: a missing Latin font keeps Latin text in its own substitute, and a run with its own complex-script font keeps it for the Thai letters, as Word does | — |
| Distributed / Thai distributed alignment | 🟡 | read; layout untested | 1–2 |
| Default font metrics (Aptos, Calibri, Cambria…) | 🟡 | Missing Arial Unicode MS tries Arial then the Arial substitutes, case-insensitively (`fonts::word::tests::unicode_arial_is_proportional`); glyph coverage and exact Word line height can differ. Carlito/Caladea/Liberation are metric-compatible for Calibri/Cambria/Arial/Times/Courier; **Aptos (Word's default since 2024) has no metric-compatible open substitute**, so new Word documents wrap differently ([typing-parity.md](typing-parity.md) Known gaps). Needs an owner decision (commission or license a metric-matched font) | 6–10 + owner |
| Colour emoji and colour fonts on screen and in page renders: COLR v0/v1 paint graphs (solid fills, linear, radial and sweep gradients, composite modes) and bitmap strikes (Apple Color Emoji `sbix`, Noto Color Emoji `CBDT`) drawn in the font's colours at the layout's position and size; a dark page doesn't invert them, the same as pictures | 🟡 | `fonts::color` (skrifa), `render::color_glyph`; `render::tests_color_glyphs::emoji_are_drawn_in_colour` and `emoji_keep_their_colours_on_a_dark_page` (system font, skip without one), `render::color_glyph::tests`, `fonts::color::tests` (#1269). CBDT is verified on a real page with Noto Color Emoji; COLR v0/v1 and sbix are implemented to the OpenType spec and tested with synthetic data, but not yet seen on a real font (Segoe UI Emoji, COLRv1 Noto, Apple Color Emoji). PDF export already drew them through krilla | 1–2 |
| Colour emoji and colour fonts on screen and in page renders: COLR v0/v1 paint graphs (solid fills, linear, radial and sweep gradients, composite modes) and bitmap strikes (Apple Color Emoji `sbix`, Noto Color Emoji `CBDT`) drawn in the font's colours at the layout's position and size; a dark page doesn't invert them, the same as pictures | 🟡 | `fonts::color` (skrifa), `render::color_glyph`; `render::tests_color_glyphs::emoji_are_drawn_in_colour` and `emoji_keep_their_colours_on_a_dark_page` (system font, skip without one), `render::color_glyph::tests`, `fonts::color::tests` (#1269). A glyph copies at most 65,536 gradient stops and clip-path elements in all, else it falls back to its outline, and the cache charges what it keeps (#3692). CBDT is verified on a real page with Noto Color Emoji; COLR v0/v1 and sbix are implemented to the OpenType spec and tested with synthetic data, but not yet seen on a real font (Segoe UI Emoji, COLRv1 Noto, Apple Color Emoji). PDF export already drew them through krilla | 1–2 |
| Full font names in document font requests | ✅ | Exact English (or first available) OpenType name ID4 lookup in loaded faces and the native lazy catalog; family and legacy-family matches keep priority. No style-suffix guessing; named variable-instance aliases remain unsupported | 0 |
| Symbol fonts (Symbol including its PCL6 alias, Wingdings, Wingdings 2/3, Webdings) without the proprietary font: each character, stored as U+F020–U+F0FF or a plain 8-bit code, is drawn as its Unicode equivalent from a face that has it; PDF text and copies read as those symbols | ✅ | `doc::symbol_fonts` tables (Adobe Symbol table and Unicode L2/12-368), `layout::tests::wingdings_text_draws_and_reads_as_its_unicode_symbols` (#776). Shapes come from Noto Sans Symbols / Symbols 2, DejaVu Sans or Segoe UI Symbol, so they differ from Word's; Wingdings 0xFF (not in Unicode) still has no symbol. A line keeps the symbol font's own height (Symbol, Wingdings: the ascent and descent Word's PDFs give them), so a Symbol bullet makes its line as tall as in Word, `layout::tests::a_symbol_bullet_sizes_its_line_by_the_symbol_font`. Apple's one-em Symbol takes the same height, as in Word for Mac (`fonts::word::tests::a_one_em_symbol_face_takes_windows_symbols_line`) | 0 |
| List bullets stored as a private-use character (U+E000–U+F8FF) in a font that isn't installed and isn't a mapped symbol-font code: drawn as the level's default bullet (• ○ ▪), not as whatever an installed icon font has at that code | ✅ | `layout::tests::private_use_bullet_no_font_draws_shows_the_levels_default_bullet` (#1683: U+F081 showed Font Awesome's Twitter logo) | — |
@@ -196,6 +196,7 @@ No overall parity estimate changes from these probes.
| Scrolling a 1,096-page document that is one 5,300-row table (#811) | ~2.4 ms idle, ~4.5 ms median scroll frame (was ~600 ms); opening still lays out for ~4–8 s | not measured | measured 2026-10-12, release, headless (`tests_perf::big_table_scroll_bench`) |
| A word with no break opportunity in it, laid out beside a floating object (#3412) | 1 MiB: 29.4 s → 0.35 s; 4 MiB: 472 s → 1.4 s | not measured | measured 2026-10-21, on 9b4b948c before the fix and with it (`layout::tests::a_long_unbreakable_word_lays_out_in_linear_time`, `layout::tests::a_megabyte_long_word_lays_out_quickly`) |
| The same word, with a font that pulls half its glyphs back so that its advances alternate in sign (#3412) | 8× the word costs 52× the time before the fix, 15× with it | not measured | measured 2026-10-22, release, on `00bc6a4b` before the fix and with it (`layout::tests::a_word_with_alternating_advances_lays_out_in_linear_time`) |
| Pages, sections and paragraphs with thousands of clipped cells, tracked authors, comment authors, sections, header paragraphs, spanning cells, hidden bidi offsets or math characters (#3692, #3706) | linear: each lookup is indexed or walked once (was a rescan per item) | not measured | `layout::tests_linear`, `math::tests::a_long_text_run_lays_out_in_linear_time` |
A word with no break opportunity in it takes a line per few characters, and measuring the word
beside a floating object walked the whole paragraph for every one of those lines, so the layout
@@ -271,6 +272,8 @@ layout suite are pending; this is not a new Word/native-IME acceptance claim.
| 2026-10-24 | trivial | Floating tables count exact line-bottom contact; shapes keep strict overlap (#3977); original boundary/skip regressions in modes 11/12/14/15, expected Word rule from cyril265; readiness estimates unchanged |
| 2026-10-23 | trivial | cu20we: preserve and render outline caps across DOCX/VML, screen and PDF; synthetic regressions; readiness estimates unchanged |
| 2026-10-23 | trivial | cu20we: paragraph-relative box floats use the pre-spacing origin; header/footer/cell/text-box regressions; readiness estimates unchanged |
| 2026-10-22 | trivial | Layout lookups linear in clipped cells, authors, sections, header paragraphs and bidi offsets (#3692, #3706) |
| 2026-10-22 | trivial | Hostile fonts (#3692): hint-reliant fonts with huge hinting zones are drawn unhinted, and a face draws its later glyphs unhinted once at least three of its hinted draws took over 50 ms and those are at least 1/16 of its hinted draws; a colour glyph's gradient stops and clip paths share one budget |
| 2026-10-22 | trivial | Group connector rerouting preserves displayed free and unresolved endpoints under nonuniform scaling while resolved ends keep their connection-site path; degenerate dimensions and unrepresentable fallback display spans are guarded; synthetic model/engine coverage only, readiness estimates unchanged |
| 2026-10-21 | trivial | A word with no break opportunity in it, laid out beside a floating object, no longer lays out quadratically: every word is measured once, before the line is broken (#3412) |
| 2026-10-23 | trivial | An empty paragraph whose mark is hidden is laid out as nothing, like a deleted one: no room, no list number (still counted, as in Word) and no shading, in the body and in table cells, headers and text boxes (#3982) |
docs/mcp.md
+2 -1
View File
@@ -70,7 +70,8 @@ config works across the suite) keep the agent to the folders you give it:
from beside that file, so they are inside the read root too (a link that leads out is not followed).
- Every file a command writes must lie inside the **write root**: `save_document`, `file.save`,
`file.saveAs`, `file.rename` (the file and its new name), `file.exportPdf`, `file.exportPng`, `file.printPdf`,
`file.saveTemplate`, and `mailings.finish` with a `path`.
`file.saveTemplate`, and `mailings.finish` with a `path`. `font.add` and `font.remove` change the
user font folder, so they are refused when it is outside the write root (#3692).
- Read and write access are separate. Give the same folder twice for both; a root left out grants
none of its access (with only `--automation-read-root`, nothing can be written to disk). Without
either flag nothing changes.
docs/releasing.md
+1 -1
View File
@@ -11,7 +11,7 @@ this file has the WordCraft specifics.
| Platform | Artifacts | Signing |
|---|---|---|
| macOS | universal `WordCraft.app` in a `.dmg`, universal CLI zip | Developer ID + notarization (`APPLE_*` secrets) |
| Windows | x64, x86 and arm64 `.msi` + portable `.zip` | Azure Trusted Signing (`AZURE_*` secrets) |
| Windows | x64, x86 and arm64 `.msi` (per-machine, and a per-user `-windows-user-<arch>.msi` that needs no admin rights) + portable `.zip` | Azure Trusted Signing (`AZURE_*` secrets) |
| Linux | x86_64 and aarch64 `.AppImage` (+ `.zsync` for AppImageUpdate), `.flatpak` bundle, `.deb`, `.rpm`, `.tar.gz`; Flathub manifest | checksums |
| Linux riscv64 | `.tar.gz` (cross-compiled on `ubuntu-24.04`, glibc >= 2.39; CLI smoke-tested under QEMU) | checksums |
| FreeBSD | x86_64 `.tar.gz` | checksums |
docs/spelling-dictionaries.md
+5 -3
View File
@@ -1,6 +1,6 @@
# Installing spelling dictionaries
> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-23 · **Change:** minor (Spanish: `es_ES`, `es_MX`, `es_419` and `es_UY` with their fallbacks, #3360, #3553; previously trivial: every Hunspell language shows (no dictionary), not only Polish and Portuguese, and Indonesian is listed, #3150; previously trivial: ISO 8859-1 dictionaries are read, #2824)
> **Last reviewed:** 2026-10-16 · **Last updated:** 2026-10-23 · **Change:** minor (ISO 8859-9 Turkish dictionaries are read, #3692; previously minor: Spanish: `es_ES`, `es_MX`, `es_419` and `es_UY` with their fallbacks, #3360, #3553; previously trivial: every Hunspell language shows (no dictionary), not only Polish and Portuguese, and Indonesian is listed, #3150; previously trivial: ISO 8859-1 dictionaries are read, #2824)
English spelling works out of the box. Polish, Russian, Portuguese, Czech, Italian, Croatian,
Turkish, Indonesian and Spanish spelling use a standard **Hunspell** dictionary (an `.aff` and a `.dic` file) that
@@ -44,8 +44,9 @@ Spanish regions of the Americas (Argentina, Colombia, the United States…) the
American one found. LibreOffice's `dict-es` has them all. Uruguayan Spanish also accepts the
`vos` forms of the dictionary's verbs (`tenés`, `vení`, `decime`, `decímelo`).
The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-1, ISO 8859-2, KOI8-R
and Windows-1251 are read as well; the original VERO `pt_BR` is ISO 8859-1).
The files may be UTF-8 or the encoding named on the `.aff`'s `SET` line (ISO 8859-1, ISO 8859-2,
ISO 8859-9, KOI8-R and Windows-1251 are read as well; the original VERO `pt_BR` is ISO 8859-1, older
Turkish `tr_TR` dictionaries ISO 8859-9).
## Where WordCraft looks
@@ -141,6 +142,7 @@ Never commit dictionary files to this repository: they stay on the user's system
|---|---|---|
| 2026-10-22 | minor | Spanish (#3360, #3553): the file names for Spain, Mexico, Latin America and Uruguay and their fallbacks |
| 2026-10-22 | trivial | Italian, Croatian, Turkish, Russian, Czech and Indonesian without a dictionary now show **(no dictionary)** (or **(dictionary unreadable)**) too, as this page already said; before, only Polish and Portuguese did. Indonesian is listed with its file names (#3150) |
| 2026-10-23 | trivial | ISO 8859-9 (`SET ISO8859-9`) dictionaries are read, so an older Turkish `tr_TR` loads without converting it to UTF-8; the Turkish dictionary is read through the same size-capped reader and `SET` decoding as the others (#3692) |
| 2026-10-18 | trivial | ISO 8859-1 (`SET ISO8859-1`) dictionaries are read, so the original VERO `pt_BR` works without converting it to UTF-8 (#2824) |
| 2026-10-18 | minor | An installed dictionary that can't be read shows **(dictionary unreadable)** in the status bar, with the file and reason in its tooltip and in File › Options › Proofing (#2830) |
| 2026-10-16 | major | First version: the file names per language, the search order and folders on Windows, macOS and Linux, installing `pt_BR` on each, and how to check it works |
docs/zotero.md
+26
View File
@@ -61,6 +61,32 @@ Word's "Field" in the file.)
- Zotero's document transfer (Export/Import Document) isn't needed and isn't supported.
- The web version can't reach Zotero (browsers can't open the socket).
## Who can drive the document
Zotero's protocol has no handshake that proves the program on port 23116 is Zotero, and that
program can read the citations and edit the open document. WordCraft limits who that can be.
In short: **every system** gets the loopback rule; **only Linux** also checks that the listener
isn't another user's. On Windows and macOS (and the BSDs) the loopback rule is the only check.
- Every system: it only connects to this computer (a loopback address such as `127.0.0.1`);
any other address is refused before connecting.
- Linux only: it checks, in the kernel's socket table (`/proc/net/tcp`, `/proc/net/tcp6`), that the
program listening on the port runs as the same user as WordCraft, before sending anything. If
another account on a shared machine took the port while Zotero wasn't running, WordCraft
refuses with an error and the document isn't touched. When the table doesn't show the
listener (WSL2 mirrored networking with Zotero running on Windows, a local proxy, a sandbox)
or can't be read, its owner is unknown and only the loopback rule applies, as below.
- Windows, macOS and the BSDs: **no same-user check.** They have no such table to read
without OS calls WordCraft doesn't make (it uses no `unsafe` code), so only the loopback rule
applies there (and on Linux when the listener isn't in the table). Remaining risk: on a
computer shared with other accounts, a program another user leaves on port 23116 while
Zotero is closed would be taken for Zotero. An owner check for these systems is tracked in
#3831. A program running as you is trusted on every system; it could change your documents
anyway.
- `wordcraft-cli zotero` prints Zotero's messages (and `--trace` lines) with control
characters, line breaks and backslashes shown as escapes (`\n`, `\u{1b}`, `\\`), so a peer
can't forge other output or drive the terminal.
## How it works
`crates/zotero` implements Zotero's published word-processor wire protocol (port 23116):
packaging/windows/package.ps1
+20 -11
View File
@@ -5,6 +5,7 @@
.DESCRIPTION
Produces, in $env:DIST (default: dist/release):
wordcraft-<version>-windows-<arch>.msi per-machine installer (WiX v5)
wordcraft-<version>-windows-user-<arch>.msi per-user installer, no admin rights (#39)
wordcraft-<version>-windows-<arch>-portable.zip wordcraft.exe + wordcraft-cli.exe
The binaries link the C runtime statically (+crt-static), so neither the MSI nor the portable
@@ -142,18 +143,26 @@ $rtf = '{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fcharset0 Tahoma;}}\vi
($paragraphs -join "\par`r`n") + "\par`r`n}`r`n"
[System.IO.File]::WriteAllText($LicenseRtf, $rtf, [System.Text.Encoding]::ASCII)
# Two MSIs from one source: per-machine (Program Files, needs admin) and per-user (#39: no admin,
# into %LOCALAPPDATA%\Programs; see wordcraft.wxs). The per-user name puts "user" before the arch
# so the per-machine MSI sorts first: the shared windows-arm64 workflow installs the first *.msi
# it finds and checks Program Files.
$Msi = Join-Path $Dist "wordcraft-$Version-windows-$Arch.msi"
Invoke-Native 'wix build' {
wix build (Join-Path $PSScriptRoot 'wordcraft.wxs') -arch $Arch `
-ext WixToolset.UI.wixext -ext WixToolset.Util.wixext `
-d "Version=$MsiVersion" -d "BinDir=$Stage" -d "IconPath=$(Join-Path $Root 'assets\app-icon\wordcraft.ico')" `
-d "LicenseRtf=$LicenseRtf" `
-d "AppNoticesDir=$AppNotices" -d "DocsDir=$Docs" -d "HasCraftFontLicences=$HasCraftFontLicences" `
-o $Msi
$UserMsi = Join-Path $Dist "wordcraft-$Version-windows-user-$Arch.msi"
foreach ($build in @(@($Msi, 'perMachine'), @($UserMsi, 'perUserOrMachine'))) {
$out = $build[0]
Invoke-Native "wix build ($($build[1]))" {
wix build (Join-Path $PSScriptRoot 'wordcraft.wxs') -arch $Arch `
-ext WixToolset.UI.wixext -ext WixToolset.Util.wixext `
-d "Version=$MsiVersion" -d "BinDir=$Stage" -d "IconPath=$(Join-Path $Root 'assets\app-icon\wordcraft.ico')" `
-d "LicenseRtf=$LicenseRtf" `
-d "AppNoticesDir=$AppNotices" -d "DocsDir=$Docs" -d "HasCraftFontLicences=$HasCraftFontLicences" -d "Scope=$($build[1])" `
-o $out
}
# wix writes its debug symbols (.wixpdb) next to the MSI; keep them out of the release assets.
Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($out, '.wixpdb'))
& (Join-Path $PSScriptRoot 'sign.ps1') $out
}
# wix writes its debug symbols (.wixpdb) next to the MSI; keep them out of the release assets.
Remove-Item -Force -ErrorAction SilentlyContinue ([IO.Path]::ChangeExtension($Msi, '.wixpdb'))
& (Join-Path $PSScriptRoot 'sign.ps1') $Msi
# ---- portable zip ------------------------------------------------------------------------------
$Portable = Join-Path $TargetDir "windows-package\wordcraft-$Version-windows-$Arch-portable"
@@ -182,4 +191,4 @@ if ($Arch -ne 'arm64' -or $HostArch -eq 'arm64') {
} else {
Write-Output "skipping wordcraft-cli --version: an $Arch build doesn't run on this $HostArch machine"
}
Get-Item $Msi, $Zip | Format-Table Name, Length
Get-Item $Msi, $UserMsi, $Zip | Format-Table Name, Length
Loaded 100 of 101 files, more files were not shown because too many files have changed in this diff. Show more