From a0b60a3b92c44121763f73997e2c21111c72bc7e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:29:38 +0200 Subject: [PATCH 001/282] Optimize: optional target_bytes retries with stronger image settings (#1889) Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 54 +++++++++++++++++++++++++-- crates/automation/src/tools.rs | 3 +- crates/automation/tests/automation.rs | 43 +++++++++++++++++++++ parity/acrobat-features.toml | 4 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 88856305b2..17c1f7293c 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -92,6 +92,8 @@ const DEFAULT_DPI: f64 = 96.0; const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; +/// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. const BOOKMARK_PAGE: usize = 100; @@ -1120,10 +1122,52 @@ impl Automation { }) .collect::>()?, }; - let (bytes, r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let target = match a.opt_int("target_bytes")? { + None => None, + Some(t) if t >= 1 => Some(t as usize), + Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), + }; + let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let mut attempts = 1; + let mut step_used = 0; + if let Some(t) = target { + // Stronger image settings, one rung per attempt, until the copy fits. Every attempt + // starts from the open document, so a failed one changes nothing; the number of + // attempts is fixed. The smallest copy is kept if none fits. + for (k, &(ppi, quality)) in TARGET_LADDER.iter().enumerate() { + if bytes.len() <= t { + break; + } + let mut s = settings.clone(); + for img in [&mut s.color, &mut s.gray] { + img.downsample = true; + img.target_ppi = img.target_ppi.min(ppi); + img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); + img.compression = match img.compression { + Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), + _ => Compression::Jpeg(quality), + }; + } + let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + attempts += 1; + if b2.len() < bytes.len() { + (bytes, r, step_used) = (b2, r2, k + 1); + } + } + } write_atomic(&path, &bytes)?; let o = &r.optimize; - Ok(json!({ + let mut extra = json!({}); + if let Some(t) = target { + let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + extra = json!({ + "target_bytes": t, + "target_reached": bytes.len() <= t, + "attempts": attempts, + "target_settings": { "step": step_used, "max_ppi": ppi, "max_jpeg_quality": quality }, + }); + } + let mut out = json!({ "path": path.to_string_lossy(), "bytes_before": before, "bytes_after": bytes.len(), @@ -1144,7 +1188,11 @@ impl Automation { "unused_xobjects": o.unused_xobjects, "merged_objects": r.merged, "discarded": r.discarded.iter().map(|(h, n)| json!({ "category": h.id(), "count": n })).collect::>(), - })) + }); + if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) { + o.extend(e.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + Ok(out) } /// A watermark or background picture: `file` (an image or a PDF) and `file_page` (1-based). diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..2365086a33 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ @@ -1313,6 +1313,7 @@ pub fn tools() -> Vec { "merge_fragments": { "type": "boolean" }, "color": { "type": "object" }, "gray": { "type": "object" }, + "target_bytes": { "type": "integer", "minimum": 1 }, "discard_thumbnails": { "type": "boolean" }, "discard_alternate_images": { "type": "boolean" }, "discard_tags": { "type": "boolean" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..8018c83321 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4752,6 +4752,49 @@ fn a_page_printed_in_strips_gets_small_through_tools() { assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "preset": "tiny" })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn optimizing_to_a_target_size() { + let dir = workdir("optimize-target"); + let (w, h) = (2400u32, 1600u32); + let px: Vec = (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + [(x * 255 / w) as u8 ^ ((x * y) & 15) as u8, (y * 255 / h) as u8, ((x + y) & 255) as u8] + }) + .collect(); + let mut jpeg = Vec::new(); + let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 95); + enc.set_pixel_density(image::codecs::jpeg::PixelDensity::dpi(600)); + enc.encode(&px, w, h, image::ExtendedColorType::Rgb8).unwrap(); + std::fs::write(dir.join("photo.jpg"), &jpeg).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["photo.jpg"] }))["doc"].as_u64().unwrap(); + // Without a target nothing changes: no target fields. + let base = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "base.pdf" })); + assert!(base.get("target_reached").is_none(), "{base}"); + let base_len = base["bytes_after"].as_u64().unwrap(); + // A target below the default result is met by a stronger step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "fit.pdf", "target_bytes": base_len - 1 })); + assert_eq!(r["target_reached"], true, "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(r["target_settings"]["step"].as_u64().unwrap() >= 1, "{r}"); + assert_eq!(std::fs::metadata(dir.join("fit.pdf")).unwrap().len(), r["bytes_after"].as_u64().unwrap()); + // A target already met by the first attempt tries nothing more. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "easy.pdf", "target_bytes": base_len })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(true), Some(1)), "{r}"); + // An impossible target: bounded attempts, smallest copy written, said plainly. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "tiny.pdf", "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(6)), "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(std::fs::metadata(dir.join("tiny.pdf")).is_ok()); + // The open document is untouched by the failed attempts, and bad targets are refused. + let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); + assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); + assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // Confinement still applies. + assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); +} + #[test] fn initial_view_through_tools() { let dir = workdir("initial-view"); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..aee6fda80f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7260,8 +7260,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced"] tools = ["doc_optimize"] -evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] -notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit" +evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/automation/tests/automation.rs::optimizing_to_a_target_size", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] +notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit; headless-only optional target_bytes retries up to 5 stronger image steps (an automation extension, not an Acrobat feature)" [[feature]] id = "optimize.downsample-images" From 68d7b4cf3b31516672f124e1a8bf65e8df150714 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:20:48 +0200 Subject: [PATCH 002/282] Redact: mark text by caller-supplied regex with optional checksum (#1886) Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 33 ++++ Cargo.toml | 2 + ROADMAP.md | 1 + crates/automation/src/redact.rs | 30 +++- crates/automation/src/tools.rs | 4 +- crates/automation/tests/automation.rs | 28 ++++ crates/engine/src/lib.rs | 1 + crates/redact/Cargo.toml | 1 + crates/redact/src/lib.rs | 1 + crates/redact/src/user_regex.rs | 211 ++++++++++++++++++++++++++ parity/acrobat-features.toml | 12 ++ 11 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 crates/redact/src/user_regex.rs diff --git a/Cargo.lock b/Cargo.lock index 91ed3c4904..268f3c745d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4284,6 +4293,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "regex", "thiserror 2.0.21", ] @@ -5136,6 +5146,29 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.11" diff --git a/Cargo.toml b/Cargo.toml index 597dd316fa..cc0ceaa4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ weezl = "0.2" # foundation thiserror = "2" +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. +regex = { version = "1", default-features = false, features = ["std", "unicode"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 66b9677569..d1e1c6febf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 1b6283c0ad..1367cb819c 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -2,8 +2,8 @@ //! (removing what they cover for good) or clear them. use pdfcraft_engine::{ - Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad, - redact_word_list, + Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator, + RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list, }; use serde_json::{Value, json}; @@ -49,16 +49,20 @@ impl Automation { a.get("find").is_some(), a.get("words").is_some(), a.get("pattern").is_some(), + a.get("regex").is_some(), a.opt_bool("whole_pages")?.unwrap_or(false), ]; if chosen.iter().filter(|c| **c).count() != 1 { - return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into())); + return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into())); } let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false); let whole_words = a.opt_bool("whole_words")?.unwrap_or(false); if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() { return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into())); } + if a.get("validate").is_some() && a.get("regex").is_none() { + return Err(ToolError::InvalidArgs("validate applies only to regex".into())); + } // Per searched word or phrase: how many marks it made. let mut matched_words: Vec<(String, usize)> = Vec::new(); if a.get("rect").is_some() { @@ -79,6 +83,19 @@ impl Automation { Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?), None => None, }; + let user_re = match a.opt_str("regex")? { + Some(src) => { + let validator = match a.opt_str("validate")? { + Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| { + let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect(); + ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", "))) + })?), + None => None, + }; + Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?) + } + None => None, + }; let needles: Vec = match (a.opt_str("find")?, a.get("words")) { (Some(f), _) => vec![f.to_owned()], (None, Some(_)) => { @@ -98,9 +115,10 @@ impl Automation { let texts = self.page_texts(id, &pages)?; let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { - let hits: Vec<(Option, _)> = match pattern { - Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), - None => needles + let hits: Vec<(Option, _)> = match (&user_re, pattern) { + (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), + (None, None) => needles .iter() .enumerate() .flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h))) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..84831b79a4 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -416,7 +416,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -428,6 +428,8 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all)"), "overlay": { "type": "string" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..b1cb1920cb 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -3295,6 +3295,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn redacting_with_a_caller_regex_through_tools() { + let dir = workdir("redact-regex"); + std::fs::write( + dir.join("ids.txt"), + "Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468", + ) + .unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap(); + let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" }))); + assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" }))); + assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_)))); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" })); + assert_eq!(r["marked"].as_u64(), Some(2)); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" })); + assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum"); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}"); + assert!(text.contains("6468"), "{text}"); +} + #[test] fn redacting_with_codes_through_tools() { let dir = workdir("redact-codes"); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 01f162ae7d..22743fdc65 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -152,6 +152,7 @@ pub use pdfcraft_redact::patterns::{ MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list, }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; +pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator}; pub use pdfcraft_sign as sign; pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 49593855e2..f6f99995ac 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +regex = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 61231bd982..fc44d941d1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,6 +27,7 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +pub mod user_regex; use interp::{Mode, Scope, process}; diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs new file mode 100644 index 0000000000..043b51d0b0 --- /dev/null +++ b/crates/redact/src/user_regex.rs @@ -0,0 +1,211 @@ +//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`). +//! +//! The expression is untrusted input. It runs on the `regex` crate, which matches in time +//! linear in the text (no backtracking, no look-around or back-references), and is compiled +//! under hard limits on source length, nesting and compiled size, so a hostile expression is +//! refused with an error instead of exhausting time or memory. An optional [`Validator`] +//! keeps a match only when its checksum holds. + +use std::ops::Range; + +use regex::RegexBuilder; +use thiserror::Error; + +/// Longest expression accepted, in bytes. +pub const MAX_REGEX_LEN: usize = 1024; +/// Most matches returned for one page; the rest are ignored. +pub const MAX_MATCHES_PER_TEXT: usize = 10_000; +/// Cap on the compiled program and on the lazy-DFA cache, in bytes. +const SIZE_LIMIT: usize = 1 << 20; +/// Cap on parenthesis nesting. +const NEST_LIMIT: u32 = 32; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum RegexError { + #[error("regex is empty")] + Empty, + #[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")] + TooLong(usize), + #[error("invalid or too complex regex: {0}")] + Invalid(String), + #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] + MatchesEmpty, +} + +/// Checksum a match must pass to be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validator { + /// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored. + Luhn, + /// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and + /// digits only, spaces and hyphens ignored. + Mod97, + /// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10. + Iban, +} + +/// Ids accepted for `validate`. +pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban]; + +impl Validator { + pub fn id(self) -> &'static str { + match self { + Validator::Luhn => "luhn", + Validator::Mod97 => "mod97", + Validator::Iban => "iban", + } + } + + pub fn from_id(id: &str) -> Option { + VALIDATORS.into_iter().find(|v| v.id() == id) + } + + /// Whether `s` passes the checksum. + pub fn check(self, s: &str) -> bool { + match self { + Validator::Luhn => luhn(s), + Validator::Mod97 => mod97(s.chars()), + Validator::Iban => { + let kept: Vec = s.chars().filter(|c| *c != ' ' && *c != '-').collect(); + let (head, tail) = kept.split_at(kept.len().min(4)); + mod97(tail.iter().chain(head).copied()) + } + } + } +} + +fn luhn(s: &str) -> bool { + let (mut sum, mut digits, mut double) = (0u32, 0usize, false); + for c in s.chars().rev() { + if c == ' ' || c == '-' { + continue; + } + let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false }; + let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d }; + sum = sum.saturating_add(d); + digits += 1; + double = !double; + } + digits >= 2 && sum % 10 == 0 +} + +fn mod97(s: impl Iterator) -> bool { + let (mut rem, mut n) = (0u32, 0usize); + for c in s { + if c == ' ' || c == '-' { + continue; + } + if !c.is_ascii_alphanumeric() { + return false; + } + // Digits stay one decimal digit, letters become two (A = 10 .. Z = 35). + let v = c.to_digit(36).unwrap_or(0); + rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 }; + n += 1; + } + n >= 2 && rem == 1 +} + +/// A compiled caller expression with an optional checksum. +#[derive(Debug, Clone)] +pub struct UserRegex { + re: regex::Regex, + validator: Option, +} + +impl UserRegex { + pub fn new(source: &str, validator: Option) -> Result { + if source.is_empty() { + return Err(RegexError::Empty); + } + if source.len() > MAX_REGEX_LEN { + return Err(RegexError::TooLong(source.len())); + } + let re = RegexBuilder::new(source) + .size_limit(SIZE_LIMIT) + .dfa_size_limit(SIZE_LIMIT) + .nest_limit(NEST_LIMIT) + .build() + .map_err(|e| RegexError::Invalid(e.to_string()))?; + if re.is_match("") { + return Err(RegexError::MatchesEmpty); + } + Ok(Self { re, validator }) + } + + /// Character ranges of `text` the expression matches (non-overlapping, left to right). + pub fn find(&self, text: &[char]) -> Vec> { + let s: String = text.iter().collect(); + // Char index of every byte offset that starts a char (and of the end). + let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; + for (ci, (bi, _)) in s.char_indices().enumerate() { + if let Some(slot) = byte_to_char.get_mut(bi) { + *slot = ci; + } + } + if let Some(slot) = byte_to_char.get_mut(s.len()) { + *slot = text.len(); + } + let mut out = Vec::new(); + for m in self.re.find_iter(&s) { + if out.len() >= MAX_MATCHES_PER_TEXT { + break; + } + if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { + continue; + } + if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end())) + && a < b + && b != usize::MAX + { + out.push(a..b); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chars(s: &str) -> Vec { + s.chars().collect() + } + + #[test] + fn finds_matches_as_char_ranges_with_multibyte_text() { + let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); + let t = chars("Ré 85 05 78 et 91 12 34 fin"); + let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + assert_eq!(found, ["85 05 78", "91 12 34"]); + } + + #[test] + fn validators_filter_matches() { + let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); + let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); + let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); + assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); + } + + #[test] + fn hostile_expressions_are_refused_or_stay_linear() { + assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty); + assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); + assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); + assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); + assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); + assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); + // Classic catastrophic pattern finishes at once on a long non-matching input. + let r = UserRegex::new(r"(a+)+$", None).unwrap(); + let mut t = chars(&"a".repeat(50_000)); + t.push('!'); + assert!(r.find(&t).is_empty()); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..159f580635 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5611,6 +5611,18 @@ tools = ["redact_mark"] evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"] notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded" +[[feature]] +id = "protect.redact-regex" +title = "Redact by caller-supplied regular expression" +area = "G" +tier = "P2" +milestone = "M8" +status = "shipped" +commands = [] +tools = ["redact_mark"] +evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/automation.rs::redacting_with_a_caller_regex_through_tools"] +notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)." + [[feature]] id = "protect.redact-pattern-locales" title = "Pattern sets per locale" From f6e335f3fe0c0ddc00f93b28a8c9f7499bcefa05 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:49:37 +0200 Subject: [PATCH 003/282] UI: allow a half-screen window, and fold the mode bar when narrow (#1913) The viewport minimum was 820x520 points, wider than half of a 1440 px screen. It is now MIN_WINDOW_SIZE = 480x520 points. At that width the five mode tabs were hidden under the search field, so the mode bar folds them into one menu named after the current mode, shrinks the search field and turns it into an icon button when it gets too narrow. Notice bars stack their buttons under the message in a narrow document area. Co-Authored-By: Claude Sonnet 5.5 --- apps/pdfcraft/src/main.rs | 2 +- crates/ui-egui/src/canvas.rs | 16 +++++++++++ crates/ui-egui/src/chrome.rs | 43 +++++++++++++++++++++++++----- crates/ui-egui/src/lib.rs | 6 +++++ crates/ui-egui/src/widgets.rs | 7 +++++ crates/ui-egui/tests/min_window.rs | 38 ++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 crates/ui-egui/tests/min_window.rs diff --git a/apps/pdfcraft/src/main.rs b/apps/pdfcraft/src/main.rs index 7a0bb20432..edc0652d0e 100644 --- a/apps/pdfcraft/src/main.rs +++ b/apps/pdfcraft/src/main.rs @@ -309,7 +309,7 @@ fn native_options(integrated: bool, app_frame: bool, renderer: eframe::Renderer) let mut viewport = egui::ViewportBuilder::default() .with_title("PdfCraft") .with_inner_size([1440.0, 920.0]) - .with_min_inner_size([820.0, 520.0]) + .with_min_inner_size(pdfcraft_ui_egui::MIN_WINDOW_SIZE) .with_drag_and_drop(true) // Wayland app id: matches packaging/linux/ai.storyteller.pdfcraft.desktop. .with_app_id(APP_ID); diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 989b731b7e..72a8316ab7 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4248,8 +4248,24 @@ fn notices( fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, text: String, buttons: impl FnOnce(&mut egui::Ui)) { const ROW: f32 = 28.0; const ICON: f32 = 16.0; + /// Content width below which the buttons stack under the message. + const STACK_BELOW: f32 = 300.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); + if width < STACK_BELOW { + // A narrow document area (a half-width window): the message takes the full width and + // the buttons sit on a row of their own below it, rather than squeezing it to nothing. + ui.with_layout(egui::Layout::left_to_right(egui::Align::Min), |ui| { + let (slot, _) = ui.allocate_exact_size(vec2(ICON, ICON), Sense::hover()); + icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); + ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); + }); + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.set_min_height(ROW); + buttons(ui); + }); + return; + } ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { ui.set_min_height(ROW); buttons(ui); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index e8bf3b48dd..88165102ad 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -487,6 +487,11 @@ fn tab( }) } +/// Width the print, save and info buttons and their gaps take in the mode bar. +const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 2.0 * 4.0 + 8.0; +/// Narrowest search field the mode bar shows; below it the field becomes an icon button. +const MODE_BAR_SEARCH_MIN: f32 = 140.0; + pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); egui::Panel::top("mode_bar") @@ -499,13 +504,32 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(6.0); ui.painter().vline(ui.cursor().left(), ui.max_rect().y_range().shrink(12.0), Stroke::new(1.0, t.divider)); ui.add_space(10.0); - for (mode, label) in - [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")] - { - if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { - app.switch_mode(mode, true); + let modes = + [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. + let avail = ui.available_width(); + let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); + let inline_tabs = avail >= tabs_w + MODE_BAR_RIGHT + MODE_BAR_SEARCH_MIN; + if inline_tabs { + for (mode, label) in modes { + if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { + app.switch_mode(mode, true); + } } + } else { + // Narrow window: the five modes fold into one menu named after the current one. + let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); + let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + egui::Popup::menu(&resp).show(|ui| { + ui.set_min_width(160.0); + for (mode, label) in modes { + if ui.add(egui::Button::selectable(app.mode == mode, tl!(label))).clicked() { + app.switch_mode(mode, true); + } + } + }); } + let left = ui.available_width(); ui.with_layout(Layout::right_to_left(Align::Center), |ui| { ui.spacing_mut().item_spacing.x = 4.0; let has_doc = app.active.is_some(); @@ -524,7 +548,14 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } }); ui.add_space(8.0); - if widgets::search_box(ui, tl!("Find tools and commands"), 260.0).clicked() { + let search_w = (left - MODE_BAR_RIGHT).clamp(0.0, 260.0); + let opened = if search_w >= MODE_BAR_SEARCH_MIN { + widgets::search_box(ui, tl!("Find tools and commands"), search_w).clicked() + } else { + // Too narrow for the field: a button opens the same command search. + icons::button(ui, "search", 32.0, false, tl!("Find tools and commands")).clicked() + }; + if opened { app.palette_open = true; } }); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d101d812e0..832ae30cb4 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -149,6 +149,12 @@ mod open_worker_tests; use pdfcraft_engine::{DocId, Session}; pub use canvas::{DocView, RasterMemory}; + +/// Smallest window the shell is laid out for, in egui points (logical pixels, so the OS scale +/// factor is already applied: 480 points are 600 px at 125 % and 720 px at 150 %). 480 is half of a +/// 1440-wide screen at 150 % scaling, the tightest common half-screen split on Windows; the mode +/// bar, tab strip and notice bars adapt below the old 820 minimum (#1913). +pub const MIN_WINDOW_SIZE: [f32; 2] = [480.0, 520.0]; pub use editing::{CloseRequest, SaveTarget}; pub use files::{ExtractDraft, FilePurpose, FileRequest, RotateDraft, SplitDraft, SplitMode, SplitPlan}; pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore}; diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index 4f5aaa79b6..cae7405ac5 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -108,6 +108,7 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { let t = Tokens::get(ui.ctx()); let font = if active { theme::semibold(13.5) } else { theme::medium(13.5) }; let w = ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), font.clone(), t.text).size().x); + // Keep in step with `mode_tab_width`. let (rect, resp) = ui.allocate_exact_size(vec2(w + 22.0, 48.0), Sense::click()); resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, ui.is_enabled(), active, label)); let press = theme::Press::track(ui, &resp); @@ -128,6 +129,12 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { theme::hand(resp) } +/// Width [`mode_tab`] takes for `label` (the semibold, active, form is the widest). +pub fn mode_tab_width(ui: &egui::Ui, label: &str) -> f32 { + let t = Tokens::get(ui.ctx()); + ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), theme::semibold(13.5), t.text).size().x) + 22.0 +} + /// Rounded pill button; `primary` fills with the accent. pub fn pill_button(ui: &mut egui::Ui, label: &str, primary: bool) -> Response { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs new file mode 100644 index 0000000000..b06e81d12d --- /dev/null +++ b/crates/ui-egui/tests/min_window.rs @@ -0,0 +1,38 @@ +//! The smallest window (#1913): at `MIN_WINDOW_SIZE` the mode bar folds the five modes into a menu +//! and keeps the command search reachable, instead of hiding them under the search field. + +use egui::accesskit::Role; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_ui_egui::{MIN_WINDOW_SIZE, PdfCraftApp}; + +fn harness(size: egui::Vec2) -> Harness<'static, PdfCraftApp> { + let mut h = Harness::builder().with_size(size).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app + }); + h.run_steps(8); + h +} + +#[test] +fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { + let mut h = harness(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + assert!(h.query_by_role_and_label(Role::Button, "Read").is_none(), "no room for five tabs"); + assert!(h.query_by_role_and_label(Role::Button, "Find tools and commands").is_some(), "command search is reachable"); + h.get_by_role_and_label(Role::Button, "All tools").click(); + h.run_steps(4); + h.get_by_role_and_label(Role::Button, "E-Sign").click(); + h.run_steps(4); + assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); + assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); +} + +#[test] +fn a_wide_window_keeps_the_five_mode_tabs() { + let h = harness(egui::vec2(1400.0, 900.0)); + for mode in ["All tools", "Read", "Edit", "Convert", "E-Sign"] { + assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); + } +} From 03a4c21cfe7f73b890ee4d93a204fe38ad2c9e11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:40:36 +0200 Subject: [PATCH 004/282] UI: stack notice buttons only below 200 pt so the 900-wide notice keeps its layout (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 72a8316ab7..ca1ff12337 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4249,7 +4249,7 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te const ROW: f32 = 28.0; const ICON: f32 = 16.0; /// Content width below which the buttons stack under the message. - const STACK_BELOW: f32 = 300.0; + const STACK_BELOW: f32 = 200.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); if width < STACK_BELOW { From c3cf27558a6d0f6c8c245b5a4bad813e6ae5de30 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:53:33 +0200 Subject: [PATCH 005/282] UI: wrap stacked notice buttons and scroll the rail in short windows (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 3 +- crates/ui-egui/src/chrome.rs | 241 +++++++++++++++-------------- crates/ui-egui/tests/min_window.rs | 82 ++++++++++ 3 files changed, 211 insertions(+), 115 deletions(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index ca1ff12337..349b4deded 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4260,7 +4260,8 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); }); - ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + // The buttons wrap onto further rows when even the full width is too narrow for them all. + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center).with_main_wrap(true), |ui| { ui.set_min_height(ROW); buttons(ui); }); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 88165102ad..d4c6f09503 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,6 +648,9 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } +/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 316.0; + pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); let Some((index, id)) = app.active_ids() else { return }; @@ -668,122 +671,132 @@ pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { .exact_size(48.0) .frame(egui::Frame::NONE.fill(t.chrome).inner_margin(egui::Margin::symmetric(6, 8)).stroke(Stroke::new(1.0, t.divider))) .show(ui, |ui| { - ui.spacing_mut().item_spacing.y = 4.0; - let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { - let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); - let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); - if has && !selected { - let c = r.rect.right_top() + vec2(-8.0, 8.0); - ui.painter().circle_filled(c, 3.0, t.accent); - } - if r.clicked() { - app.choose_right_panel(if selected { None } else { Some(panel) }); - } - }; - rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); - rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); - rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); - rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); - rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); - rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); - rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); - if has_check { - rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); - } - rail_button(ui, RightPanel::Tags, "tag", "Tags", false); + let panel_h = ui.available_height(); + // A window too short for both clusters scrolls the rail instead of overlapping them (#1913). + egui::ScrollArea::vertical().auto_shrink(false).scroll_bar_visibility(egui::scroll_area::ScrollBarVisibility::AlwaysHidden).show( + ui, + |ui| { + ui.spacing_mut().item_spacing.y = 4.0; + let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { + let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); + let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); + if has && !selected { + let c = r.rect.right_top() + vec2(-8.0, 8.0); + ui.painter().circle_filled(c, 3.0, t.accent); + } + if r.clicked() { + app.choose_right_panel(if selected { None } else { Some(panel) }); + } + }; + rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); + rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); + rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); + rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); + rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); + rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); + rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); + if has_check { + rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); + } + rail_button(ui, RightPanel::Tags, "tag", "Tags", false); - // Page navigation cluster at the bottom (as in Acrobat's rail). - let view = &mut app.views[index]; - let mut page_display = None; - ui.with_layout(Layout::bottom_up(Align::Center), |ui| { - ui.spacing_mut().item_spacing.y = 2.0; - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); - if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { - view.zoom_step(false); - } - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); - if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { - view.zoom_step(true); - } - // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. - let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); - let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); - page_display = egui::Popup::menu(&resp) - .show(|ui| { - ui.set_min_width(220.0); - rail_view_menu(ui, view) - }) - .and_then(|r| r.inner); - let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); - if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { - view.rotate_view(true); - } - // Not `columns-2` while fitting the page: that is the two-page view's icon. - let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; - if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { - view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; - view.goto = Some((view.current, 0.0)); - } - let zoom = format!("{:.0}%", view.zoom * 100.0); - ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { - ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); - }); - ui.add_space(6.0); - if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { - view.step_page(true); - } - if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { - view.step_page(false); - } - // History is distinct from stepping to the adjacent page, and belongs to this tab. - for (forward, icon, label, shortcut, enabled) in [ - (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), - (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), - ] { - let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); - if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { - view.view_history(forward); + // Page navigation cluster at the bottom (as in Acrobat's rail). + let view = &mut app.views[index]; + let mut page_display = None; + let cluster_w = ui.available_width(); + // Inside the scroll area `available_height` is unbounded: the panel's height was taken outside. + let cluster_h = (panel_h - ui.min_rect().height() - ui.spacing().item_spacing.y).max(RAIL_BOTTOM_H); + ui.allocate_ui_with_layout(vec2(cluster_w, cluster_h), Layout::bottom_up(Align::Center), |ui| { + ui.spacing_mut().item_spacing.y = 2.0; + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); + if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { + view.zoom_step(false); + } + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); + if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { + view.zoom_step(true); + } + // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. + let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); + let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); + page_display = egui::Popup::menu(&resp) + .show(|ui| { + ui.set_min_width(220.0); + rail_view_menu(ui, view) + }) + .and_then(|r| r.inner); + let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); + if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { + view.rotate_view(true); + } + // Not `columns-2` while fitting the page: that is the two-page view's icon. + let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; + if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { + view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; + view.goto = Some((view.current, 0.0)); + } + let zoom = format!("{:.0}%", view.zoom * 100.0); + ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { + ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); + }); + ui.add_space(6.0); + if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { + view.step_page(true); + } + if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { + view.step_page(false); + } + // History is distinct from stepping to the adjacent page, and belongs to this tab. + for (forward, icon, label, shortcut, enabled) in [ + (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), + (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), + ] { + let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); + if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { + view.view_history(forward); + } + } + ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); + let page_input_id = view.page_input_id(); + // Consume replacement selection once, including when the editor has lost focus. + let reselect = std::mem::take(&mut view.page_input_reselect); + // Navigation updates the idle display, never the text being edited. + if !ui.memory(|m| m.has_focus(page_input_id)) { + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } else if reselect { + // The replacement changed the label while egui retained this editor's cursor. + let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); + let end = egui::text::CCursor::new(view.page_input.chars().count()); + state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); + state.store(ui.ctx(), page_input_id); + } + let edit = egui::TextEdit::singleline(&mut view.page_input) + .id(page_input_id) + .desired_width(34.0) + .horizontal_align(Align::Center) + .font(theme::medium(12.0)) + .margin(vec2(2.0, 4.0)); + let r = egui::Frame::NONE + .fill(t.field) + .stroke(Stroke::new(1.0, t.border)) + .corner_radius(CornerRadius::same(5)) + .show(ui, |ui| ui.add(edit)) + .inner + .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); + if r.lost_focus() { + if ui.input(|i| i.key_pressed(egui::Key::Enter)) { + // A page label first (logical page numbers, as Acrobat), then a number. + let typed = view.page_input.clone(); + view.go_to_typed(&typed, &labels); + } + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } + }); + if let Some(id) = page_display { + app.execute(id); } - } - ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); - let page_input_id = view.page_input_id(); - // Consume replacement selection once, including when the editor has lost focus. - let reselect = std::mem::take(&mut view.page_input_reselect); - // Navigation updates the idle display, never the text being edited. - if !ui.memory(|m| m.has_focus(page_input_id)) { - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } else if reselect { - // The replacement changed the label while egui retained this editor's cursor. - let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); - let end = egui::text::CCursor::new(view.page_input.chars().count()); - state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); - state.store(ui.ctx(), page_input_id); - } - let edit = egui::TextEdit::singleline(&mut view.page_input) - .id(page_input_id) - .desired_width(34.0) - .horizontal_align(Align::Center) - .font(theme::medium(12.0)) - .margin(vec2(2.0, 4.0)); - let r = egui::Frame::NONE - .fill(t.field) - .stroke(Stroke::new(1.0, t.border)) - .corner_radius(CornerRadius::same(5)) - .show(ui, |ui| ui.add(edit)) - .inner - .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); - if r.lost_focus() { - if ui.input(|i| i.key_pressed(egui::Key::Enter)) { - // A page label first (logical page numbers, as Acrobat), then a number. - let typed = view.page_input.clone(); - view.go_to_typed(&typed, &labels); - } - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } - }); - if let Some(id) = page_display { - app.execute(id); - } + }, + ); }); } diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index b06e81d12d..889d952520 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -36,3 +36,85 @@ fn a_wide_window_keeps_the_five_mode_tabs() { assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); } } + +fn fixture(n: usize) -> Vec { + let mut objs: Vec = vec!["<< /Type /Catalog /Pages 2 0 R >>".into()]; + let kids: Vec = (0..n).map(|i| format!("{} 0 R", 4 + 2 * i)).collect(); + objs.push(format!("<< /Type /Pages /Kids [{}] /Count {n} /MediaBox [0 0 200 300] >>", kids.join(" "))); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into()); + for i in 0..n { + objs.push(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i)); + let body = format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1); + objs.push(format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len())); + } + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn protected(user: &str, owner: &str, permissions: i32) -> Vec { + protected_with(pdfcraft_cos::Algorithm::Aes256, user, owner, permissions) +} + +fn protected_with(algorithm: pdfcraft_cos::Algorithm, user: &str, owner: &str, permissions: i32) -> Vec { + let mut doc = pdfcraft_cos::Document::open(std::sync::Arc::new(fixture(2))).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm, + user_password: user, + owner_password: owner, + permissions, + encrypt_metadata: true, + seed: [4; 32], + }) + .unwrap(); + pdfcraft_cos::write_full(&doc, &Default::default()).unwrap() +} + +/// A narrow window: the stacked notice bar keeps both buttons inside the window and apart. +#[test] +fn the_stacked_notice_keeps_its_buttons_in_view() { + let mut h = Harness::builder().with_size(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("locked.pdf", None, protected("", "owner", 0b0100)).unwrap(); + app + }); + h.run_steps(8); + let window = egui::Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + let msg = h.get_by_label_contains("This document is secured").rect(); + let (unlock, settings) = (h.get_by_label("Unlock…").rect(), h.get_by_label("Security settings").rect()); + for (name, r) in [("unlock", unlock), ("settings", settings)] { + assert!(window.contains_rect(r), "{name} stays inside the window: {r:?}"); + assert!(r.left() >= msg.left() - 30.0, "{name} is not cut off at the bar's left edge: {r:?} vs {msg:?}"); + assert!(r.top() >= msg.bottom() - 0.5 || r.bottom() <= msg.top() + 0.5, "{name} does not cover the message"); + } + assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); +} + +/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +#[test] +fn a_short_window_does_not_overlap_the_rail_clusters() { + let h = { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("a.pdf", None, fixture(2)).unwrap(); + app + }); + h.run_steps(8); + h + }; + let tags = h.get_by_label("Tags").rect(); + let next = h.get_by_label("Previous page").rect(); + assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); +} From fb7f7252dbc470ef5736295da94134f2c3e5146f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 15 Oct 2026 23:33:25 +0200 Subject: [PATCH 006/282] UI: size the rail's bottom cluster for the history buttons, fix a pixel-fragile flick test (#1913) The rail's page-navigation cluster measures 360 pt since the two view-history buttons landed; RAIL_BOTTOM_H was 316, so in a short window the cluster overflowed its allocation and overlapped. The min-window test now uses the minimum height and checks the page box, history and zoom buttons. a_quick_flick_on_either_edge_rewraps_the_paragraph grabbed the widened edge 2 px outside the box, at the limit of the grab zone; the rail's scroll area moved the canvas width by 3 px and the press missed. It now grabs 1 px outside. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 4 ++-- crates/ui-egui/tests/editing.rs | 4 +++- crates/ui-egui/tests/min_window.rs | 12 ++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index d4c6f09503..51f269dc43 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,8 +648,8 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } -/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. -const RAIL_BOTTOM_H: f32 = 316.0; +/// Height the rail's page-navigation cluster (zoom, fit, page, history buttons, page box) needs (360 pt measured); below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 364.0; pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/editing.rs b/crates/ui-egui/tests/editing.rs index f8db525751..cef7214485 100644 --- a/crates/ui-egui/tests/editing.rs +++ b/crates/ui-egui/tests/editing.rs @@ -3487,7 +3487,9 @@ fn a_quick_flick_on_either_edge_rewraps_the_paragraph() { assert_eq!(text, ["Page", "1"], "narrowed from the right edge"); // Flick the right edge outward, past the box and its handle: back onto one line. let b = block(&h); - let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(2.0, 0.0); + // 1 px outside the box: the grab zone's outer limit sits near 2 px, where a pixel of page + // scale (the canvas width depends on the rail) decides whether the press lands on the handle. + let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(1.0, 0.0); flick(&mut h, edge, edge + egui::vec2(80.0 * k, 0.0)); let text: Vec = lines(&h).iter().map(|l| l.text.clone()).collect(); assert_eq!(text, ["Page 1"], "widened by a fast drag that left the handle at once"); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 889d952520..8d28c6aff6 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -101,11 +101,11 @@ fn the_stacked_notice_keeps_its_buttons_in_view() { assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); } -/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +/// #1913 (the minimum height): the rail's top buttons and its page-navigation cluster must not overlap. #[test] fn a_short_window_does_not_overlap_the_rail_clusters() { let h = { - let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { let mut app = PdfCraftApp::new(); app.set_option("language", "en").unwrap(); app.open_bytes("a.pdf", None, fixture(2)).unwrap(); @@ -117,4 +117,12 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { let tags = h.get_by_label("Tags").rect(); let next = h.get_by_label("Previous page").rect(); assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); + // The history buttons are the cluster's topmost buttons (above the page box). + let history = h.get_by_label_contains("Previous view").rect(); + assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); + // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. + let page_box = h.get_by_role(Role::TextInput).rect(); + assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + let zoom_out = h.get_by_label_contains("Zoom out").rect(); + assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From d6d4999a355325e701506df85318ddf50f1689dd Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 19 Oct 2026 02:39:14 +0200 Subject: [PATCH 007/282] Test: the rail now has two text fields (zoom and page box) After merging main's editable zoom field, get_by_role(TextInput) in the min-window test found two nodes. Check every text field of the rail sits below the rail buttons. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/min_window.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 8d28c6aff6..005c97dc5f 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -120,9 +120,11 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { // The history buttons are the cluster's topmost buttons (above the page box). let history = h.get_by_label_contains("Previous view").rect(); assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); - // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. - let page_box = h.get_by_role(Role::TextInput).rect(); - assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + // The cluster holds two text fields, the zoom field and the page box; the page box is the topmost thing in it, the lowest is the zoom-out button. + for field in h.get_all_by_role(Role::TextInput) { + let r = field.rect(); + assert!(r.top() >= tags.bottom(), "the rail's text fields sit below the rail buttons: {tags:?} vs {r:?}"); + } let zoom_out = h.get_by_label_contains("Zoom out").rect(); assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From 8239f9c4eda76ed126ddf2f063544cc5a7819289 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 20 Oct 2026 00:55:51 +0200 Subject: [PATCH 008/282] Optimize target_bytes: keep zip/retain image encoding on retries, only tighten ppi Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 11 +++++++---- crates/automation/src/tools.rs | 2 +- crates/automation/tests/automation.rs | 8 ++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 17c1f7293c..a279856fb5 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -93,6 +93,7 @@ const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; /// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +/// The quality applies only to images the caller left on JPEG; zip and retain keep their encoding. const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. @@ -1143,10 +1144,10 @@ impl Automation { img.downsample = true; img.target_ppi = img.target_ppi.min(ppi); img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); - img.compression = match img.compression { - Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), - _ => Compression::Jpeg(quality), - }; + // Only a caller who chose JPEG gets a lower quality; zip and retain stay as asked. + if let Compression::Jpeg(q) = img.compression { + img.compression = Compression::Jpeg(q.min(quality)); + } } let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; attempts += 1; @@ -1160,6 +1161,8 @@ impl Automation { let mut extra = json!({}); if let Some(t) = target { let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + let lossy = matches!(settings.color.compression, Compression::Jpeg(_)) || matches!(settings.gray.compression, Compression::Jpeg(_)); + let quality = quality.filter(|_| lossy); extra = json!({ "target_bytes": t, "target_reached": bytes.len() <= t, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 2365086a33..550fa119d8 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 8018c83321..b6e5305a69 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4791,6 +4791,14 @@ fn optimizing_to_a_target_size() { let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // A caller who asked for lossless images keeps them: retries only tighten the ppi, no JPEG quality. + let lossless = json!({ "compression": "zip" }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "zip.pdf", "color": lossless, "gray": lossless, "target_bytes": 1 })); + assert_eq!(r["attempts"].as_u64(), Some(6), "{r}"); + assert!(r["target_settings"]["max_ppi"].is_number() && r["target_settings"]["max_jpeg_quality"].is_null(), "{r}"); + // A JPEG caller gets the quality step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); + assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); // Confinement still applies. assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); } From 86af84171f004d1e06564a01152406bf4d997ff4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:11:43 +0200 Subject: [PATCH 009/282] doc_optimize: reject a target_bytes too large for usize Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 53e99e008d..ffde1dfab3 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -1308,7 +1308,7 @@ impl Automation { }; let target = match a.opt_int("target_bytes")? { None => None, - Some(t) if t >= 1 => Some(t as usize), + Some(t) if t >= 1 => Some(usize::try_from(t).map_err(|_| ToolError::InvalidArgs(format!("target_bytes {t} is too large")))?), Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), }; let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; From 0f64d9589ed5469490bd5d176ce47101f2c1d45a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:45:07 +0200 Subject: [PATCH 010/282] Run rustfmt on the mode bar after the upstream merge Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index b08c48518d..e5b239ecbb 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -507,8 +507,7 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(10.0); // Acrobat's mode bar has no Read tab: reading is what the viewer does with no tool // in use. Read mode (the panes hidden) stays a View command and a default workspace. - let modes = - [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + let modes = [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. let avail = ui.available_width(); let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); From 589c4af8e7138aa22eb51322353835d2ab73b772 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 25 Oct 2026 10:19:17 +0100 Subject: [PATCH 011/282] Redact regex: fail when a page exceeds the match cap UserRegex::find now returns an error instead of silently stopping at 10,000 matches, so redact_mark cannot report success while later matches stay readable. Empty-match detection probes several short texts so patterns like \b are refused with the clear message. Schema notes that the 1024 limit is in bytes. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/redact.rs | 19 ++++++++++++- crates/automation/src/tools.rs | 4 +-- crates/redact/src/user_regex.rs | 48 ++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 840020c570..808b6eabb7 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -126,7 +126,24 @@ impl Automation { let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { let hits: Vec<(Option, _)> = match (&user_re, pattern) { - (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (Some(re), _) => { + // A page with too many matches fails the call: marking only the first ones + // would leave the rest readable after redact_apply. + let too_many = std::cell::Cell::new(false); + let found = text.find_with(|chars| { + re.find(chars).unwrap_or_else(|_| { + too_many.set(true); + Vec::new() + }) + }); + if too_many.get() { + return Err(ToolError::InvalidArgs(format!( + "regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked", + p + 1 + ))); + } + found.into_iter().map(|h| (None, h)).collect() + } (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), (None, None) => needles .iter() diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4c651c00dd..0a41cacc0a 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -486,7 +486,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -498,7 +498,7 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, - "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." }, "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs index 043b51d0b0..9d1c1b0004 100644 --- a/crates/redact/src/user_regex.rs +++ b/crates/redact/src/user_regex.rs @@ -13,7 +13,8 @@ use thiserror::Error; /// Longest expression accepted, in bytes. pub const MAX_REGEX_LEN: usize = 1024; -/// Most matches returned for one page; the rest are ignored. +/// Most matches accepted for one page; one more makes [`UserRegex::find`] fail, so a page is never +/// marked only in part. pub const MAX_MATCHES_PER_TEXT: usize = 10_000; /// Cap on the compiled program and on the lazy-DFA cache, in bytes. const SIZE_LIMIT: usize = 1 << 20; @@ -30,6 +31,8 @@ pub enum RegexError { Invalid(String), #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] MatchesEmpty, + #[error("regex has more than {MAX_MATCHES_PER_TEXT} matches on one page; narrow it so that every match can be marked")] + TooManyMatches, } /// Checksum a match must pass to be kept. @@ -106,6 +109,12 @@ fn mod97(s: impl Iterator) -> bool { n >= 2 && rem == 1 } +/// Whether `re` can match the empty string, which `is_match("")` alone misses for assertions such +/// as `\b`. Tries a few short texts with letters, digits, space, newline and a non-ASCII letter. +fn can_match_empty(re: ®ex::Regex) -> bool { + ["", " ", "a", "1", "a a", "a\n", "\n", "é", "aa 11"].iter().any(|p| re.find_iter(p).any(|m| m.is_empty())) +} + /// A compiled caller expression with an optional checksum. #[derive(Debug, Clone)] pub struct UserRegex { @@ -127,14 +136,17 @@ impl UserRegex { .nest_limit(NEST_LIMIT) .build() .map_err(|e| RegexError::Invalid(e.to_string()))?; - if re.is_match("") { + if can_match_empty(&re) { return Err(RegexError::MatchesEmpty); } Ok(Self { re, validator }) } /// Character ranges of `text` the expression matches (non-overlapping, left to right). - pub fn find(&self, text: &[char]) -> Vec> { + /// + /// Fails with [`RegexError::TooManyMatches`] when more than [`MAX_MATCHES_PER_TEXT`] matches + /// are kept, instead of returning a partial list the caller would take for complete. + pub fn find(&self, text: &[char]) -> Result>, RegexError> { let s: String = text.iter().collect(); // Char index of every byte offset that starts a char (and of the end). let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; @@ -148,9 +160,6 @@ impl UserRegex { } let mut out = Vec::new(); for m in self.re.find_iter(&s) { - if out.len() >= MAX_MATCHES_PER_TEXT { - break; - } if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { continue; } @@ -158,10 +167,13 @@ impl UserRegex { && a < b && b != usize::MAX { + if out.len() >= MAX_MATCHES_PER_TEXT { + return Err(RegexError::TooManyMatches); + } out.push(a..b); } } - out + Ok(out) } } @@ -177,7 +189,7 @@ mod tests { fn finds_matches_as_char_ranges_with_multibyte_text() { let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); let t = chars("Ré 85 05 78 et 91 12 34 fin"); - let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + let found: Vec = r.find(&t).unwrap().into_iter().map(|g| t[g].iter().collect()).collect(); assert_eq!(found, ["85 05 78", "91 12 34"]); } @@ -185,10 +197,10 @@ mod tests { fn validators_filter_matches() { let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); } @@ -199,13 +211,25 @@ mod tests { assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); - assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + for empty in ["a*", r"\b", "(?m)^", r"\B", "(?m)$", r"\d*|x"] { + assert_eq!(UserRegex::new(empty, None).unwrap_err(), RegexError::MatchesEmpty, "{empty}"); + } + assert!(UserRegex::new(r"\bfoo\b", None).is_ok()); assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); // Classic catastrophic pattern finishes at once on a long non-matching input. let r = UserRegex::new(r"(a+)+$", None).unwrap(); let mut t = chars(&"a".repeat(50_000)); t.push('!'); - assert!(r.find(&t).is_empty()); + assert!(r.find(&t).unwrap().is_empty()); + } + + #[test] + fn more_than_the_cap_of_matches_is_an_error_not_a_partial_list() { + let r = UserRegex::new("x", None).unwrap(); + let at_cap = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT)); + assert_eq!(r.find(&at_cap).unwrap().len(), MAX_MATCHES_PER_TEXT); + let over = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT + 1)); + assert_eq!(r.find(&over).unwrap_err(), RegexError::TooManyMatches); } } From 90fb9b8c9e37adb0663bacca887248672f68d8e4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 19:18:16 +0100 Subject: [PATCH 012/282] Test the menu bar's File item instead of the removed Menu button Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/min_window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/min_window.rs b/crates/ui-egui/tests/it/min_window.rs index fc395e7cdf..f384eef0eb 100644 --- a/crates/ui-egui/tests/it/min_window.rs +++ b/crates/ui-egui/tests/it/min_window.rs @@ -26,7 +26,7 @@ fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { h.get_by_role_and_label(Role::Button, "E-Sign").click(); h.run_steps(4); assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); - assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); + assert!(h.query_by_role_and_label(Role::MenuItem, "File").is_some(), "main menu reachable"); } #[test] From 88649a8955fa674adcaa35af48340de74209e294 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 20:35:42 +0100 Subject: [PATCH 013/282] Name Read mode in the folded mode menu The narrow-window mode menu fell back to "All tools" while Read was active. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index f831d26d9c..5e1fef8860 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -505,8 +505,13 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } } else { // Narrow window: the modes fold into one menu named after the current one. - let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); - let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + // Read is not one of the four choices but can be active (View command): name it too. + let current = if app.mode == Mode::Read { + tl!("Read") + } else { + tl!(modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l)) + }; + let resp = widgets::ghost_button(ui, "chevron-down", current); egui::Popup::menu(&resp).show(|ui| { ui.set_min_width(160.0); for (mode, label) in modes { From fe03703a3b71a92c96a1d5318b3ad4dd9d02ab88 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:55:16 +0100 Subject: [PATCH 014/282] perf(cos): keep a running total of the scan reader's cache size ObjectReader::try_get measured every parsed object in its cache (up to 128) after each read to check the byte budget. Keep a running total instead and measure the whole cache only when something other than one freshly parsed object changed it. Eviction timing is unchanged, so output is identical. Fixes #4867 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..74b773e026 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -445,6 +445,10 @@ pub struct Document { pub(crate) struct ObjectReader { pub(crate) document: Document, decoded_count: usize, + /// Objects in the parsed-object cache and the bytes of their own, as last measured. Kept + /// as a running total so a read does not measure the whole cache again. + counted_objects: usize, + counted_bytes: usize, } impl ObjectReader { @@ -488,8 +492,25 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))) > Self::STREAM_BYTES { + // The common read parsed exactly this object and nothing else (a hit changes + // nothing): add its size to the running total. Anything else, such as a failed + // load, a /Length that cached further objects or a cache changed from outside, + // measures the whole cache again. + let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); + if object.is_ok() && cache.len() == self.counted_objects { + // Unchanged: a hit, or an object held by the overlay. + } else if let (true, Ok(read)) = (grew_by_this_object, &object) { + self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); + self.counted_objects = cache.len(); + } else { + self.counted_bytes = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))); + self.counted_objects = cache.len(); + } + if cache.len() >= Self::OBJECTS || self.counted_bytes > Self::STREAM_BYTES { cache.clear(); + self.counted_objects = 0; + self.counted_bytes = 0; } } object @@ -1176,7 +1197,7 @@ impl Document { let mut document = self.clone(); document.cache = Arc::default(); document.objstms = Arc::default(); - ObjectReader { document, decoded_count: 0 } + ObjectReader { document, decoded_count: 0, counted_objects: 0, counted_bytes: 0 } } fn load(&self, num: u32, depth: u32) -> Result { @@ -3004,11 +3025,27 @@ endobj // Data of its own (decrypted streams, strings) counts: past the budget the cache is // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned)); + reader.document.cache.lock().unwrap().insert(5, Arc::new(owned)); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } + #[test] + fn temporary_reader_running_total_matches_the_cache() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>".to_owned(), "<< /Type /Pages /Kids [] /Count 0 >>".to_owned()]; + bodies.extend((0..300).map(|i| format!("<< /Value {i} /Name ({}) >>", "s".repeat(i % 40)))); + let refs: Vec<_> = bodies.iter().map(String::as_str).collect(); + let doc = Document::open(Arc::new(build(&refs, "/Root 1 0 R"))).unwrap(); + let mut reader = doc.object_reader(); + for num in (1..=302).chain([5, 301, 99_999, 7]) { + let _ = reader.try_get(ObjRef::new(num, 0)); + let cache = reader.document.cache.lock().unwrap(); + let measured = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &reader.document.data, 0))); + assert_eq!((reader.counted_objects, reader.counted_bytes), (cache.len(), measured), "after object {num}"); + assert!(cache.len() < ObjectReader::OBJECTS); + } + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From be8345aa3f2d9930201e766a4c8a8f315a48f8d6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:57:29 +0100 Subject: [PATCH 015/282] perf(cos): rebuild a damaged cross-reference index without parsing every object Reconstruct parsed every object to see whether it is an object stream, and searched each object's next 1,024 bytes and the whole file byte by byte. Only objects whose dictionary holds ObjStm or a # (a name can hide letters as #xx) are now parsed for that check. Every /XRef position is found once, and find() scans for the needle's rarest byte in 32-byte blocks. The index, trailer and repair log are unchanged. Fixes #4868 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 72 ++++++++++++++++++++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/reconstruct.rs | 57 +++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 crates/cos/tests/it/reconstruct.rs diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..4ad24a11b1 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2052,10 +2052,29 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - for (num, offset, generation, body) in object_headers(&data) { + // Objects whose bytes (the last object of a number wins, as in `entries`) could hold an + // `/ObjStm` type; the others are not parsed for the object-stream check below. + let mut candidates: HashSet = HashSet::new(); + let headers = object_headers(&data); + // Where `/XRef` occurs in the file, found in one pass. + let mut xref_marks = Vec::new(); + let mut mark_from = 0; + while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { + xref_marks.push(p); + mark_from = p + 1; + } + for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if find(&data, b"/XRef", body, body.saturating_add(1024)).is_some() + let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); + if may_be_object_stream(&data, body, end) { + candidates.insert(num); + } else { + candidates.remove(&num); + } + // `/XRef` within the 1024 bytes after the header (the first one that is there). + let first = xref_marks.partition_point(|&p| p < body); + if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) && let Ok(at) = usize::try_from(offset) { xref_streams.push(at); @@ -2068,7 +2087,8 @@ impl Document { self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); // Objects inside object streams. for num in objstms { - if let Ok(o) = self.try_get(num) + if candidates.contains(&num) + && let Ok(o) = self.try_get(num) && let Object::Stream(s) = &*o && s.dict.name(b"Type") == Some(b"ObjStm") && let Ok(stm) = self.objstm(num) @@ -2165,6 +2185,24 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } +/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary +/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's +/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. +fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { + let Some(body) = data.get(from..to.min(data.len())) else { return false }; + let mut dict = body; + let mut at = 0; + while let Some(p) = find(body, b"stream", at, body.len()) { + at = p + 6; + let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); + if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { + dict = &body[..p]; + break; + } + } + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() +} + /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position /// after `obj`). fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { @@ -2241,7 +2279,33 @@ pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option< if from >= to || needle.is_empty() || to - from < needle.len() { return None; } - hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + let hay = &hay[from..to]; + // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. + const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; + let (k, rare) = + needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); + let mut i = k; + while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { + let start = i + p - k; + if hay.get(start..start + needle.len()) == Some(needle) { + return Some(start + from); + } + i += p + 1; + } + None +} + +/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which +/// the compiler turns into vector compares; only a block that holds the byte is searched. +fn position_of(hay: &[u8], byte: u8) -> Option { + const BLOCK: usize = 32; + let (blocks, rest) = hay.as_chunks::(); + for (i, block) in blocks.iter().enumerate() { + if block.iter().fold(false, |any, &b| any | (b == byte)) { + return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); + } + } + rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) } fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..07883e7b5a 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod reconstruct; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs new file mode 100644 index 0000000000..5f61867b1b --- /dev/null +++ b/crates/cos/tests/it/reconstruct.rs @@ -0,0 +1,57 @@ +//! Rebuilding the cross-reference index of a damaged file finds the objects inside object +//! streams however the stream's type is written. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; + +/// A file whose `startxref` points nowhere, with three object streams (type written plainly, as +/// `/Obj#53tm`, and after a string that contains the word `stream`), and one ordinary stream +/// whose data mentions `ObjStm` and `#`. +fn damaged() -> Vec { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut add = |n: u32, body: &[u8]| { + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let objstm = |dict_prefix: &str, members: &[(u32, &str)]| { + let mut header = String::new(); + let mut body = String::new(); + for (n, text) in members { + header.push_str(&format!("{n} {} ", body.len())); + body.push_str(text); + body.push('\n'); + } + let mut b = + format!("<< {dict_prefix} /N {} /First {} /Length {} >>\nstream\n", members.len(), header.len(), header.len() + body.len()).into_bytes(); + b.extend_from_slice(header.as_bytes()); + b.extend_from_slice(body.as_bytes()); + b.extend_from_slice(b"\nendstream"); + b + }; + add(1, b"<< /Type /Catalog /Pages 2 0 R >>"); + add(2, b"<< /Type /Pages /Kids [] /Count 0 >>"); + add(3, &objstm("/Type /ObjStm", &[(10, "<< /From 3 >>"), (11, "[3]")])); + add(4, &objstm("/Type /Obj#53tm", &[(12, "<< /From 4 >>")])); + add(5, &objstm("/Note (a stream) /Type /ObjStm", &[(13, "<< /From 5 >>")])); + let data = b"/Type /ObjStm # 14 0 << /From 6 >>"; + let mut plain = format!("<< /Length {} >>\nstream\n", data.len()).into_bytes(); + plain.extend_from_slice(data); + plain.extend_from_slice(b"\nendstream"); + add(6, &plain); + out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); + out +} + +#[test] +fn objects_in_object_streams_are_found_however_the_type_is_written() { + let doc = Document::open(Arc::new(damaged())).expect("opens"); + assert!(doc.reconstructed()); + for n in [10, 11, 12, 13] { + let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); + assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); + } + // The data of an ordinary stream is not an object stream, whatever it says. + assert!(!doc.object_numbers().contains(&14)); +} From a7baf8a2af97eeed166325a990bb116cdf3e1d9b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:59:12 +0100 Subject: [PATCH 016/282] perf(cos): write_full keeps the objects it parsed while numbering write_full parsed each object once to number it and again, after the 128-object reader cache had dropped it, to write it. Keep the parsed object from the traversal and write that, within a budget of 64 MiB of owned bytes and 2M tree nodes; past it the object is read again as before, which keeps the memory bound and the "read differently while saving" check. Fixes #4869 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 2 +- crates/cos/src/writer.rs | 89 +++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..9dca286d7e 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -530,7 +530,7 @@ const OWNED_BYTES_DEPTH: usize = 256; /// Bytes `object` holds of its own: string and stream data that is not a view into `data` /// (the input file). An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured /// and counts as too large to keep. -fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { +pub(crate) fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { if depth > OWNED_BYTES_DEPTH { return usize::MAX; } diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..1ba4574318 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -15,7 +15,7 @@ use std::io::Write; use std::sync::Arc; use crate::CosError; -use crate::document::{Document, ObjectReader, XrefEntry}; +use crate::document::{Document, ObjectReader, XrefEntry, owned_bytes}; use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; /// Serialize one object. Deterministic: same object → same bytes. @@ -280,6 +280,10 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. let mut order: Vec<(ObjRef, bool)> = Vec::new(); + // The parsed object of each entry in `order`, kept for the write pass while the budget + // allows it. Past the budget the write pass reads the object again. + let mut kept: Vec>> = Vec::new(); + let mut kept_budget = KeptObjects::default(); let mut queue: VecDeque = VecDeque::new(); let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); for key in [&b"Root"[..], b"Info", b"Encrypt"] { @@ -305,6 +309,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro map.insert(r, output_number(order.len())?); order.push((r, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); + kept.push(kept_budget.admit(&o, reader.document.bytes()).then_some(o)); } let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; @@ -329,14 +334,14 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut rows: BTreeMap = BTreeMap::new(); // Keep only references, not renamed copies of the entire object graph. Standalone // objects still precede object streams, preserving the established output order. - let mut packed: Vec<(u32, ObjRef)> = Vec::new(); + let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); for (i, (r, is_stream)) in order.iter().enumerate() { let num = output_number(i)?; let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(r) && root_outside != Some(*r); if in_stream { - packed.push((num, *r)); + packed.push((num, *r, kept[i].take())); } else { - let o = renumber(&*reread(&mut reader, *r, *is_stream)?, &map); + let o = renumber(&*written_object(&mut reader, *r, *is_stream, kept[i].take())?, &map); rows.insert(num, Row::InFile(out.len() as u64, 0)); write_indirect(num, 0, &prepared(&reader.document, r.num, num, 0, &o), &mut out); } @@ -345,9 +350,9 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro // individually: the whole stream is (§7.5.7, §7.6.2). let mut next = output_number(order.len())?; let mut batch = ObjectStreamBatch::default(); - for (num, reference) in packed { + for (num, reference, parsed) in packed { let mut encoded = Vec::new(); - serialize(&renumber(&*reread(&mut reader, reference, false)?, &map), &mut encoded); + serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; } @@ -400,6 +405,56 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro Ok(out) } +/// What `write_full` keeps of the objects it parsed while numbering them: owned bytes (see +/// [`owned_bytes`]; streams read from the file are views and cost only their dictionary) and +/// tree nodes. Beyond either limit objects are read again when written, which bounds the memory +/// of a save of a huge document. +#[derive(Default)] +struct KeptObjects { + bytes: usize, + nodes: usize, +} + +impl KeptObjects { + const BYTES: usize = 64 * 1024 * 1024; + const NODES: usize = 2_000_000; + + /// Whether `object` still fits the budget; if so it is counted against it. + fn admit(&mut self, object: &Object, data: &Arc>) -> bool { + let bytes = self.bytes.saturating_add(owned_bytes(object, data, 0)); + let nodes = self.nodes.saturating_add(node_count(object, 0)); + if bytes > Self::BYTES || nodes > Self::NODES { + return false; + } + (self.bytes, self.nodes) = (bytes, nodes); + true + } +} + +/// Values in `object`'s tree, counting the object itself. Past the parser's nesting limit +/// the count is `usize::MAX`, so nothing that deep is kept. +fn node_count(object: &Object, depth: usize) -> usize { + if depth > 256 { + return usize::MAX; + } + let children = match object { + Object::Stream(s) => s.dict.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + Object::Array(a) => a.iter().fold(0usize, |n, v| n.saturating_add(node_count(v, depth + 1))), + Object::Dict(d) => d.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + _ => 0, + }; + children.saturating_add(1) +} + +/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read +/// again. +fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { + match kept { + Some(o) => Ok(o), + None => reread(reader, r, is_stream), + } +} + /// Object `r` read again for writing. The traversal read it once and only kept its number, so /// a second read that fails, finds nothing or finds a different kind of object must fail the /// save rather than write `null` in its place. @@ -674,6 +729,28 @@ mod tests { Lexer::new(&out, 0).object().expect("parses") } + #[test] + fn kept_objects_stop_at_their_byte_and_node_budget() { + let data = Arc::new(Vec::new()); + let string = Object::String(PdfString::literal(vec![b'x'; 1000])); + let array = Object::Array(vec![Object::Int(1), Object::Null]); + let mut kept = KeptObjects::default(); + assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); + assert_eq!((kept.bytes, kept.nodes), (1000, 4)); + let mut full = KeptObjects { bytes: KeptObjects::BYTES - 999, nodes: 0 }; + assert!(!full.admit(&string, &data), "one byte over the byte budget"); + assert_eq!((full.bytes, full.nodes), (KeptObjects::BYTES - 999, 0), "a refused object is not counted"); + assert!(full.admit(&Object::Int(1), &data)); + let mut full = KeptObjects { bytes: 0, nodes: KeptObjects::NODES - 2 }; + assert!(!full.admit(&array, &data), "a node past the node budget"); + assert!(full.admit(&Object::Int(1), &data)); + let mut deep = Object::Null; + for _ in 0..300 { + deep = Object::Array(vec![deep]); + } + assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); + } + #[test] fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { let mut doc = Document::new_empty(); From 929d350f1dfacb6da308569f2a7de8d412a4049a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:00:43 +0100 Subject: [PATCH 017/282] perf(cos): write hex strings, integers and references without the formatter serialize wrote every byte of a hex string, every integer and every object number through write!. A table lookup and a small digit loop give the same bytes in a fraction of the instructions. The octal escape in literal strings and the #xx escape in names use the same helpers. Fixes #4870 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/writer.rs | 53 ++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/serialize_format.rs | 110 ++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 crates/cos/tests/it/serialize_format.rs diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..b8a9c53e9c 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -23,9 +23,7 @@ pub fn serialize(o: &Object, out: &mut Vec) { match o { Object::Null => out.extend_from_slice(b"null"), Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), - Object::Int(i) => { - let _ = write!(out, "{i}"); - } + Object::Int(i) => write_int(*i, out), Object::Real(r) => write_real(*r, out), Object::String(s) => write_string(s, out), Object::Name(n) => write_name(n, out), @@ -42,11 +40,46 @@ pub fn serialize(o: &Object, out: &mut Vec) { Object::Dict(d) => write_dict(d, out), Object::Stream(s) => write_stream(s, out), Object::Ref(r) => { - let _ = write!(out, "{} {} R", r.num, r.generation); + write_digits(u64::from(r.num), out); + out.push(b' '); + write_digits(u64::from(r.generation), out); + out.extend_from_slice(b" R"); } } } +/// Upper-case hexadecimal digits. +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. +fn write_digits(mut v: u64, out: &mut Vec) { + // `u64::MAX` has 20 digits, so `at` never goes below 0. + let mut buf = [0u8; 20]; + let mut at = buf.len(); + loop { + at -= 1; + buf[at] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { + break; + } + } + out.extend_from_slice(&buf[at..]); +} + +/// A signed integer in decimal, as `write!(out, "{i}")` writes it. +fn write_int(i: i64, out: &mut Vec) { + if i < 0 { + out.push(b'-'); + } + write_digits(i.unsigned_abs(), out); +} + +/// `b` as two upper-case hexadecimal digits. +fn hex_pair(b: u8) -> [u8; 2] { + [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] +} + fn write_real(r: f64, out: &mut Vec) { if !r.is_finite() { out.push(b'0'); @@ -55,7 +88,7 @@ fn write_real(r: f64, out: &mut Vec) { // A whole number is written as an integer only within the integer range Annex C gives // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { - let _ = write!(out, "{}", r as i32); + write_int(i64::from(r as i32), out); return; } // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the @@ -76,7 +109,8 @@ fn write_name(n: &[u8], out: &mut Vec) { if regular { out.push(b); } else { - let _ = write!(out, "#{b:02X}"); + out.push(b'#'); + out.extend_from_slice(&hex_pair(b)); } } } @@ -84,9 +118,10 @@ fn write_name(n: &[u8], out: &mut Vec) { fn write_string(s: &PdfString, out: &mut Vec) { let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); if s.hex || binary * 4 > s.bytes.len().max(1) { + out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); out.push(b'<'); - for b in &s.bytes { - let _ = write!(out, "{b:02X}"); + for &b in &s.bytes { + out.extend_from_slice(&hex_pair(b)); } out.push(b'>'); return; @@ -105,7 +140,7 @@ fn write_string(s: &PdfString, out: &mut Vec) { 0x0C => out.extend_from_slice(b"\\f"), b' '..=b'~' => out.push(b), _ => { - let _ = write!(out, "\\{b:03o}"); + out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); } } } diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..4b2fe32560 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod serialize_format; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs new file mode 100644 index 0000000000..19781a802f --- /dev/null +++ b/crates/cos/tests/it/serialize_format.rs @@ -0,0 +1,110 @@ +//! `serialize` writes integers, references, hexadecimal strings, octal escapes and `#xx` name +//! escapes with its own digit loops. They must give the bytes the standard formatter gives. + +use pdfcraft_cos::{ObjRef, Object, PdfString, serialize}; +use proptest::prelude::*; + +fn bytes_of(o: &Object) -> Vec { + let mut v = Vec::new(); + serialize(o, &mut v); + v +} + +#[test] +fn integer_extremes_and_digit_boundaries() { + let mut cases = vec![0i64, 1, -1, 9, 10, 99, 100, i64::MAX, i64::MIN, i64::MIN + 1, i64::from(i32::MAX), i64::from(i32::MIN)]; + let mut p = 1i64; + while let Some(next) = p.checked_mul(10) { + cases.extend([next - 1, next, next + 1, -next]); + p = next; + } + for i in cases { + assert_eq!(bytes_of(&Object::Int(i)), i.to_string().into_bytes(), "{i}"); + } +} + +#[test] +fn reference_extremes() { + for (n, g) in [(0, 0), (1, 0), (u32::MAX, u16::MAX), (10, 65535), (4_000_000_000, 7)] { + assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } +} + +#[test] +fn every_byte_value_in_hex_strings_octal_escapes_and_names() { + let all: Vec = (0..=255).collect(); + let hex: String = all.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(bytes_of(&Object::String(PdfString { bytes: all.clone(), hex: true })), format!("<{hex}>").into_bytes()); + // A literal string with a few control bytes escapes them in octal. + let lit = bytes_of(&Object::String(PdfString { bytes: b"abcdefghijkl\x01\x7f\xffz".to_vec(), hex: false })); + assert_eq!(lit, b"(abcdefghijkl\\001\\177\\377z)"); + let name = bytes_of(&Object::Name(all.clone())); + let mut want = b"/".to_vec(); + for &b in &all { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + want.push(b); + } else { + want.extend_from_slice(format!("#{b:02X}").as_bytes()); + } + } + assert_eq!(name, want); +} + +proptest! { + #[test] + fn integers_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Int(i)), format!("{i}").into_bytes()); + } + + #[test] + fn whole_reals_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); + } + + #[test] + fn references_match_the_formatter(n in any::(), g in any::()) { + prop_assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } + + #[test] + fn hex_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + let mut want = String::from("<"); + for b in &bytes { + want.push_str(&format!("{b:02X}")); + } + want.push('>'); + prop_assert_eq!(bytes_of(&Object::String(PdfString { bytes, hex: true })), want.into_bytes()); + } + + #[test] + fn binary_literal_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + // Mostly-binary input is written as hex; mostly-text input with escapes is a literal. + let got = bytes_of(&Object::String(PdfString { bytes: bytes.clone(), hex: false })); + let binary = bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + let mut want = Vec::new(); + if binary * 4 > bytes.len().max(1) { + want.push(b'<'); + for b in &bytes { + want.extend_from_slice(format!("{b:02X}").as_bytes()); + } + want.push(b'>'); + } else { + want.push(b'('); + for &b in &bytes { + match b { + b'(' | b')' | b'\\' => want.extend_from_slice(&[b'\\', b]), + b'\n' => want.extend_from_slice(b"\\n"), + b'\r' => want.extend_from_slice(b"\\r"), + b'\t' => want.extend_from_slice(b"\\t"), + 0x08 => want.extend_from_slice(b"\\b"), + 0x0C => want.extend_from_slice(b"\\f"), + b' '..=b'~' => want.push(b), + _ => want.extend_from_slice(format!("\\{b:03o}").as_bytes()), + } + } + want.push(b')'); + } + prop_assert_eq!(got, want); + } +} From d04fad3acd79906445a47e46496fb09fbfffdc11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:45:53 +0100 Subject: [PATCH 018/282] fix(cos): recount the scan reader's cache when a read replaces entries The running total trusted cache length and pointer equality to mean that a read only added the requested object. A read whose nested loads made the editor cache evict (it then keeps only the newcomer) leaves the length unchanged, so a large string stayed cached past the 8 MiB budget. Count the inserts a read makes instead; anything but a hit or exactly one insert measures the whole cache again. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..b7fe54bc19 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -21,6 +21,13 @@ thread_local! { static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); } +thread_local! { + /// Objects this thread has put into a document's parsed-object cache. A reader compares + /// it before and after a read: a read that inserted exactly one object cannot have + /// evicted or replaced anything else, whatever the cache length says. + static CACHE_INSERTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream /// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is /// bounded as well. @@ -460,7 +467,9 @@ impl ObjectReader { } pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { + let inserts_before = CACHE_INSERTS.with(|c| c.get()); let object = self.document.try_get(reference.num); + let inserted = CACHE_INSERTS.with(|c| c.get()).wrapping_sub(inserts_before); // A missed eviction here only costs a re-parse, so never fail the read on a poisoned // lock. The editing-cache evictors propagate instead; they guard user edits. if let Ok(mut streams) = self.document.objstms.lock() { @@ -492,13 +501,16 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - // The common read parsed exactly this object and nothing else (a hit changes - // nothing): add its size to the running total. Anything else, such as a failed - // load, a /Length that cached further objects or a cache changed from outside, - // measures the whole cache again. - let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + // A hit inserted nothing; a read that inserted exactly this object (no nested + // /Length or object stream load) added only it. Cache length and pointer equality + // alone cannot tell that from a read that evicted and replaced entries (a large + // nested object makes the editor cache keep only the newcomer), so count the + // inserts. Anything else, such as a failed load, a /Length that cached further + // objects or a cache changed from outside, measures the whole cache again. + let grew_by_this_object = inserted == 1 + && cache.len() == self.counted_objects.saturating_add(1) && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); - if object.is_ok() && cache.len() == self.counted_objects { + if object.is_ok() && inserted == 0 && cache.len() == self.counted_objects { // Unchanged: a hit, or an object held by the overlay. } else if let (true, Ok(read)) = (grew_by_this_object, &object) { self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); @@ -1075,6 +1087,7 @@ impl Document { // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); + CACHE_INSERTS.with(|c| c.set(c.get().wrapping_add(1))); self.evict_editor_cache_if_over_budget(num, &obj)?; Ok(obj) } @@ -3046,6 +3059,22 @@ endobj } } + #[test] + fn temporary_reader_recounts_when_a_read_replaces_cache_entries() { + // One cached object, then a read of a string over the editor cache's byte budget: the + // editor eviction keeps only the newcomer, so the cache length stays at one. The + // string is data of its own and must not stay cached past the reader's budget. + let big = format!("({})", "a".repeat(Document::EDITOR_CACHE_BYTES + 1)); + let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Small 1 >>", &big], "/Root 1 0 R"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let mut reader = doc.object_reader(); + let _ = reader.get(ObjRef::new(3, 0)); + assert_eq!((reader.document.cache.lock().unwrap().len(), reader.counted_objects), (1, 1)); + assert!(matches!(&*reader.get(ObjRef::new(4, 0)), Object::String(_))); + assert!(reader.document.cache.lock().unwrap().is_empty(), "a replaced entry must be counted: the string is over budget"); + assert_eq!((reader.counted_objects, reader.counted_bytes), (0, 0)); + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From 8bdd2d37504689fbff50eb198740d384a6a2c054 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:46:44 +0100 Subject: [PATCH 019/282] fix(cos): read the dictionary's syntax before the object-stream pre-check The pre-check cut the dictionary at the next object header or at a '>>' + 'stream' byte pattern, so text inside a string could hide a later /Type /ObjStm. It now reads the dictionary as the parser does (strings, hex strings, comments, nesting) and treats an object whose end is not certain as a candidate, so the reconstructed object set stays the same. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 173 ++++++++++++++++++++++++++--- crates/cos/tests/it/reconstruct.rs | 7 +- 2 files changed, 162 insertions(+), 18 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 4ad24a11b1..ffc4f6be72 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2063,11 +2063,10 @@ impl Document { xref_marks.push(p); mark_from = p + 1; } - for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { + for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); - if may_be_object_stream(&data, body, end) { + if may_be_object_stream(&data, body) { candidates.insert(num); } else { candidates.remove(&num); @@ -2185,22 +2184,107 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } -/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary -/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's -/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. -fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { - let Some(body) = data.get(from..to.min(data.len())) else { return false }; - let mut dict = body; - let mut at = 0; - while let Some(p) = find(body, b"stream", at, body.len()) { - at = p + 6; - let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); - if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { - dict = &body[..p]; - break; +/// Longest dictionary [`may_be_object_stream`] reads before it gives up and says "maybe". +const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; + +/// Where the dictionary that starts at `data[from..]` (after `N G obj`) ends, reading it as the +/// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the +/// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end +/// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must +/// then treat the object as a possible object stream. +fn dictionary_end(data: &[u8], from: usize) -> Option> { + let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT)); + let mut i = from; + while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { + if data[i] == b'%' { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } else { + i += 1; + } + } + if data.get(i..i + 2) != Some(b"<<") { + return if i < limit { Some(None) } else { None }; + } + i += 2; + let (mut dicts, mut arrays) = (1usize, 0usize); + while i < limit { + match data[i] { + b'(' => { + let mut depth = 1usize; + i += 1; + while i < limit && depth > 0 { + match data[i] { + b'\\' => i += 1, + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + i += 1; + } + if depth > 0 { + return None; + } + } + b'<' if data.get(i + 1) == Some(&b'<') => { + dicts += 1; + i += 2; + } + b'<' => { + i += 1; + while i < limit && data[i] != b'>' { + if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { + return None; + } + i += 1; + } + i += 1; + } + b'>' if data.get(i + 1) == Some(&b'>') => { + if arrays > 0 { + return None; + } + dicts -= 1; + i += 2; + if dicts == 0 { + return Some(Some(i)); + } + } + b'%' => { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } + b'[' => { + arrays += 1; + i += 1; + } + b']' => { + arrays = arrays.saturating_sub(1); + i += 1; + } + _ => i += 1, + } + } + None +} + +/// Whether the object whose body starts at `data[from..]` could be an object stream: its +/// dictionary holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in +/// `/Obj#53tm`). Only the dictionary is searched, not the data of a stream. Where the +/// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a +/// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a +/// candidate, so a doubtful object is parsed as it always was. +fn may_be_object_stream(data: &[u8], from: usize) -> bool { + match dictionary_end(data, from) { + Some(None) => false, + None => true, + Some(Some(end)) => { + let dict = &data[from..end]; + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } } - dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position @@ -3584,6 +3668,61 @@ endobj /// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used /// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after /// editing one Info field dropped every other. + /// Whether parsing `body` as an object gives a stream whose /Type is /ObjStm. + fn parses_as_object_stream(file: &[u8]) -> bool { + matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) + } + + #[test] + fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { + // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a + // string that comes before the /Type. + for dict in [ + "<< /Note (9 0 obj\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (>>stream\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (a (nested\n9 0 obj\nstream) \\) )) /Type /ObjStm >>", + "<< /Note <3e3e> /Other << /A (>>\nstream\n) >> /Type /ObjStm >>", + ] { + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); + assert!(may_be_object_stream(&file, 7), "{dict}"); + } + // Ordinary streams stay out; their data is never read. + let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; + assert!(!may_be_object_stream(plain, 7)); + assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7)); + // Unterminated: not certain, so a candidate. + assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7)); + // Random dictionaries built from tricky pieces: whatever parses as an object stream is + // a candidate. + let pieces: [&str; 22] = [ + "/Type /ObjStm", "/Type/ObjStm", "/Type /Obj#53tm", "/T#79pe /ObjStm", "(", ")", "\\", "(>>stream\n)", "(9 0 obj\n)", "<3e3e>", "<", ">", ">>", + "<<", "[", "]", "% c\n", "% >>\n", "/N 0", " ", "\n", "", + ]; + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut hits = 0; + for _ in 0..200_000 { + let mut dict = String::from("<<"); + for _ in 0..(next() % 9) { + dict.push_str(pieces[(next() % pieces.len() as u64) as usize]); + dict.push(' '); + } + dict.push_str(if next() % 8 == 0 { "" } else { ">>" }); + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + if parses_as_object_stream(&file) { + hits += 1; + assert!(may_be_object_stream(&file, 7), "rejected an object stream: {dict:?}"); + } + } + assert!(hits > 1000, "the generator must produce object streams ({hits})"); + } + #[test] fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { let table = build( diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs index 5f61867b1b..4611c0d17e 100644 --- a/crates/cos/tests/it/reconstruct.rs +++ b/crates/cos/tests/it/reconstruct.rs @@ -40,6 +40,11 @@ fn damaged() -> Vec { plain.extend_from_slice(data); plain.extend_from_slice(b"\nendstream"); add(6, &plain); + // Strings before the type that look like an object header, or like the end of the + // dictionary and the `stream` keyword, and a hex string and a comment. + add(7, &objstm("/Note (\n99 0 obj\n) /Type /ObjStm", &[(15, "<< /From 7 >>")])); + add(8, &objstm("/Note (>>stream\n) /Type /ObjStm", &[(16, "<< /From 8 >>")])); + add(9, &objstm("/Note <3e3e> /Other << /A (a (b) \\) >>\nstream\n) >> % >>\nstream\n\n/Type /ObjStm", &[(17, "<< /From 9 >>")])); out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); out } @@ -48,7 +53,7 @@ fn damaged() -> Vec { fn objects_in_object_streams_are_found_however_the_type_is_written() { let doc = Document::open(Arc::new(damaged())).expect("opens"); assert!(doc.reconstructed()); - for n in [10, 11, 12, 13] { + for n in [10, 11, 12, 13, 15, 16, 17] { let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); } From ce12a1278876f14792d0fea578773353af6c1c5f Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:36:37 +0200 Subject: [PATCH 020/282] Exports keep a wrapped paragraph as one paragraph Reading splits a line wherever its style changes, and export joined those blocks along one line only (#4158). A sentence with a bold name in one line and an italic name in the next came out of Word, HTML, RTF and PowerPoint as one paragraph per line. A paragraph whose last line reaches the right edge of the page's text now runs on into the line just below it, when that line starts at the same left edge in the same size. Headings, list items, a line that ends a sentence, an indented line and a line after paragraph space keep their break. A slanted run whose box leans into the next word by up to 0.3 of its size still runs on into it. The new export and automation tests fail without the change. --- crates/automation/tests/it/automation.rs | 71 ++++++++++++ crates/export/src/flow.rs | 135 +++++++++++++++++++++++ crates/export/src/layout.rs | 28 ++++- crates/export/src/lib.rs | 11 +- crates/export/src/pptx.rs | 3 +- crates/export/src/tests.rs | 135 +++++++++++++++++++++++ parity/acrobat-features.toml | 15 +-- 7 files changed, 381 insertions(+), 17 deletions(-) create mode 100644 crates/export/src/flow.rs diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 6a5765b911..4d3aa44f01 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -7614,6 +7614,77 @@ BT /F1 12 Tf 72 640 Td (A paragraph further down.) Tj ET"; assert_eq!(paragraphs.len(), 2, "{xml}"); } +/// A one-page PDF (612 x 792 pt) drawing `content`, with the standard fonts `fonts` as /F1, /F2 +/// and so on, each in its built-in encoding. +fn standard_fonts_pdf(content: &str, fonts: &[&str]) -> Vec { + let names: String = (1..=fonts.len()).map(|i| format!("/F{i} {} 0 R ", i + 4)).collect(); + let mut objs = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << {names}>> >> >>"), + format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), + ]; + objs.extend(fonts.iter().map(|f| format!("<< /Type /Font /Subtype /Type1 /BaseFont /{f} >>"))); + let mut pdf = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + pdf.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + pdf +} + +/// The text of each paragraph that holds any in `word/document.xml` of the .docx at `path`. +fn word_paragraph_texts(path: &Path) -> Vec { + let zip = std::fs::read(path).unwrap(); + let mut at = 0; + let mut xml = String::new(); + while zip[at..].starts_with(b"PK\x03\x04") { + let size = u32::from_le_bytes(zip[at + 18..at + 22].try_into().unwrap()) as usize; + let len = u16::from_le_bytes(zip[at + 26..at + 28].try_into().unwrap()) as usize; + let data = at + 30 + len; + if &zip[at + 30..data] == b"word/document.xml" { + std::io::Read::read_to_string(&mut flate2::read::DeflateDecoder::new(&zip[data..data + size]), &mut xml).unwrap(); + break; + } + at = data + size; + } + xml.split("") + .skip(1) + .map(|p| { + let p = p.split_once("").map_or(p, |(inside, _)| inside); + p.split("").skip(1).filter_map(|t| t.split_once("").map(|(t, _)| t)).collect::() + }) + .filter(|text| !text.is_empty()) + .collect() +} + +/// #4158: a sentence that wraps onto the next lines after a bold name and a slanted one reaches +/// Word as one paragraph, though reading splits each of its lines where the style changes. +#[test] +fn exporting_to_word_keeps_a_wrapped_sentence_in_one_paragraph() { + let dir = workdir("wrapped-sentence-word"); + let content = "BT /F1 10 Tf 72 700 Td (The parties to this lease are the owner, ) Tj /F2 10 Tf (Harbour Lane Properties) Tj \ +/F1 10 Tf (, of 4 Quay Street, and the) Tj ET \ +BT /F1 10 Tf 72 686 Td (tenant named in the schedule, ) Tj /F3 10 Tf (Blue Finch Studio) Tj \ +/F1 10 Tf (, who agree as follows to rent the upper) Tj ET \ +BT /F1 10 Tf 72 672 Td (rooms for the whole of the term.) Tj ET \ +BT /F1 10 Tf 72 640 Td (Rent is paid monthly in advance.) Tj ET"; + std::fs::write(dir.join("lease.pdf"), standard_fonts_pdf(content, &["Helvetica", "Helvetica-Bold", "Helvetica-Oblique"])).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "lease.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_office", json!({ "doc": doc, "path": "lease.docx" })); + let sentence = "The parties to this lease are the owner, Harbour Lane Properties, of 4 Quay Street, and the tenant named in the \ +schedule, Blue Finch Studio, who agree as follows to rent the upper rooms for the whole of the term."; + assert_eq!(word_paragraph_texts(&dir.join("lease.docx")), [sentence, "Rent is paid monthly in advance."]); +} + /// Hundreds of calculations run on opening and again after a change through the tools. The /// time limits are raised so that a busy machine does not stop them (the engine's tests check /// that the limits stop them). diff --git a/crates/export/src/flow.rs b/crates/export/src/flow.rs new file mode 100644 index 0000000000..e064772864 --- /dev/null +++ b/crates/export/src/flow.rs @@ -0,0 +1,135 @@ +//! Text that runs on from one line to the next. +//! +//! The reading blocks keep style boundaries (#4158), so a sentence with a bold name or an italic +//! word in it reaches the exporter as several blocks, and `layout::paragraphs` joins only the ones +//! that run on along one line. A sentence that wraps onto the next line after +//! such a run came out as one paragraph per line. [`join_wrapped`] joins a paragraph whose last +//! line reaches the right edge of the page's text with the one that starts just below it at the +//! same left edge, in the same size: the text wrapped there. Headings, list items, a line that +//! ends a sentence and a line with paragraph space after it stay apart. + +use super::layout::Run; + +/// Bullets that mark a list item wherever they stand, including the Symbol and Wingdings bullets +/// Word writes into its PDFs (U+F0B7, U+F0A7). +const BULLETS: [char; 11] = ['•', '◦', '▪', '‣', '⁃', '●', '○', '■', '□', '\u{F0B7}', '\u{F0A7}']; + +/// Dashes and a middle dot: list markers at the start of a line, punctuation inside one. +const DASHES: [char; 4] = ['-', '\u{2013}', '\u{2014}', '·']; + +/// A run taller than this many times its font size is more than one line: a block of several +/// rows ends where its widest row does, not where its last one does. +const ONE_LINE: f64 = 1.6; + +/// How far (as a multiple of the font size) the space between two lines may be from none, and +/// they still follow each other within a paragraph: more is paragraph spacing. +const LINE_GAP: f64 = 0.5; + +/// How close to the right edge of the page's text a full line ends: within this many times the +/// font size, or this fraction of the text's width, whichever is more (a ragged line ends short +/// of the edge by up to a long word). +const FULL_LINE: (f64, f64) = (3.0, 0.12); + +/// How far (points) the first line of a paragraph may start from the left edge of the line above +/// it and still be at the same left edge. +const SAME_LEFT: f64 = 2.0; + +/// Whether two boxes ([x0, y0, x1, y1]) share a line: they overlap vertically by more than half +/// the shorter one, as `layout::continued_from` has it. +fn on_one_line(a: [f64; 4], b: [f64; 4]) -> bool { + a[3].min(b[3]) - a[1].max(b[1]) > 0.5 * (a[3] - a[1]).min(b[3] - b[1]) +} + +/// Whether `text` opens like a list item: a bullet, a dash or `*`; one to three digits followed +/// by `.` or `)`; or one letter followed by `)` ("a)", "а)"); then a space or nothing. "1.5 kg", +/// "i.e." and "2026. Year" are text. +pub(crate) fn list_item(text: &str) -> bool { + let t = text.trim_start(); + let Some(first) = t.chars().next() else { return false }; + if BULLETS.contains(&first) || DASHES.contains(&first) || first == '*' { + return true; + } + let digits = t.chars().take_while(char::is_ascii_digit).count(); + // ASCII digits are one byte each and `first` is one whole character, so both cuts fall on + // character boundaries. + let (rest, marks): (&str, &[char]) = if (1..=3).contains(&digits) { + (t.get(digits..).unwrap_or_default(), &['.', ')']) + } else if digits == 0 && first.is_alphabetic() { + (t.get(first.len_utf8()..).unwrap_or_default(), &[')']) + } else { + return false; + }; + let mut after = rest.chars(); + after.next().is_some_and(|c| marks.contains(&c)) && after.next().is_none_or(char::is_whitespace) +} + +/// Whether a line's text ends a sentence (closing quotes and brackets aside). Text set without +/// paragraph spacing gives no other sign that a paragraph ends there, so such a line is left +/// apart from the next rather than joined to it wrongly. +fn ends_sentence(text: &str) -> bool { + text.trim_end().trim_end_matches(['"', '\'', '”', '’', '“', '»', ')', ']']).ends_with(['.', '!', '?', ':', ';', '…']) +} + +/// `paras` (a page's paragraphs, in any order) top to bottom, with each paragraph that wraps onto +/// the next line joined to it. `level` is a paragraph's heading level (0 for body text). +pub(crate) fn join_wrapped<'a>(mut paras: Vec>>, level: impl Fn(&[Run<'a>]) -> u8) -> Vec>> { + // The left and right edges of the page's text: where a full line starts and ends. + let (left, right) = paras + .iter() + .flatten() + .map(|r| r.block.rect) + .filter(|r| r[0].is_finite() && r[2].is_finite()) + .fold((f64::INFINITY, f64::NEG_INFINITY), |(l, r), x| (l.min(x[0]), r.max(x[2]))); + if right <= left { + return paras; + } + // Top to bottom, then left to right, by each paragraph's first run. + paras.sort_by(|a, b| { + let top_left = |p: &[Run]| p.first().map_or([0.0; 2], |r| [r.block.rect[3], r.block.rect[0]]); + let (a, b) = (top_left(a), top_left(b)); + b[0].total_cmp(&a[0]).then(a[1].total_cmp(&b[1])) + }); + // Each paragraph with its heading level. A joined paragraph is body text: only body text joins. + let mut out: Vec<(Vec>, u8)> = Vec::with_capacity(paras.len()); + for para in paras { + let lvl = level(¶); + match out.last_mut() { + Some((prev, 0)) if lvl == 0 && wraps_into(prev, ¶, [left, right]) => { + // The line break was a word space, unless the line ends in a hyphen; the hyphen + // stays, as it may belong to the word ("well-known"). + if let Some(end) = prev.last_mut() + && !end.text.ends_with(char::is_whitespace) + && !end.text.ends_with('-') + { + end.text.push(' '); + } + prev.extend(para); + } + _ => out.push((para, lvl)), + } + } + out.into_iter().map(|(para, _)| para).collect() +} + +/// Whether the body text paragraph `prev` ends in a full line whose text runs on in `next`, the +/// body text paragraph just below it. `[left, right]` are the edges of the page's text. +fn wraps_into(prev: &[Run], next: &[Run], [left, right]: [f64; 2]) -> bool { + let (Some(last), Some(first)) = (prev.last(), next.first()) else { return false }; + let (a, b) = (last.block, first.block); + let size = a.size; + let alike = size.is_finite() && size > 0.0 && (a.size - b.size).abs() <= 0.5 && a.rotation.abs() <= 0.1 && b.rotation.abs() <= 0.1; + if !alike || a.rect[3] - a.rect[1] > ONE_LINE * size || list_item(&first.text) || ends_sentence(&last.text) { + return false; + } + // The last line: the runs at the end of `prev` that share the last run's line. + let (line_left, line_right) = prev + .iter() + .rev() + .take_while(|r| on_one_line(r.block.rect, a.rect)) + .fold((f64::INFINITY, f64::NEG_INFINITY), |(l, r), x| (l.min(x.block.rect[0]), r.max(x.block.rect[2]))); + let full = line_right >= right - (FULL_LINE.0 * size).max(FULL_LINE.1 * (right - left)); + let same_left = (b.rect[0] - line_left).abs() <= SAME_LEFT; + // Just below: no paragraph space between the two lines. + let gap = a.rect[1] - b.rect[3]; + full && same_left && (-LINE_GAP * size..=LINE_GAP * size).contains(&gap) +} diff --git a/crates/export/src/layout.rs b/crates/export/src/layout.rs index d1d074ac61..84c59f82bb 100644 --- a/crates/export/src/layout.rs +++ b/crates/export/src/layout.rs @@ -138,9 +138,20 @@ const INLINE_GAP: f64 = 0.5; /// page laid out so that many end at the same place stays quick. const CONTINUATION_PROBE: usize = 64; +/// How far (as a fraction of the font size) the block before may end past where a block starts +/// and still be the one it continues: a slanted italic's box leans into the next word by about a +/// quarter of its size. +const OVERHANG: f64 = 0.3; + +/// [`OVERHANG`] in points for a block of `size`: at least a point (rounding), at most six. +fn overhang(size: f64) -> f64 { + if size.is_finite() { (OVERHANG * size).clamp(1.0, 6.0) } else { 1.0 } +} + /// Per block, whether it continues another on the same line: it starts where that one ends (up to -/// [`INLINE_GAP`] later, or a point earlier for rounding), so it is the same line of text running -/// on (a link inside a sentence, a superscript), not a cell of its own. +/// [`INLINE_GAP`] later, or up to [`overhang`] earlier, as a slanted italic leans into the word +/// after it), so it is the same line of text running on (a link inside a sentence, a +/// superscript), not a cell of its own. fn continuations(blocks: &[Block]) -> Vec { continued_from(blocks).into_iter().map(|from| from.is_some()).collect() } @@ -159,7 +170,7 @@ fn continued_from(blocks: &[Block]) -> Vec> { .get(k..) .unwrap_or_default() .iter() - .take_while(|(end, _)| *end <= b.rect[0] + 1.0) + .take_while(|(end, _)| *end <= b.rect[0] + overhang(b.size)) .take(CONTINUATION_PROBE) .filter_map(|(_, ai)| blocks.get(*ai).filter(|_| *ai != bi).map(|a| (*ai, a))) .filter(|(_, a)| a.rect[3].min(b.rect[3]) - a.rect[1].max(b.rect[1]) > 0.5 * (a.rect[3] - a.rect[1]).min(b.rect[3] - b.rect[1])) @@ -234,6 +245,12 @@ pub(crate) fn paragraphs<'a>(blocks: &'a [Block], skip: &[bool]) -> Vec(blocks: &'a [Block], skip: &[bool], body: f64) -> Vec>> { + super::flow::join_wrapped(paragraphs(blocks, skip), |runs| para_level(runs, body)) +} + /// A paragraph's heading level: the lowest of its runs', so body text that runs on after a bold /// lead-in on the same line keeps the paragraph body text. A marker set smaller than the first run /// (a footnote number, "st" in "1st") does not count, so a heading keeps its footnote marker. @@ -450,8 +467,9 @@ pub(crate) fn items(pages: &[Page]) -> Vec> { } let (tables, consumed) = tables(&p.blocks); // Blocks and images by their top edge, top to bottom. - // A paragraph sits where its first run does: its other runs follow along the same line. - let mut parts: Vec<(f64, f64, Item)> = paragraphs(&p.blocks, &consumed) + // A paragraph sits where its first run does: its other runs follow along its line and the + // lines it wraps onto. + let mut parts: Vec<(f64, f64, Item)> = flowing_paragraphs(&p.blocks, &consumed, body) .into_iter() .filter_map(|runs| { let first = runs.first()?.block; diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index db53ee59c8..d1c4753aa8 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -2,9 +2,9 @@ //! //! The engine reduces each page to [`Page`]: paragraphs (text, box, size, bold/italic) and //! images (encoded bytes and box), in reading order. The writers turn that into a flowing -//! document: headings are the paragraphs set larger than the body text, and a block that -//! continues another on its line (an italic word, a link) is a run of that block's paragraph. -//! Word keeps pages with +//! document: headings are the paragraphs set larger than the body text; a block that +//! continues another on its line (an italic word, a link) is a run of that block's paragraph, and +//! a paragraph that wraps onto the next line runs on there. Word keeps pages with //! rotated text, or text overlapped by artwork or watermark-sized text, as positioned, editable //! shapes to avoid reflowing overlays; other pages flow with images inline. @@ -20,6 +20,7 @@ mod as_docx; mod as_html; mod as_markdown; mod as_rtf; +mod flow; mod layout; pub use as_docx::docx; @@ -34,10 +35,12 @@ pub use as_markdown::markdown; #[cfg(test)] use as_markdown::{emphasis, paragraph}; pub use as_rtf::rtf; +#[cfg(test)] +use flow::list_item; pub use layout::{Block, Cell, Image, Page, Table}; #[cfg(test)] use layout::{LOOKUP_STEPS, MAX_COLS, body_size}; -pub(crate) use layout::{Run, paragraphs, tables}; +pub(crate) use layout::{Run, flowing_paragraphs, tables}; #[cfg(test)] mod tests; diff --git a/crates/export/src/pptx.rs b/crates/export/src/pptx.rs index 49765000d3..ff1d1bad7d 100644 --- a/crates/export/src/pptx.rs +++ b/crates/export/src/pptx.rs @@ -125,6 +125,7 @@ pub fn pptx(pages: &[Page], title: &str) -> Vec { let mut global_media: Vec<(String, &Image)> = Vec::new(); let mut slides: Vec<(String, String)> = Vec::new(); // (slide_xml, slide_rels) + let body = crate::layout::body_size(pages); for p in pages { let (tables, consumed) = crate::tables(&p.blocks); @@ -139,7 +140,7 @@ pub fn pptx(pages: &[Page], title: &str) -> Vec { let mut items: Vec<(f64, f64, SlideBodyItem)> = Vec::new(); // The title is a shape of its own, so it joins no paragraph. let skip: Vec = (0..p.blocks.len()).map(|bi| Some(bi) == title_idx || consumed.get(bi).copied().unwrap_or(false)).collect(); - for runs in crate::paragraphs(&p.blocks, &skip) { + for runs in crate::flowing_paragraphs(&p.blocks, &skip, body) { let Some(first) = runs.first().map(|r| r.block) else { continue }; items.push((first.rect[3], first.rect[0], SlideBodyItem::Para(runs))); } diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 27cd8264e0..11ad393206 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -841,3 +841,138 @@ fn markdown_escapes_markup_and_keeps_emphasis_and_tables() { assert!(rows.contains(&"| A1 | kWh | 37414.00 |"), "{md}"); assert!(rows.iter().all(|r| r.matches(" | ").count() == 2), "every row has the header's three cells: {md}"); } + +/// A line of 10 pt text from `x0` to `x1` whose box starts at `y0`: 12 pt tall, so lines 14 pt +/// apart leave 2 pt between their boxes. +fn line(t: &str, x0: f64, x1: f64, y0: f64) -> Block { + Block { size: 10.0, ..block(t, x0, y0, x1, y0 + 12.0) } +} + +/// The text of each flowing paragraph Word gets for a page of `blocks`. +fn word_texts(blocks: Vec) -> Vec { + let p = Page { width: 612.0, height: 792.0, blocks, ..Page::default() }; + let xml = part(&docx(&[p], "Lines"), "word/document.xml"); + word_paragraphs(&xml).into_iter().map(|runs| runs.into_iter().map(|(t, _)| t).collect()).collect() +} + +/// A sentence over three lines, as the reading blocks give it: split at each style change, a +/// bold name in the first line and a slanted name in the second whose box leans 2.5 pt into the +/// comma after it (#4158). The next paragraph follows after paragraph space. +fn wrapped_sentence() -> Vec { + vec![ + line("The parties to this lease are the owner,", 72.0, 262.0, 700.0), + Block { bold: true, ..line("Harbour Lane Properties", 265.0, 400.0, 700.0) }, + line(", of 4 Quay Street, and the", 400.5, 540.0, 700.0), + line("tenant named in the schedule,", 72.0, 215.0, 686.0), + Block { italic: true, ..line("Blue Finch Studio", 218.0, 312.0, 686.0) }, + line(", who agree as follows to rent the upper", 309.5, 538.0, 686.0), + line("rooms for the whole of the term.", 72.0, 230.0, 672.0), + line("Rent is paid monthly in advance.", 72.0, 230.0, 640.0), + ] +} + +/// #4158: a sentence that wraps onto the next line after a change of style is one paragraph in +/// every format, its lines joined by a word space. +#[test] +fn a_sentence_that_wraps_after_a_style_change_is_one_paragraph() { + let sentence = "The parties to this lease are the owner, Harbour Lane Properties, of 4 Quay Street, and the tenant named in the \ +schedule, Blue Finch Studio, who agree as follows to rent the upper rooms for the whole of the term."; + assert_eq!(word_texts(wrapped_sentence()), [sentence, "Rent is paid monthly in advance."]); + let p = Page { width: 612.0, height: 792.0, blocks: wrapped_sentence(), ..Page::default() }; + let h = html(std::slice::from_ref(&p), "Lease"); + assert_eq!(h.matches("

").count(), 2, "{h}"); + assert!( + h.contains( + "

The parties to this lease are the owner, Harbour Lane Properties, of 4 Quay Street, and the tenant \ +named in the schedule, Blue Finch Studio, who agree as follows to rent the upper rooms for the whole of the term.

" + ), + "{h}" + ); + let r = rtf(std::slice::from_ref(&p)); + assert_eq!(r.matches("{\\pard").count(), 2, "{r}"); + // On a slide the last of the paragraphs set largest is the title; the sentence is one paragraph. + let s = part(&pptx(std::slice::from_ref(&p), "Lease"), "ppt/slides/slide1.xml"); + let slide: Vec = + s.split("").skip(1).map(|p| p.split("").skip(1).filter_map(|t| t.split_once("").map(|(t, _)| t)).collect()).collect(); + assert!(slide.iter().any(|p| p == sentence), "{s}"); +} + +/// A slanted word's box leans into the word after it by about a quarter of its size, so that +/// word still runs on from it; text that starts further back is drawn over the text before it, +/// not after it. +#[test] +fn a_slanted_word_that_leans_into_the_next_stays_in_its_line() { + let leaning = vec![ + line("named in the schedule,", 72.0, 180.0, 700.0), + Block { italic: true, ..line("Blue Finch Studio", 183.0, 276.0, 700.0) }, + line(", who agree", 273.5, 330.0, 700.0), + ]; + assert_eq!(word_texts(leaning), ["named in the schedule, Blue Finch Studio, who agree"]); + let over = vec![line("A label", 72.0, 120.0, 700.0), line("over another", 115.0, 180.0, 700.0)]; + assert_eq!(word_texts(over), ["A label", "over another"]); +} + +/// A line runs on into the next only when it reaches the right edge of the text and the next +/// starts just below at the same left edge, in the same size. A line that ends a sentence or ends +/// short, and a next line that is a heading, a list item, indented or after paragraph space, keep +/// their paragraph break. Each case is a page of its own. +#[test] +fn a_paragraph_breaks_where_its_line_does_not_run_on() { + let full = "The tenant keeps the rooms, the stairs and the shared yard clean and in good repair"; + let next = "throughout the term of the lease"; + let first = |t: &str| line(t, 72.0, 540.0, 700.0); + let second = |t: &str| line(t, 72.0, 240.0, 686.0); + assert_eq!(word_texts(vec![first(full), second(next)]), [format!("{full} {next}")]); + assert_eq!(word_texts(vec![first("Изпълнителят поддържа сървърите и мрежата на"), second("възложителя в работно състояние")]).len(), 1); + let heading = |b: Block| Block { bold: true, ..b }; + for (why, blocks) in [ + ("a sentence ends", vec![first(&format!("{full}.")), second(next)]), + ("a sentence ends in quotes", vec![first("Страните се договориха „за срок от една година.“"), second(next)]), + ("the line ends short", vec![line(next, 72.0, 240.0, 700.0), line(full, 72.0, 540.0, 686.0)]), + ("paragraph space", vec![first(full), line(next, 72.0, 240.0, 676.0)]), + ("an indent", vec![first(full), line(next, 90.0, 258.0, 686.0)]), + ("a larger size", vec![first(full), Block { size: 12.0, ..second(next) }]), + ("a numbered item", vec![first(full), second("2. The tenant pays the rent")]), + ("a lettered item", vec![first(full), second("а) поддръжка на сървърите")]), + ("a bulleted item", vec![first(full), second("• the stairs")]), + ("a heading below", vec![first(full), heading(second("Rent and deposit"))]), + ("a heading above", vec![heading(first("Obligations of the tenant")), second(next)]), + ] { + assert_eq!(word_texts(blocks).len(), 2, "{why}"); + } +} + +#[test] +fn list_items_are_told_from_text() { + for item in ["• item", "\u{F0B7} item", "- item", "\u{2013} item", "* item", "1. Term", "12) Rent", "3.", "a) rooms", "а) стаи"] { + assert!(list_item(item), "{item}"); + } + for text in ["1.5 kg of", "2026. Year", "i.e. this", "т.е. така", "38 rooms", "Rent 12)", "1234) many", ""] { + assert!(!list_item(text), "{text}"); + } +} + +/// Wrapped lines join on hostile boxes and sizes without a panic, and never lose a line's text. +#[test] +fn joining_lines_survives_hostile_boxes() { + let mut blocks = Vec::new(); + for (i, v) in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -1.0, 0.0, 1e300].into_iter().enumerate() { + let y = 700.0 - 14.0 * i as f64; + blocks.push(Block { size: v, ..line(&format!("size {i}"), 72.0, 540.0, y) }); + blocks.push(Block { rect: [v, y, 540.0, y + 12.0], ..line(&format!("left {i}"), 72.0, 540.0, y) }); + blocks.push(Block { rect: [72.0, v, v, y + 12.0], ..line(&format!("box {i}"), 72.0, 540.0, y) }); + blocks.push(Block { rotation: v, ..line(&format!("turn {i}"), 72.0, 540.0, y) }); + } + let p = Page { width: 612.0, height: 792.0, blocks, ..Page::default() }; + let outputs = [ + html(std::slice::from_ref(&p), "Hostile"), + rtf(std::slice::from_ref(&p)), + part(&pptx(std::slice::from_ref(&p), "Hostile"), "ppt/slides/slide1.xml"), + word_texts(p.blocks.clone()).concat(), + ]; + for output in outputs { + for b in &p.blocks { + assert!(output.contains(&b.text), "{} was lost", b.text); + } + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 1404247cd7..cffbff4176 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7076,8 +7076,8 @@ milestone = "M10" status = "partial" commands = ["export.docx"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158). Not yet: a style change that wraps onto the next line still splits its paragraph there (the reading blocks keep style boundaries, #4158), exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." +evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", "crates/export/src/tests.rs::a_slanted_word_that_leans_into_the_next_stays_in_its_line", "crates/export/src/tests.rs::a_paragraph_breaks_where_its_line_does_not_run_on", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_a_wrapped_sentence_in_one_paragraph"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158); a slanted run whose box leans into the next word by up to 0.3 of its size still runs on into it. A paragraph whose last line reaches the right edge of the page's text runs on into the line just below it that starts at the same left edge in the same size, so a sentence the reading blocks split at a style change stays one paragraph across its lines; headings, list items, a line that ends a sentence, an indented line and one after paragraph space keep their break (#4158). Not yet: a paragraph whose rows the reading blocks already grouped past a style change (the exporter does not see where such a block's last row ends), a wrapped line that ends in a full stop or colon, a first-line indent, lines with more than half their font size of space between them (as 1.5 or double line spacing leaves), text in columns, exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." [[feature]] id = "create.export-rtf" @@ -7088,8 +7088,8 @@ milestone = "M10" status = "shipped" commands = ["export.rtf"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::rtf_escapes_and_sizes", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Paragraphs with sizes, bold, italic, text colour (colortbl), Unicode; a block that continues another on its line is a formatted group inside that paragraph (#4158), though a style change that wraps onto the next line still splits it; page breaks. Images are left out." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::rtf_escapes_and_sizes", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Paragraphs with sizes, bold, italic, text colour (colortbl), Unicode; a block that continues another on its line is a formatted group inside that paragraph (#4158), and a paragraph that wraps onto the next line after a style change runs on there, with the same rules and limits as Word export; page breaks. Images are left out." [[feature]] @@ -7140,8 +7140,9 @@ evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", + "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", ] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. PowerPoint presentation: each page becomes a slide, with the largest non-table text as slide title, remaining paragraphs as body text, images and detected tables in reading order; DrawingML formatting (font size, bold, italic, text fill colour, cell borders); wired into engine command export.pptx, automation tool doc_export_office, and Export dialog. Missing: content past the slide bottom is not paginated or shrunk across multiple slides (no normAutofit), all slides use the first page's dimensions, title is determined by largest font size rather than semantic structure, and there is no layout-preserving mode (Acrobat keeps positions). Not yet verified in PowerPoint or Keynote: pptx_opens_in_libreoffice only converts the file when LibreOffice is installed and passes silently otherwise (CI has none), so the package structure is checked by pptx_package_parts_are_complete_and_consistent instead." +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. PowerPoint presentation: each page becomes a slide, with the largest non-table text as slide title, remaining paragraphs as body text (a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158), images and detected tables in reading order; DrawingML formatting (font size, bold, italic, text fill colour, cell borders); wired into engine command export.pptx, automation tool doc_export_office, and Export dialog. Missing: content past the slide bottom is not paginated or shrunk across multiple slides (no normAutofit), all slides use the first page's dimensions, title is determined by largest font size rather than semantic structure, and there is no layout-preserving mode (Acrobat keeps positions). Not yet verified in PowerPoint or Keynote: pptx_opens_in_libreoffice only converts the file when LibreOffice is installed and passes silently otherwise (CI has none), so the package structure is checked by pptx_package_parts_are_complete_and_consistent instead." [[feature]] id = "create.export-png" @@ -7200,8 +7201,8 @@ milestone = "M10" status = "partial" commands = ["export.html"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour), inline images, and detected tables with spans/empty cells. Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour; a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158), inline images, and detected tables with spans/empty cells. Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." [[feature]] id = "create.export-text" From b98b20633cfab25a675c17fd1414d74aaf7e5248 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:49:19 +0200 Subject: [PATCH 021/282] Exports keep a bulleted list as paragraphs, not a table Word and LibreOffice often draw a list's bullet in a symbol font, a tab stop before the item's text, so each bullet is a block of its own. Table detection read the bullets and the items as two columns, and every export wrote the list as a borderless two-column table. Each bullet is now paired with the text that starts within three times its size after it on its line. The text runs on from its bullet, and bullets stay out of table detection, so Word gets one paragraph per item with the bullet first. The Symbol and Wingdings bullets Word writes (U+F0B7, U+F0A7) count. A dash or a middle dot counts only when nothing starts left of it on its line, so a dash that stands for "none" in a table row stays a cell. The new export and automation tests fail without the change. --- crates/automation/tests/it/automation.rs | 50 +++++++++++-- crates/export/src/flow.rs | 75 +++++++++++++++++++- crates/export/src/layout.rs | 25 +++++-- crates/export/src/tests.rs | 89 ++++++++++++++++++++++++ parity/acrobat-features.toml | 15 ++-- 5 files changed, 237 insertions(+), 17 deletions(-) diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 4d3aa44f01..674d8e7666 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -7615,8 +7615,9 @@ BT /F1 12 Tf 72 640 Td (A paragraph further down.) Tj ET"; } /// A one-page PDF (612 x 792 pt) drawing `content`, with the standard fonts `fonts` as /F1, /F2 -/// and so on, each in its built-in encoding. -fn standard_fonts_pdf(content: &str, fonts: &[&str]) -> Vec { +/// and so on, each in its built-in encoding. A font given `bfchar` pairs (" ") reads +/// its codes through that ToUnicode map, as a symbol font Word embeds does. +fn standard_fonts_pdf(content: &str, fonts: &[(&str, Option<&str>)]) -> Vec { let names: String = (1..=fonts.len()).map(|i| format!("/F{i} {} 0 R ", i + 4)).collect(); let mut objs = vec![ "<< /Type /Catalog /Pages 2 0 R >>".to_string(), @@ -7624,7 +7625,19 @@ fn standard_fonts_pdf(content: &str, fonts: &[&str]) -> Vec { format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << {names}>> >> >>"), format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), ]; - objs.extend(fonts.iter().map(|f| format!("<< /Type /Font /Subtype /Type1 /BaseFont /{f} >>"))); + let mut maps = Vec::new(); + for (font, map) in fonts { + let to_unicode = map.map_or(String::new(), |map| { + let cmap = format!( + "/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CMapType 2 def 1 begincodespacerange <00> endcodespacerange \ +1 beginbfchar {map} endbfchar endcmap CMapName currentdict /CMap defineresource pop end end" + ); + maps.push(format!("<< /Length {} >>\nstream\n{cmap}\nendstream", cmap.len())); + format!(" /ToUnicode {} 0 R", fonts.len() + 4 + maps.len()) + }); + objs.push(format!("<< /Type /Font /Subtype /Type1 /BaseFont /{font}{to_unicode} >>")); + } + objs.extend(maps); let mut pdf = b"%PDF-1.7\n".to_vec(); let mut offsets = Vec::new(); for (i, o) in objs.iter().enumerate() { @@ -7676,7 +7689,8 @@ BT /F1 10 Tf 72 686 Td (tenant named in the schedule, ) Tj /F3 10 Tf (Blue Finch /F1 10 Tf (, who agree as follows to rent the upper) Tj ET \ BT /F1 10 Tf 72 672 Td (rooms for the whole of the term.) Tj ET \ BT /F1 10 Tf 72 640 Td (Rent is paid monthly in advance.) Tj ET"; - std::fs::write(dir.join("lease.pdf"), standard_fonts_pdf(content, &["Helvetica", "Helvetica-Bold", "Helvetica-Oblique"])).unwrap(); + let fonts = [("Helvetica", None), ("Helvetica-Bold", None), ("Helvetica-Oblique", None)]; + std::fs::write(dir.join("lease.pdf"), standard_fonts_pdf(content, &fonts)).unwrap(); let mut a = auto(&dir); let doc = ok(&mut a, "doc_open", json!({ "path": "lease.pdf" }))["doc"].as_u64().unwrap(); ok(&mut a, "doc_export_office", json!({ "doc": doc, "path": "lease.docx" })); @@ -7685,6 +7699,34 @@ schedule, Blue Finch Studio, who agree as follows to rent the upper rooms for th assert_eq!(word_paragraph_texts(&dir.join("lease.docx")), [sentence, "Rent is paid monthly in advance."]); } +/// A bulleted list drawn as Word draws one, each bullet in the Symbol font (read as U+F0B7, as +/// Word maps it) a tab stop before its item's text, reaches Word as one paragraph per item and +/// not as a two-column table. +#[test] +fn exporting_to_word_keeps_a_bulleted_list_as_paragraphs() { + let dir = workdir("bulleted-list-word"); + let mut content = String::from("BT /F1 10 Tf 72 700 Td (The tenant looks after:) Tj ET\n"); + for (n, item) in ["the stairs and the landing;", "the shared yard and its gate;", "the bins, every week."].iter().enumerate() { + let y = 686 - 14 * n; + content.push_str(&format!("BT /F2 10 Tf 90 {y} Td (\\267) Tj ET BT /F1 10 Tf 108 {y} Td ({item}) Tj ET\n")); + } + content.push_str("BT /F1 10 Tf 72 630 Td (Rent is paid monthly in advance.) Tj ET"); + std::fs::write(dir.join("list.pdf"), standard_fonts_pdf(&content, &[("Helvetica", None), ("Symbol", Some(" "))])).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "list.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_office", json!({ "doc": doc, "path": "list.docx" })); + assert_eq!( + word_paragraph_texts(&dir.join("list.docx")), + [ + "The tenant looks after:", + "\u{F0B7} the stairs and the landing;", + "\u{F0B7} the shared yard and its gate;", + "\u{F0B7} the bins, every week.", + "Rent is paid monthly in advance." + ] + ); +} + /// Hundreds of calculations run on opening and again after a change through the tools. The /// time limits are raised so that a busy machine does not stop them (the engine's tests check /// that the limits stop them). diff --git a/crates/export/src/flow.rs b/crates/export/src/flow.rs index e064772864..49ef73e38d 100644 --- a/crates/export/src/flow.rs +++ b/crates/export/src/flow.rs @@ -7,8 +7,13 @@ //! line reaches the right edge of the page's text with the one that starts just below it at the //! same left edge, in the same size: the text wrapped there. Headings, list items, a line that //! ends a sentence and a line with paragraph space after it stay apart. +//! +//! A list item's bullet is often a block of its own: drawn in a symbol font, a tab stop before +//! the item's text. Table detection read such a list as a two-column table, bullets in one +//! column and the items in the other. [`bullet_items`] pairs each bullet with its text, so the +//! text runs on from its bullet and the bullets stay out of table detection. -use super::layout::Run; +use super::layout::{Block, Run}; /// Bullets that mark a list item wherever they stand, including the Symbol and Wingdings bullets /// Word writes into its PDFs (U+F0B7, U+F0A7). @@ -34,12 +39,80 @@ const FULL_LINE: (f64, f64) = (3.0, 0.12); /// it and still be at the same left edge. const SAME_LEFT: f64 = 2.0; +/// How far after its bullet (as a multiple of the bullet's size) a list item's text may start: a +/// tab stop away. +const BULLET_GAP: f64 = 3.0; + +/// How many blocks [`bullet_items`] may look at on one page, so that a page of many bullets stays +/// quick; past it, the bullets left are not taken for bullets. +const BULLET_BUDGET: usize = 1_000_000; + +/// Takes a step from `budget`: whether there was one left. +fn step(budget: &mut usize) -> bool { + budget.checked_sub(1).map(|left| *budget = left).is_some() +} + /// Whether two boxes ([x0, y0, x1, y1]) share a line: they overlap vertically by more than half /// the shorter one, as `layout::continued_from` has it. fn on_one_line(a: [f64; 4], b: [f64; 4]) -> bool { a[3].min(b[3]) - a[1].max(b[1]) > 0.5 * (a[3] - a[1]).min(b[3] - b[1]) } +/// Whether a block is a bullet drawn on its own: one bullet, dash or middle dot, no wider than one +/// and a half times its size. +fn is_bullet(b: &Block) -> bool { + let mut chars = b.text.trim().chars(); + let (Some(c), None) = (chars.next(), chars.next()) else { return false }; + (BULLETS.contains(&c) || DASHES.contains(&c)) && b.rect[2] - b.rect[0] <= 1.5 * b.size +} + +/// Whether no other block on `blocks[i]`'s line starts left of it. Each block looked at takes a +/// step from `budget`; when the budget runs out the answer is no. +fn opens_line(i: usize, blocks: &[Block], budget: &mut usize) -> bool { + let Some(a) = blocks.get(i) else { return false }; + for (j, b) in blocks.iter().enumerate() { + if !step(budget) { + return false; + } + if j != i && b.rect[0] < a.rect[0] && on_one_line(a.rect, b.rect) { + return false; + } + } + true +} + +/// The list items of a page whose bullet is a block of its own, as (bullet, text): the text is the +/// block that starts nearest after the bullet on its line, within [`BULLET_GAP`] times its size. +/// A dash or a middle dot is a bullet only when it opens its line; inside a line it is +/// punctuation, or a table cell's "none". The items' texts of a list all start at one tab stop, +/// so the search looks through every block that starts there, within [`BULLET_BUDGET`]. +pub(crate) fn bullet_items(blocks: &[Block]) -> Vec<(usize, usize)> { + let mut by_start: Vec<(f64, usize)> = blocks.iter().enumerate().map(|(i, b)| (b.rect[0], i)).collect(); + by_start.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut budget = BULLET_BUDGET; + let mut out = Vec::new(); + for (ai, a) in blocks.iter().enumerate().filter(|(_, b)| is_bullet(b)) { + // The text may start a point inside the bullet's box, as a run that continues another can. + let k = by_start.partition_point(|(x, _)| *x < a.rect[2] - 1.0); + let text = by_start + .get(k..) + .unwrap_or_default() + .iter() + .take_while(|(x, _)| *x <= a.rect[2] + BULLET_GAP * a.size) + .take_while(|_| step(&mut budget)) + .filter(|(_, bi)| *bi != ai) + .filter_map(|(_, bi)| blocks.get(*bi).map(|b| (*bi, b))) + .find(|(_, b)| on_one_line(a.rect, b.rect) && (a.rotation - b.rotation).abs() <= 0.1 && !is_bullet(b)) + .map(|(bi, _)| bi); + let Some(text) = text else { continue }; + let dash = a.text.trim().chars().next().is_some_and(|c| DASHES.contains(&c)); + if !dash || opens_line(ai, blocks, &mut budget) { + out.push((ai, text)); + } + } + out +} + /// Whether `text` opens like a list item: a bullet, a dash or `*`; one to three digits followed /// by `.` or `)`; or one letter followed by `)` ("a)", "а)"); then a space or nothing. "1.5 kg", /// "i.e." and "2026. Year" are text. diff --git a/crates/export/src/layout.rs b/crates/export/src/layout.rs index 84c59f82bb..133862b1d9 100644 --- a/crates/export/src/layout.rs +++ b/crates/export/src/layout.rs @@ -157,11 +157,12 @@ fn continuations(blocks: &[Block]) -> Vec { } /// Per block, the block it continues on the same line (see [`continuations`]): of those it could -/// continue, the one that ends nearest to where it starts. +/// continue, the one that ends nearest to where it starts. A list item's text continues its +/// bullet, a tab stop before it (`flow::bullet_items`). fn continued_from(blocks: &[Block]) -> Vec> { let mut by_end: Vec<(f64, usize)> = blocks.iter().enumerate().map(|(i, b)| (b.rect[2], i)).collect(); by_end.sort_by(|a, b| a.0.total_cmp(&b.0)); - blocks + let mut from: Vec> = blocks .iter() .enumerate() .map(|(bi, b)| { @@ -177,7 +178,13 @@ fn continued_from(blocks: &[Block]) -> Vec> { .min_by(|(_, a), (_, c)| (a.rect[2] - b.rect[0]).abs().total_cmp(&(c.rect[2] - b.rect[0]).abs())) .map(|(ai, _)| ai) }) - .collect() + .collect(); + for (bullet, text) in super::flow::bullet_items(blocks) { + if let Some(slot @ None) = from.get_mut(text) { + *slot = Some(bullet); + } + } + from } /// A gap between two runs wider than this (as a fraction of the font size) is a word space the @@ -270,8 +277,16 @@ fn is_cell_like(b: &Block) -> bool { /// per block, whether a table consumed it (so `items` can drop it from the paragraph flow). pub(crate) fn tables(blocks: &[Block]) -> (Vec, Vec) { let mut consumed = vec![false; blocks.len()]; + // A list's bullets are not cells: beside their items' text they read as a two-column table. + let mut bullet = vec![false; blocks.len()]; + for (i, _) in super::flow::bullet_items(blocks) { + if let Some(b) = bullet.get_mut(i) { + *b = true; + } + } + let cell_like = |i: usize, b: &Block| is_cell_like(b) && !bullet.get(i).copied().unwrap_or(false); // Candidate columns: cluster the left edges of cell-like blocks. - let mut xs: Vec<(f64, usize)> = blocks.iter().enumerate().filter(|(_, b)| is_cell_like(b)).map(|(i, b)| (b.rect[0], i)).collect(); + let mut xs: Vec<(f64, usize)> = blocks.iter().enumerate().filter(|(i, b)| cell_like(*i, b)).map(|(i, b)| (b.rect[0], i)).collect(); xs.sort_by(|a, b| a.0.total_cmp(&b.0)); let mut cols: Vec<(f64, Vec)> = Vec::new(); // (left edge, member block indices) for (x, i) in xs { @@ -307,7 +322,7 @@ pub(crate) fn tables(blocks: &[Block]) -> (Vec
, Vec) { order.sort_by(|a, b| blocks[*b].rect[3].total_cmp(&blocks[*a].rect[3]).then(blocks[*a].rect[0].total_cmp(&blocks[*b].rect[0]))); for &bi in &order { let b = &blocks[bi]; - if !is_cell_like(b) { + if !cell_like(bi, b) { continue; } let Some(ci) = column_at(b.rect[0]) else { continue }; diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 11ad393206..dc99894d1b 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -976,3 +976,92 @@ fn joining_lines_survives_hostile_boxes() { } } } + +/// A bulleted list as Word and LibreOffice draw one: each bullet a block of its own in a symbol +/// font, a tab stop before its item's text, between a lead-in line and the next paragraph. +fn bulleted_list(bullet: &str) -> Vec { + let mut blocks = vec![line("The tenant looks after:", 72.0, 180.0, 700.0)]; + for (n, (text, end)) in + [("the stairs and the landing;", 230.0), ("the shared yard and its gate;", 240.0), ("the bins, every week.", 200.0)].into_iter().enumerate() + { + let y = 686.0 - 14.0 * n as f64; + blocks.push(line(bullet, 90.0, 94.6, y)); + blocks.push(line(text, 108.0, end, y)); + } + blocks.push(line("Rent is paid monthly in advance.", 72.0, 230.0, 630.0)); + blocks +} + +/// A bulleted list whose bullets are blocks of their own is one paragraph per item, each item's +/// text after its bullet, in every format: not a two-column table of bullets and text. The +/// Symbol and Wingdings bullets Word writes count, and so does a dash that opens its line. +#[test] +fn a_bulleted_list_is_paragraphs_not_a_table() { + for bullet in ["•", "\u{F0B7}", "\u{F0A7}", "\u{2013}"] { + let p = Page { width: 612.0, height: 792.0, blocks: bulleted_list(bullet), ..Page::default() }; + let (found, _) = tables(&p.blocks); + assert!(found.is_empty(), "{bullet}: {found:?}"); + let items = ["the stairs and the landing;", "the shared yard and its gate;", "the bins, every week."].map(|t| format!("{bullet} {t}")); + let mut expected = vec!["The tenant looks after:".to_string()]; + expected.extend(items.iter().cloned()); + expected.push("Rent is paid monthly in advance.".into()); + assert_eq!(word_texts(p.blocks.clone()), expected, "{bullet}"); + let h = html(std::slice::from_ref(&p), "List"); + assert!(!h.contains("
") && items.iter().all(|i| h.contains(&format!("

{i}

"))), "{bullet}: {h}"); + assert!(!rtf(std::slice::from_ref(&p)).contains("\\trowd"), "{bullet}"); + let s = part(&pptx(std::slice::from_ref(&p), "List"), "ppt/slides/slide1.xml"); + assert!(!s.contains(""), "{bullet}: {s}"); + } +} + +/// A dash in a table row stands for "none": it is a cell like the others, not a bullet, so the +/// table keeps all its rows. +#[test] +fn a_dash_in_a_table_row_stays_a_cell() { + let mut blocks = Vec::new(); + for (n, (name, first, second)) in [("Apples", "\u{2013}", "5"), ("Pears", "3", "4"), ("Plums", "2", "6")].into_iter().enumerate() { + let y = 700.0 - 14.0 * n as f64; + blocks.push(line(name, 72.0, 110.0, y)); + blocks.push(line(first, 200.0, 205.0, y)); + blocks.push(line(second, 225.0, 230.0, y)); + } + let (found, consumed) = tables(&blocks); + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].rows.len(), 3, "{found:?}"); + assert!(consumed.iter().all(|used| *used), "{consumed:?}"); +} + +/// A bullet is never a table cell, even when its items carry a second column such as an amount: +/// every item keeps its bullet in front of its text. +#[test] +fn a_bullet_is_never_a_table_cell() { + let mut blocks = Vec::new(); + for (n, (item, amount)) in [("apples", "5 kg"), ("pears", "4 kg"), ("plums", "2 kg")].into_iter().enumerate() { + let y = 700.0 - 14.0 * n as f64; + blocks.push(line("•", 90.0, 94.6, y)); + blocks.push(line(item, 108.0, 140.0, y)); + blocks.push(line(amount, 300.0, 325.0, y)); + } + let (found, _) = tables(&blocks); + assert!(found.iter().flat_map(|t| &t.rows).flatten().all(|cell| cell.text != "•"), "{found:?}"); + let paragraphs = word_texts(blocks); + for item in ["• apples", "• pears", "• plums"] { + assert!(paragraphs.iter().any(|p| p.starts_with(item)), "{item}: {paragraphs:?}"); + } +} + +/// Every item of a long list keeps its bullet, though all the items' texts start at one tab stop. +#[test] +fn a_long_bulleted_list_keeps_every_bullet_with_its_item() { + let mut blocks = Vec::new(); + for n in 0..150 { + let y = 2200.0 - 14.0 * f64::from(n); + blocks.push(line("\u{F0B7}", 90.0, 94.6, y)); + blocks.push(line(&format!("item {n};"), 108.0, 160.0, y)); + } + let (found, _) = tables(&blocks); + assert!(found.is_empty(), "{found:?}"); + let paragraphs = word_texts(blocks); + assert_eq!(paragraphs.len(), 150, "{paragraphs:?}"); + assert!(paragraphs.iter().enumerate().all(|(n, p)| *p == format!("\u{F0B7} item {n};")), "{paragraphs:?}"); +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index cffbff4176..a349c91fd8 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7076,8 +7076,8 @@ milestone = "M10" status = "partial" commands = ["export.docx"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", "crates/export/src/tests.rs::a_slanted_word_that_leans_into_the_next_stays_in_its_line", "crates/export/src/tests.rs::a_paragraph_breaks_where_its_line_does_not_run_on", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_a_wrapped_sentence_in_one_paragraph"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158); a slanted run whose box leans into the next word by up to 0.3 of its size still runs on into it. A paragraph whose last line reaches the right edge of the page's text runs on into the line just below it that starts at the same left edge in the same size, so a sentence the reading blocks split at a style change stays one paragraph across its lines; headings, list items, a line that ends a sentence, an indented line and one after paragraph space keep their break (#4158). Not yet: a paragraph whose rows the reading blocks already grouped past a style change (the exporter does not see where such a block's last row ends), a wrapped line that ends in a full stop or colon, a first-line indent, lines with more than half their font size of space between them (as 1.5 or double line spacing leaves), text in columns, exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." +evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", "crates/export/src/tests.rs::a_slanted_word_that_leans_into_the_next_stays_in_its_line", "crates/export/src/tests.rs::a_paragraph_breaks_where_its_line_does_not_run_on", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_a_wrapped_sentence_in_one_paragraph", "crates/export/src/tests.rs::a_bulleted_list_is_paragraphs_not_a_table", "crates/export/src/tests.rs::a_bullet_is_never_a_table_cell", "crates/export/src/tests.rs::a_long_bulleted_list_keeps_every_bullet_with_its_item", "crates/export/src/tests.rs::a_dash_in_a_table_row_stays_a_cell", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_a_bulleted_list_as_paragraphs"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158); a slanted run whose box leans into the next word by up to 0.3 of its size still runs on into it. A paragraph whose last line reaches the right edge of the page's text runs on into the line just below it that starts at the same left edge in the same size, so a sentence the reading blocks split at a style change stays one paragraph across its lines; headings, list items, a line that ends a sentence, an indented line and one after paragraph space keep their break (#4158). A list whose bullets are blocks of their own (a bullet in a symbol font a tab stop before the item's text, as Word writes it, U+F0B7 and U+F0A7 included) is one paragraph per item, the bullet first, and not a borderless two-column table: a bullet is never a table cell, and a dash or middle dot counts as a bullet only when nothing starts left of it on its line. Not yet: Word list numbering (an item is a plain paragraph that starts with its bullet), a bullet drawn in the standard Symbol font without a ToUnicode map (it never reaches export, though text extraction reads it), a paragraph whose rows the reading blocks already grouped past a style change (the exporter does not see where such a block's last row ends), a wrapped line that ends in a full stop or colon, a first-line indent, lines with more than half their font size of space between them (as 1.5 or double line spacing leaves), text in columns, exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." [[feature]] id = "create.export-rtf" @@ -7088,8 +7088,8 @@ milestone = "M10" status = "shipped" commands = ["export.rtf"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::rtf_escapes_and_sizes", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Paragraphs with sizes, bold, italic, text colour (colortbl), Unicode; a block that continues another on its line is a formatted group inside that paragraph (#4158), and a paragraph that wraps onto the next line after a style change runs on there, with the same rules and limits as Word export; page breaks. Images are left out." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::rtf_escapes_and_sizes", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", "crates/export/src/tests.rs::a_bulleted_list_is_paragraphs_not_a_table"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Paragraphs with sizes, bold, italic, text colour (colortbl), Unicode; a block that continues another on its line is a formatted group inside that paragraph (#4158), and a paragraph that wraps onto the next line after a style change runs on there, with the same rules and limits as Word export; a list whose bullets are blocks of their own is a paragraph per item, not a table, as in Word export; page breaks. Images are left out." [[feature]] @@ -7141,8 +7141,9 @@ evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", + "crates/export/src/tests.rs::a_bulleted_list_is_paragraphs_not_a_table", ] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. PowerPoint presentation: each page becomes a slide, with the largest non-table text as slide title, remaining paragraphs as body text (a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158), images and detected tables in reading order; DrawingML formatting (font size, bold, italic, text fill colour, cell borders); wired into engine command export.pptx, automation tool doc_export_office, and Export dialog. Missing: content past the slide bottom is not paginated or shrunk across multiple slides (no normAutofit), all slides use the first page's dimensions, title is determined by largest font size rather than semantic structure, and there is no layout-preserving mode (Acrobat keeps positions). Not yet verified in PowerPoint or Keynote: pptx_opens_in_libreoffice only converts the file when LibreOffice is installed and passes silently otherwise (CI has none), so the package structure is checked by pptx_package_parts_are_complete_and_consistent instead." +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. PowerPoint presentation: each page becomes a slide, with the largest non-table text as slide title, remaining paragraphs as body text (a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158; a list whose bullets are blocks of their own is a paragraph per item, not a table), images and detected tables in reading order; DrawingML formatting (font size, bold, italic, text fill colour, cell borders); wired into engine command export.pptx, automation tool doc_export_office, and Export dialog. Missing: content past the slide bottom is not paginated or shrunk across multiple slides (no normAutofit), all slides use the first page's dimensions, title is determined by largest font size rather than semantic structure, and there is no layout-preserving mode (Acrobat keeps positions). Not yet verified in PowerPoint or Keynote: pptx_opens_in_libreoffice only converts the file when LibreOffice is installed and passes silently otherwise (CI has none), so the package structure is checked by pptx_package_parts_are_complete_and_consistent instead." [[feature]] id = "create.export-png" @@ -7201,8 +7202,8 @@ milestone = "M10" status = "partial" commands = ["export.html"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour; a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158), inline images, and detected tables with spans/empty cells. Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops", "crates/export/src/tests.rs::a_sentence_that_wraps_after_a_style_change_is_one_paragraph", "crates/export/src/tests.rs::a_bulleted_list_is_paragraphs_not_a_table"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour; a paragraph that wraps onto the next line after a style change stays one paragraph, with the same rules and limits as Word export, #4158; a list whose bullets are blocks of their own is a paragraph per item, not a table, though not yet a