From a699e1330535bd0a7f6b8a79ea9bedefd97325bd Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:00:11 +0200 Subject: [PATCH 01/37] Word and HTML exports name the document's language Word export named no language, so Word checked the spelling of every export as English and underlined a whole document in any other language. The PDF already says its language in the catalog's /Lang. When that is a well formed language tag, Word gets it as the default language of the text (w:docDefaults), in the attribute Word keeps for its script: w:eastAsia for Chinese, Japanese and Korean, w:bidi for Arabic, Hebrew, Persian and Urdu, w:val for the rest. Both HTML exports, with and without CSS, get . Without /Lang, or with a malformed one, the files are exactly as before. docx_with_language, html_with_language and html_simple_with_language take the language; docx, html and html_simple stay as they are and call them with none. The engine reads /Lang through the existing initial view. The new export and engine tests fail without the change. --- crates/engine/src/compare.rs | 8 +++++--- crates/engine/src/tests.rs | 29 +++++++++++++++++++++++++++ crates/export/src/as_docx.rs | 30 +++++++++++++++++++++++++--- crates/export/src/as_html.rs | 36 ++++++++++++++++++++++++++++++---- crates/export/src/lib.rs | 4 ++-- crates/export/src/tests.rs | 38 ++++++++++++++++++++++++++++++++++++ parity/acrobat-features.toml | 8 ++++---- 7 files changed, 137 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..0996ed76ee 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -300,11 +300,13 @@ impl crate::Document { let (pages, truncated_pages) = self.export_pages_checked(!matches!(format, OfficeFormat::Rtf | OfficeFormat::Markdown), MAX_EXPORT_IMAGE_BYTES); let title = self.info.title.clone().unwrap_or_else(|| self.name.trim_end_matches(".pdf").to_string()); + // Word and HTML name the document's language (`/Lang`), so spelling is checked in it. + let language = self.initial_view().language; let bytes = match format { - OfficeFormat::Docx => pdfcraft_export::docx(&pages, &title), + OfficeFormat::Docx => pdfcraft_export::docx_with_language(&pages, &title, language.as_deref()), OfficeFormat::Pptx => pdfcraft_export::pptx(&pages, &title), - OfficeFormat::Html => pdfcraft_export::html(&pages, &title).into_bytes(), - OfficeFormat::HtmlSimple => pdfcraft_export::html_simple(&pages, &title).into_bytes(), + OfficeFormat::Html => pdfcraft_export::html_with_language(&pages, &title, language.as_deref()).into_bytes(), + OfficeFormat::HtmlSimple => pdfcraft_export::html_simple_with_language(&pages, &title, language.as_deref()).into_bytes(), OfficeFormat::Rtf => pdfcraft_export::rtf(&pages).into_bytes(), OfficeFormat::Markdown => pdfcraft_export::markdown(&pages).into_bytes(), }; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 6c54131d02..9673f2beee 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5912,6 +5912,35 @@ fn actions_fail_when_hidden_information_could_not_be_removed() { assert!(e.starts_with("Sanitize document:"), "{e}"); } +/// A PDF whose catalog names its language (`/Lang`) exports to Word and HTML in that language, so +/// Word checks the spelling in it, not in English; one that names none exports as before. +#[test] +fn office_exports_take_the_documents_language() { + let pdf = |catalog: &str| { + pdfcraft_testkit::Pdf::new() + .obj(catalog) + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >> >>") + .stream("", b"BT /F1 12 Tf 20 200 Td (Hello) Tj ET") + .build() + }; + let mut s = Session::new(); + let bg = s.open("lang.pdf", None, Arc::new(pdf("<< /Type /Catalog /Pages 2 0 R /Lang (bg-BG) >>")), None).unwrap(); + let plain = s.open("plain.pdf", None, Arc::new(pdf("<< /Type /Catalog /Pages 2 0 R >>")), None).unwrap(); + let d = s.get(bg).unwrap(); + let docx = d.export_office(compare::OfficeFormat::Docx); + assert!(docx == pdfcraft_export::docx_with_language(&d.export_pages(), "lang", Some("bg-BG")), "Word names bg-BG"); + assert!(docx != pdfcraft_export::docx(&d.export_pages(), "lang"), "Word names a language"); + for format in [compare::OfficeFormat::Html, compare::OfficeFormat::HtmlSimple] { + let html = String::from_utf8(d.export_office(format)).unwrap(); + assert!(html.contains(""), "{html}"); + } + let d = s.get(plain).unwrap(); + assert!(d.export_office(compare::OfficeFormat::Docx) == pdfcraft_export::docx(&d.export_pages(), "plain"), "no language, as before"); + assert!(String::from_utf8(d.export_office(compare::OfficeFormat::Html)).unwrap().contains("\n")); + assert!(String::from_utf8(d.export_office(compare::OfficeFormat::HtmlSimple)).unwrap().contains("\n")); +} + #[test] fn exporting_office_files_keeps_images() { let mut s = Session::new(); diff --git a/crates/export/src/as_docx.rs b/crates/export/src/as_docx.rs index ab7451dc0e..107f3676e2 100644 --- a/crates/export/src/as_docx.rs +++ b/crates/export/src/as_docx.rs @@ -1,5 +1,5 @@ use super::Zip; -use super::as_html::esc; +use super::as_html::{esc, language_tag}; use super::layout::{Image, Item, Page, Table, body_size, items}; /// A colour's 8-bit sRGB components (NaN and out-of-range values clamp; NaN reads as 0). @@ -228,9 +228,28 @@ fn word_background(pages: &[Page]) -> Option { (same && first != [255, 255, 255]).then(|| format!("{:02X}{:02X}{:02X}", first[0], first[1], first[2])) } +/// The `w:lang` that makes `tag` the language of a Word document's text. Word keeps one language +/// per script: Chinese, Japanese and Korean are its East Asian text, Arabic, Hebrew, Persian and +/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. +fn word_lang(tag: &str) -> String { + let attribute = match tag.split('-').next().unwrap_or_default().to_ascii_lowercase().as_str() { + "zh" | "ja" | "ko" => "w:eastAsia", + "ar" | "he" | "fa" | "ur" => "w:bidi", + _ => "w:val", + }; + format!("", esc(tag)) +} + /// A Word document (.docx, Office Open XML). Ordinary pages flow as paragraphs and tables; /// pages with rotated text, or text that overlaps artwork, use floating shapes to retain their layout. pub fn docx(pages: &[Page], title: &str) -> Vec { + docx_with_language(pages, title, None) +} + +/// [`docx`] for a document in `language` (a BCP 47 tag such as `bg-BG`, as a PDF's `/Lang` gives +/// it): the default language of its text, so Word checks the spelling in that language rather +/// than in its own. A language that is not a well formed tag is left out. +pub fn docx_with_language(pages: &[Page], title: &str, language: Option<&str>) -> Vec { let mut body = String::new(); let mut media: Vec<(String, &Image)> = Vec::new(); // Word needs a paragraph between adjacent tables and after the last one in the body. @@ -338,11 +357,16 @@ xmlns:wp=\"http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawin doc_rels.push_str(""); } doc_rels.push_str(""); - let styles = "\n\ + let defaults = language_tag(language) + .map(|tag| format!("{}", word_lang(tag))) + .unwrap_or_default(); + let styles = format!( + "\n{defaults}\ \ \ \ -"; +" + ); let core = format!( "\n{}", esc(title) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 3bc6709e69..6ae767d09c 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -8,6 +8,20 @@ pub(crate) fn allowed(c: char) -> bool { matches!(c, '\t' | '\n' | '\r') || !(c.is_ascii_control() || c == '\u{FFFE}' || c == '\u{FFFF}') } +/// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary +/// language of two or three letters, then subtags of one to eight letters or digits, joined by +/// hyphens, 35 characters at most. Anything else names no language. +pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { + let tag = lang?.trim(); + let mut subtags = tag.split('-'); + let primary = subtags.next()?; + let well_formed = tag.len() <= 35 + && (2..=3).contains(&primary.len()) + && primary.bytes().all(|b| b.is_ascii_alphabetic()) + && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); + well_formed.then_some(tag) +} + pub(crate) fn esc(s: &str) -> String { let mut o = String::with_capacity(s.len()); for c in s.chars().filter(|c| allowed(*c)) { @@ -38,17 +52,31 @@ pub(crate) fn base64(data: &[u8]) -> String { /// One HTML file: headings and paragraphs, tables as real `` elements, images inline, a /// rule between pages. pub fn html(pages: &[Page], title: &str) -> String { - html_with_mode(pages, title, false) + html_with_language(pages, title, None) } /// A simple HTML export with no CSS; page layout and colours are omitted. pub fn html_simple(pages: &[Page], title: &str) -> String { - html_with_mode(pages, title, true) + html_simple_with_language(pages, title, None) } -fn html_with_mode(pages: &[Page], title: &str, simple: bool) -> String { +/// [`html`] for a document in `language` (a BCP 47 tag such as `bg-BG`, as a PDF's `/Lang` gives +/// it), which the page names (``) for spell checking, speech and hyphenation. A +/// language that is not a well formed tag is left out. +pub fn html_with_language(pages: &[Page], title: &str, language: Option<&str>) -> String { + html_with_mode(pages, title, false, language) +} + +/// [`html_simple`] for a document in `language`, named as [`html_with_language`] names it: a +/// language is not styling, so the simple export keeps it. +pub fn html_simple_with_language(pages: &[Page], title: &str, language: Option<&str>) -> String { + html_with_mode(pages, title, true, language) +} + +fn html_with_mode(pages: &[Page], title: &str, simple: bool, language: Option<&str>) -> String { + let lang = language_tag(language).map(|tag| format!(" lang=\"{}\"", esc(tag))).unwrap_or_default(); let mut s = format!( - "\n\n\n\n{}\n{}\n\n\n", + "\n\n\n\n{}\n{}\n\n\n", esc(title), if simple { "" diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 2b82919bbf..b8f10e7b68 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -22,14 +22,14 @@ mod as_markdown; mod as_rtf; mod layout; -pub use as_docx::docx; pub(crate) use as_docx::hex; #[cfg(test)] use as_docx::{MAX_MEDIA, docx_table}; +pub use as_docx::{docx, docx_with_language}; pub(crate) use as_html::esc; #[cfg(test)] use as_html::{allowed, base64}; -pub use as_html::{html, html_simple}; +pub use as_html::{html, html_simple, html_simple_with_language, html_with_language}; pub use as_markdown::markdown; #[cfg(test)] use as_markdown::{emphasis, paragraph}; diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index f630d98d5d..d62148aaf2 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -245,6 +245,44 @@ pub(crate) fn part(zip: &[u8], name: &str) -> String { panic!("{name} missing") } +/// The PDF's language (`/Lang`) is the default language of the Word document's text, in the +/// attribute Word keeps for its script, and the HTML page's. No language, or one that is not a +/// well formed tag, leaves both files as they were. +#[test] +fn word_and_html_take_the_documents_language() { + let pages = [page()]; + let styles = |lang| part(&docx_with_language(&pages, "Report", lang), "word/styles.xml"); + let defaults = |lang: &str| format!("{lang}"); + assert!(styles(Some("bg-BG")).contains(&defaults("")), "{}", styles(Some("bg-BG"))); + assert!(styles(Some(" en-GB ")).contains(""), "the tag is trimmed"); + assert!(styles(Some("zh-Hant-TW")).contains(&defaults("")), "East Asian text"); + assert!(styles(Some("ja")).contains(""), "East Asian text"); + assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); + assert!(styles(Some("HE")).contains(""), "right-to-left text"); + assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); + for lang in [ + None, + Some(""), + Some(" "), + Some("bg_BG"), + Some("b"), + Some("bulg-BG"), + Some("bg--BG"), + Some("bg-"), + Some("en-US-abcdefghi"), + Some("en-abcdefgh-abcdefgh-abcdefgh-abcdefgh"), + Some("en-\"x\""), + Some("1a-BG"), + Some("\u{431}\u{433}-BG"), + ] { + assert!(docx_with_language(&pages, "Report", lang) == docx(&pages, "Report"), "{lang:?}"); + assert_eq!(html_with_language(&pages, "Report", lang), html(&pages, "Report"), "{lang:?}"); + assert_eq!(html_simple_with_language(&pages, "Report", lang), html_simple(&pages, "Report"), "{lang:?}"); + } + // The simple export leaves out styling, not the language. + assert!(html_simple_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); +} + #[test] fn word_opens_text_with_control_characters_and_long_pages() { // #72: text extracted without a usable /ToUnicode holds C0 controls, which XML forbids; diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 252fb43fd2..36c415ec6b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7131,8 +7131,8 @@ milestone = "M10" status = "partial" commands = ["export.docx"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158). Not yet: a style change that wraps onto the next line still splits its paragraph there (the reading blocks keep style boundaries, #4158), exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." +evidence = ["crates/export/src/tests.rs::word_table_cell_widths_follow_the_columns_they_span", "crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/edit/src/tests.rs::reading_keeps_dotted_leader_entries_whole_and_in_order", "crates/engine/src/tests.rs::exporting_office_keeps_dotted_leader_entries_as_paragraphs", "crates/export/src/tests.rs::word_does_not_invent_borders_for_aligned_invoice_fields", "crates/export/src/tests.rs::word_overlap_does_not_consume_invoice_body_space", "crates/export/src/tests.rs::word_keeps_a_page_with_a_clear_image_and_big_title_in_flow", "crates/export/src/tests.rs::word_rotated_pages_keep_separate_page_sizes_and_unique_shapes", "crates/engine/src/tests.rs::office_export_positions_follow_the_displayed_crop_rotation_and_units", "crates/automation/tests/it/automation.rs::exporting_an_invoice_to_word_keeps_the_watermark_out_of_flow", "crates/export/src/tests.rs::docx_is_a_word_package", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/engine/src/tests.rs::exporting_office_files_keeps_text_colour", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/export/src/tests.rs::a_link_inside_a_sentence_is_not_a_table_column", "crates/export/src/tests.rs::sentences_that_run_on_into_links_are_not_a_table", "crates/export/src/tests.rs::a_superscript_beside_a_number_keeps_its_table", "crates/export/src/tests.rs::word_opens_text_with_control_characters_and_long_pages", "crates/engine/src/tests.rs::exporting_office_files_keeps_images", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/export/src/tests.rs::docx_keeps_a_shared_page_background", "crates/edit/src/tests.rs::a_page_painted_first_with_a_full_page_fill_has_that_background", "crates/engine/src/tests.rs::exporting_to_word_keeps_the_page_background", "crates/export/src/tests.rs::an_italic_word_stays_in_its_sentence", "crates/export/src/tests.rs::a_sentence_keeps_its_reading_order", "crates/export/src/tests.rs::runs_keep_the_spaces_the_source_had_between_them", "crates/export/src/tests.rs::a_paragraph_is_a_heading_only_when_its_text_is", "crates/automation/tests/it/automation.rs::exporting_to_word_keeps_an_italic_word_inside_its_sentence", "crates/export/src/tests.rs::word_and_html_take_the_documents_language", "crates/engine/src/tests.rs::office_exports_take_the_documents_language"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Flowing Word document: paragraphs, Heading 1/2 from larger text, bold/italic, text colour (fill as sRGB; gray/CMYK converted naively, colour spaces via cs/sc read as black), borderless detected tables with spans/empty cells (cell widths sum the grid columns they occupy, including cells after spans, verified in the XML; a row whose blocks run on into each other, such as a link or superscript inside a sentence, or overlap, as a paragraph's lines do, is text and not a table row, #2806), per-page sections/sizes, the page colour (a full-page opaque fill painted first, such as Edit a PDF ▸ Background) as the document background when every page shares one non-white colour (Word has no per-page background, so differing colours are dropped; #2992; checked in the XML only, not yet opened in Word), and positioned editable text/artwork only for pages with rotated text or text overlapped by an image or watermark-sized text, other pages flowing with inline images (#773); synthetic invoice regression; dotted-leader contents entries keep the title, dots and page number together in separate paragraphs (#1757); the emitted package validates with Open XML SDK 2.20 (Office 2010 schema). Word/LibreOffice rendering of the positioned output remains unverified. Pictures on positioned pages are DrawingML anchors behind the text (a VML rectangle with an image showed no picture, #1936; not yet opened in Word); a Word file embeds at most 1000 pictures and 512 MiB of them. The earlier flowing output was checked with macOS textutil and opened in Microsoft Word (Windows). Characters XML forbids are dropped; the page size stays within Word's 22 in. A block that continues another on its line (an italic case name, a link, a superscript) is a run in its own size, bold/italic and colour inside that paragraph, keeping the source's word space between them, rather than a paragraph of its own; a lead-in in bold no longer makes its whole line a heading (#4158). The PDF's language (`/Lang`, when it is a well formed BCP 47 tag) is the default language of the text, in the attribute Word keeps for its script (East Asian, right to left or other), so Word checks the spelling in it rather than in English (checked in the XML only, not yet opened in Word). Not yet: a style change that wraps onto the next line still splits its paragraph there (the reading blocks keep style boundaries, #4158), exact font matching, vector rules, arbitrary paint-order/transparency, general exact layout fidelity." [[feature]] id = "create.export-rtf" @@ -7255,8 +7255,8 @@ milestone = "M10" status = "partial" commands = ["export.html"] tools = ["doc_export_office"] -evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops"] -notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour), inline images, and detected tables with spans/empty cells. Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." +evidence = ["crates/export/src/tests.rs::table_spans_do_not_consume_text_they_do_not_emit", "crates/export/src/tests.rs::html_has_headings_paragraphs_and_images_in_order", "crates/export/src/tests.rs::text_colour_survives_every_format", "crates/export/src/tests.rs::grid_text_becomes_a_real_table_in_every_format", "crates/automation/tests/it/automation.rs::exporting_to_word_html_and_rtf", "crates/ui-egui/tests/it/compare.rs::export_to_word_html_and_rtf", "crates/automation/tests/it/automation.rs::exporting_office_files_says_which_pages_left_form_text_out", "crates/edit/src/tests.rs::reading_follows_forms_as_far_as_the_renderer_and_says_where_it_stops", "crates/export/src/tests.rs::word_and_html_take_the_documents_language", "crates/engine/src/tests.rs::office_exports_take_the_documents_language"] +notes = "Text in slots covered by a preceding table-cell span remains in paragraph flow instead of being discarded. Single HTML file with headings, paragraphs (with text colour), inline images, and detected tables with spans/empty cells. The page names the PDF's language (``, from a well formed `/Lang`). Text and images in form XObjects are read 50 levels deep, as the renderer does, and up to 50 000 forms a page; the tool lists pages past that in truncated_pages, with a warning (#3120). Not yet: multi-page folder output, CSS from fonts." [[feature]] id = "create.export-text" From 0cae1c518b863ab660272b375710a3261e1a3c34 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:15 +0200 Subject: [PATCH 02/37] A language tag longer than 35 characters is still a tag BCP 47 sets no maximum length; 35 characters is only the least a protocol must hold. A well formed tag such as en-US-u-ca-gregory-nu-latn-co-phonebk was left out of both exports. The bound is now 255, kept only so a PDF's /Lang can't put an unbounded string in the file. --- crates/export/src/as_html.rs | 8 ++++++-- crates/export/src/tests.rs | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 6ae767d09c..45f6ecafca 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -8,14 +8,18 @@ pub(crate) fn allowed(c: char) -> bool { matches!(c, '\t' | '\n' | '\r') || !(c.is_ascii_control() || c == '\u{FFFE}' || c == '\u{FFFF}') } +/// The longest language tag the exports write. BCP 47 sets no maximum (35 characters is only +/// the least a protocol must hold), so this bounds only what a PDF's `/Lang` can put in a file. +const LONGEST_TAG: usize = 255; + /// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary /// language of two or three letters, then subtags of one to eight letters or digits, joined by -/// hyphens, 35 characters at most. Anything else names no language. +/// hyphens, [`LONGEST_TAG`] characters at most. Anything else names no language. pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { let tag = lang?.trim(); let mut subtags = tag.split('-'); let primary = subtags.next()?; - let well_formed = tag.len() <= 35 + let well_formed = tag.len() <= LONGEST_TAG && (2..=3).contains(&primary.len()) && primary.bytes().all(|b| b.is_ascii_alphabetic()) && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index d62148aaf2..ba5f95f66d 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -260,6 +260,11 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); assert!(styles(Some("HE")).contains(""), "right-to-left text"); assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); + // A tag longer than 35 characters is still a tag. + let long = "en-US-u-ca-gregory-nu-latn-co-phonebk"; + assert!(styles(Some(long)).contains(&format!("")), "{long}"); + assert!(html_with_language(&pages, "Report", Some(long)).contains(&format!("")), "{long}"); + let too_long = format!("en{}", "-abcdefgh".repeat(29)); for lang in [ None, Some(""), @@ -270,7 +275,7 @@ fn word_and_html_take_the_documents_language() { Some("bg--BG"), Some("bg-"), Some("en-US-abcdefghi"), - Some("en-abcdefgh-abcdefgh-abcdefgh-abcdefgh"), + Some(too_long.as_str()), Some("en-\"x\""), Some("1a-BG"), Some("\u{431}\u{433}-BG"), From 561cc0d7079fef5d2f8b2a3af4715c91b3c1cb92 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:33 +0200 Subject: [PATCH 03/37] A language tag can't end in a singleton A singleton (one letter or digit) starts an extension, or with x private use, and BCP 47 needs a subtag after it. en-x and en-a were written as the document's language. They and a singleton straight after another are now left out; one-character subtags inside private use stay allowed. --- crates/export/src/as_html.rs | 29 +++++++++++++++++++++++------ crates/export/src/tests.rs | 6 ++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/export/src/as_html.rs b/crates/export/src/as_html.rs index 45f6ecafca..8c250ca484 100644 --- a/crates/export/src/as_html.rs +++ b/crates/export/src/as_html.rs @@ -14,16 +14,33 @@ const LONGEST_TAG: usize = 255; /// `lang` when it is a well formed language tag (BCP 47, as a PDF's `/Lang` holds one): a primary /// language of two or three letters, then subtags of one to eight letters or digits, joined by -/// hyphens, [`LONGEST_TAG`] characters at most. Anything else names no language. +/// hyphens, [`LONGEST_TAG`] characters at most. A singleton (one letter or digit) starts an +/// extension, or with `x` private use, and needs a subtag after it. Anything else names no +/// language. pub(crate) fn language_tag(lang: Option<&str>) -> Option<&str> { let tag = lang?.trim(); let mut subtags = tag.split('-'); let primary = subtags.next()?; - let well_formed = tag.len() <= LONGEST_TAG - && (2..=3).contains(&primary.len()) - && primary.bytes().all(|b| b.is_ascii_alphabetic()) - && subtags.all(|s| (1..=8).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric())); - well_formed.then_some(tag) + if tag.len() > LONGEST_TAG || !(2..=3).contains(&primary.len()) || !primary.bytes().all(|b| b.is_ascii_alphabetic()) { + return None; + } + // In private use every subtag is data, one character long or not. + let (mut private, mut singleton) = (false, false); + for s in subtags { + if !(1..=8).contains(&s.len()) || !s.bytes().all(|b| b.is_ascii_alphanumeric()) { + return None; + } + if s.len() == 1 && !private { + if singleton { + return None; + } + private = s.eq_ignore_ascii_case("x"); + singleton = true; + } else { + singleton = false; + } + } + (!singleton).then_some(tag) } pub(crate) fn esc(s: &str) -> String { diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index ba5f95f66d..5820fc57bc 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -265,6 +265,8 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some(long)).contains(&format!("")), "{long}"); assert!(html_with_language(&pages, "Report", Some(long)).contains(&format!("")), "{long}"); let too_long = format!("en{}", "-abcdefgh".repeat(29)); + // Private use takes one-character subtags. + assert!(html_with_language(&pages, "Report", Some("en-x-a-b")).contains("")); for lang in [ None, Some(""), @@ -274,6 +276,10 @@ fn word_and_html_take_the_documents_language() { Some("bulg-BG"), Some("bg--BG"), Some("bg-"), + Some("en-x"), + Some("en-a"), + Some("en-a-x-private"), + Some("en-US-u-ca-x"), Some("en-US-abcdefghi"), Some(too_long.as_str()), Some("en-\"x\""), From a427cd2de59e981449ebdb8072493d69d15b5320 Mon Sep 17 00:00:00 2001 From: oracola91 <118899664+oracola91@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:10:57 +0200 Subject: [PATCH 04/37] Word takes the script a language tag names pa-Arab-PK is Punjabi in Arabic script, which Word keeps as right-to-left text, but word_lang looked only at the language and wrote w:val. A script subtag (Arab, Hebr, Syrc, Thaa; Hans, Hant, Hani, Jpan, Kore, Hang, Hira, Kana; Latn, Cyrl, Grek) now decides over the language. It is read only in its own place, after the language and any extended languages, so private use such as en-x-arab names no script. --- crates/export/src/as_docx.rs | 20 ++++++++++++++++---- crates/export/src/tests.rs | 6 ++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/crates/export/src/as_docx.rs b/crates/export/src/as_docx.rs index 107f3676e2..4fa7edfce1 100644 --- a/crates/export/src/as_docx.rs +++ b/crates/export/src/as_docx.rs @@ -230,11 +230,23 @@ fn word_background(pages: &[Page]) -> Option { /// The `w:lang` that makes `tag` the language of a Word document's text. Word keeps one language /// per script: Chinese, Japanese and Korean are its East Asian text, Arabic, Hebrew, Persian and -/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. +/// Urdu its right-to-left text, and any other language its Latin, Greek and Cyrillic text. A +/// script the tag names (`pa-Arab`, `zh-Latn`) decides over its language. fn word_lang(tag: &str) -> String { - let attribute = match tag.split('-').next().unwrap_or_default().to_ascii_lowercase().as_str() { - "zh" | "ja" | "ko" => "w:eastAsia", - "ar" | "he" | "fa" | "ur" => "w:bidi", + let mut subtags = tag.split('-'); + let primary = subtags.next().unwrap_or_default().to_ascii_lowercase(); + // The script comes straight after the language and any three-letter extended languages; a + // singleton before it starts an extension or private use, which names no script. + let script = subtags + .find(|s| s.len() != 3 || !s.bytes().all(|b| b.is_ascii_alphabetic())) + .filter(|s| s.len() == 4 && s.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_ascii_lowercase); + let attribute = match (script.as_deref(), primary.as_str()) { + (Some("arab" | "hebr" | "syrc" | "thaa"), _) => "w:bidi", + (Some("hans" | "hant" | "hani" | "jpan" | "kore" | "hang" | "hira" | "kana"), _) => "w:eastAsia", + (Some("latn" | "cyrl" | "grek"), _) => "w:val", + (_, "zh" | "ja" | "ko") => "w:eastAsia", + (_, "ar" | "he" | "fa" | "ur") => "w:bidi", _ => "w:val", }; format!("", esc(tag)) diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 5820fc57bc..baaf323117 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -259,6 +259,12 @@ fn word_and_html_take_the_documents_language() { assert!(styles(Some("ja")).contains(""), "East Asian text"); assert!(styles(Some("ar-EG")).contains(&defaults("")), "right-to-left text"); assert!(styles(Some("HE")).contains(""), "right-to-left text"); + // A script subtag decides over the language; private use names no script. + assert!(styles(Some("pa-Arab-PK")).contains(""), "Punjabi in Arabic script"); + assert!(styles(Some("sr-Latn")).contains(""), "Serbian in Latin script"); + assert!(styles(Some("zh-Latn")).contains(""), "Chinese in Latin script"); + assert!(styles(Some("en-x-arab")).contains(""), "private use"); + assert!(styles(Some("az-Latn-x-arab")).contains(""), "private use"); assert!(html_with_language(&pages, "Report", Some("bg-BG")).starts_with("\n\n")); // A tag longer than 35 characters is still a tag. let long = "en-US-u-ca-gregory-nu-latn-co-phonebk"; From 77cebccda21e7bc7dccc2d8f5e66f81865656688 Mon Sep 17 00:00:00 2001 From: xullul <32987931+xullul@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:01:31 +0700 Subject: [PATCH 05/37] M7: adapt embedded Thai source fonts without replacing their programs Add: - Recover editor-only Unicode cmaps for Identity-H TrueType subsets and shape verified THSarabunPSK positional glyphs into their original PDF character codes. - Recognize narrowly fingerprinted damaged Thai exporter maps without guessing mappings for unrelated fonts; callers must record and isolate any repairs. Change: - Preserve the original PDF font programs and refuse missing contextual glyphs. - Allow modern Thai GPOS positioning without requiring optional GSUB, adapting that correction from dekkeng's PR #2811 at fba0a29a. Refs #3185 and #2142. Font/edit tests: 535 passed. Formatting, scoped strict Clippy, layers, assets, parity and the fonts/edit wasm check passed. --- crates/fonts/README.md | 7 + crates/fonts/src/pdf.rs | 89 +++++-- crates/fonts/src/pdf/tests.rs | 41 ++++ crates/fonts/src/pdf/thai.rs | 372 +++++++++++++++++++++++++++++ crates/fonts/src/source_shaping.rs | 53 +++- 5 files changed, 543 insertions(+), 19 deletions(-) create mode 100644 crates/fonts/src/pdf/thai.rs diff --git a/crates/fonts/README.md b/crates/fonts/README.md index 096f1a263e..c462c428d4 100644 --- a/crates/fonts/README.md +++ b/crates/fonts/README.md @@ -86,3 +86,10 @@ Document font inputs use `embedding`: bounded static TrueType/OpenType-CFF valid exact program preservation, Unicode/CID/GID mapping and full-face subset completion. `shaping` uses the same program with harfrust and UAX #9; source UI fallback fonts are never exported. See [existing-text editing](../../docs/editing-existing-text.md). + +`pdf::embedded_sfnt_within` reconstructs a missing Unicode cmap for Identity-H TrueType +subsets from the PDF's ToUnicode and CID-to-glyph maps. Only the in-memory editor copy changes; +the PDF's font program stays intact. `pdf::LegacyThaiFont` shapes verified THSarabunPSK +positional glyphs into their original PDF codes, retaining the source face and weight. Missing +contextual glyphs are refused. `pdf::recovered_thai_map` recognizes narrowly fingerprinted +damaged exporter maps; callers must record repairs and apply them to page-owned font copies. diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 240b1c4d55..4fab8a0313 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -116,22 +116,23 @@ pub fn embedded_sfnt_within(doc: &Document, font: &Dict, budget: &mut DecodeBudg if face.charmap().mappings().next().is_some() { return Some(bytes); } - // Some PDF generators keep a Mac format-6 code→glyph cmap. egui accepts Unicode - // format-4/12 cmaps. Combine the PDF's Unicode meanings with the source glyph IDs; - // only this in-memory copy gets a new cmap, never the working PDF or a font asset. - let Ok(cmap) = face.cmap() else { continue }; + // PDF subsets may remove the Unicode cmap entirely. Rebuild an editor-only + // cmap from ToUnicode and the actual CID-to-glyph map; source bytes stay unchanged. let metrics = Metrics::from_dict(doc, font); - if metrics.composite { - continue; - } let mut mappings = std::collections::BTreeMap::new(); - for code in 0..=255 { - let Some(text) = metrics.unicode.text(code) else { continue }; - let Some(ch) = only_char(&text) else { continue }; - let Some(glyph) = cmap.map_codepoint(code).or_else(|| cmap.map_codepoint(code + 0xf000)) else { continue }; - let glyph = glyph.to_u32(); - if glyph > 0 { - mappings.entry(u32::from(ch)).or_insert(glyph); + if metrics.composite { + if let Some((map, _)) = composite_glyph_maps(doc, font, &bytes, &metrics, budget) { + mappings = map; + } + } else if let Ok(cmap) = face.cmap() { + for code in 0..=255 { + let Some(text) = metrics.unicode.text(code) else { continue }; + let Some(ch) = only_char(&text) else { continue }; + let Some(glyph) = cmap.map_codepoint(code).or_else(|| cmap.map_codepoint(code + 0xf000)) else { continue }; + let glyph = glyph.to_u32(); + if glyph > 0 { + mappings.entry(u32::from(ch)).or_insert(glyph); + } } } if !mappings.is_empty() @@ -214,14 +215,14 @@ fn embedded_widths(doc: &Document, font: &Dict, m: &mut Metrics) { } } -/// Replace an existing cmap directory entry with a bounded Unicode format-12 table. +/// Add or replace an editor-only cmap with a bounded Unicode format-12 table. /// Checked ranges prevent damaged sfnt offsets from indexing outside the source program. fn append_editor_cmap(bytes: &mut Vec, mappings: &std::collections::BTreeMap) -> Option<()> { if mappings.len() > 65536 { return None; } let count = u16::from_be_bytes(bytes.get(4..6)?.try_into().ok()?) as usize; - if count > 256 { + if count == 0 || count > 256 { return None; } let mut entry = None; @@ -236,7 +237,10 @@ fn append_editor_cmap(bytes: &mut Vec, mappings: &std::collections::BTreeMap head = Some(u32::from_be_bytes(record.get(8..12)?.try_into().ok()?) as usize); } } - let entry = entry?; + let (entry, head) = match entry { + Some(entry) => (entry, head), + None => insert_editor_cmap_slot(bytes, count, head)?, + }; let mut cmap = Vec::with_capacity(28 + mappings.len() * 12); cmap.extend_from_slice(&[0, 0, 0, 1, 0, 3, 0, 10, 0, 0, 0, 12, 0, 12, 0, 0]); cmap.extend_from_slice(&(16_u32 + u32::try_from(mappings.len()).ok()? * 12).to_be_bytes()); @@ -264,6 +268,17 @@ fn append_editor_cmap(bytes: &mut Vec, mappings: &std::collections::BTreeMap bytes.get_mut(entry + 8..entry + 12)?.copy_from_slice(&offset.to_be_bytes()); bytes.get_mut(entry + 12..entry + 16)?.copy_from_slice(&u32::try_from(cmap.len()).ok()?.to_be_bytes()); bytes.extend_from_slice(&cmap); + // SFNT directories are sorted by tag. Some readers tolerate the unsorted directory of + // a PDF subset, but shaping engines may binary-search it and otherwise miss the cmap. + // Adding a slot increased the directory count; include the new record in this sort. + let tables = usize::from(u16::from_be_bytes(bytes.get(4..6)?.try_into().ok()?)); + let end = 12usize.checked_add(tables.checked_mul(16)?)?; + let mut records: Vec<[u8; 16]> = bytes.get(12..end)?.as_chunks::<16>().0.to_vec(); + records.sort_by_key(|r| [r[0], r[1], r[2], r[3]]); + for (index, record) in records.iter().enumerate() { + let at = 12 + index * 16; + bytes.get_mut(at..at + 16)?.copy_from_slice(record); + } if let Some(head) = head { bytes.get_mut(head.checked_add(8)?..head.checked_add(12)?)?.fill(0); let adjustment = 0xb1b0afba_u32.wrapping_sub(checksum(bytes)); @@ -272,6 +287,42 @@ fn append_editor_cmap(bytes: &mut Vec, mappings: &std::collections::BTreeMap Some(()) } +/// Reserve a new cmap directory slot, retaining every original table payload. +fn insert_editor_cmap_slot(bytes: &mut Vec, mut count: usize, mut head: Option) -> Option<(usize, Option)> { + if count >= 256 { + return None; + } + let end = 12usize.checked_add(count.checked_mul(16)?)?; + bytes.get(..end)?; + // Keep all original table bytes, shifting them by one directory record. + for i in 0..count { + let at = 12 + i * 16; + let offset = u32::from_be_bytes(bytes.get(at + 8..at + 12)?.try_into().ok()?); + let len = u32::from_be_bytes(bytes.get(at + 12..at + 16)?.try_into().ok()?); + if (offset as usize) < end { + return None; + } + bytes.get(offset as usize..offset.checked_add(len)? as usize)?; + } + bytes.splice(end..end, [0; 16]); + for i in 0..count { + let at = 12 + i * 16; + let offset = u32::from_be_bytes(bytes.get(at + 8..at + 12)?.try_into().ok()?).checked_add(16)?; + bytes.get_mut(at + 8..at + 12)?.copy_from_slice(&offset.to_be_bytes()); + } + head = head.and_then(|offset| offset.checked_add(16)); + bytes.get_mut(end..end + 4)?.copy_from_slice(b"cmap"); + count += 1; + let count = u16::try_from(count).ok()?; + let selector = count.ilog2() as u16; + let range = (1u16.checked_shl(u32::from(selector))?).checked_mul(16)?; + bytes.get_mut(4..6)?.copy_from_slice(&count.to_be_bytes()); + bytes.get_mut(6..8)?.copy_from_slice(&range.to_be_bytes()); + bytes.get_mut(8..10)?.copy_from_slice(&selector.to_be_bytes()); + bytes.get_mut(10..12)?.copy_from_slice(&count.checked_mul(16)?.checked_sub(range)?.to_be_bytes()); + Some((end, head)) +} + fn descriptor_flags(doc: &Document, descriptor: &Dict) -> u32 { let value = descriptor.get(b"Flags").and_then(|v| doc.resolve(v).as_f64()).unwrap_or(0.0) as i64; u32::try_from(value).unwrap_or(0) @@ -1507,3 +1558,7 @@ mod encoding_tests { assert_eq!(budget.remaining(), 0); } } + +mod thai; +use thai::composite_glyph_maps; +pub use thai::{LegacyThaiFont, recovered_thai_map}; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index c2acef191c..f6051feedd 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -795,3 +795,44 @@ fn low_codes_resolve_to_the_covering_range() { assert_eq!(covered, [5, 6, 7, 100, 101]); assert_eq!((u.covering(0xFFFF).is_some(), u.covering(0x1_0001).is_some(), u.covering(0x1_0002).is_none()), (true, true, true)); } + +#[test] +fn composite_subset_without_cmap_uses_pdf_glyph_ids_in_editor_and_shaper_without_replacing_the_program() { + use skrifa::{MetadataProvider as _, raw::TableProvider as _}; + let original = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let expected = skrifa::FontRef::new(original).unwrap().charmap().map('Ω').unwrap(); + let mut subset = original.to_vec(); + let count = u16::from_be_bytes(subset[4..6].try_into().unwrap()) as usize; + let records: Vec<_> = subset[12..12 + count * 16].as_chunks::<16>().0.iter().filter(|r| &r[..4] != b"cmap").map(|r| r.to_vec()).collect(); + subset[4..6].copy_from_slice(&(records.len() as u16).to_be_bytes()); + for (index, record) in records.iter().enumerate() { + subset[12 + index * 16..28 + index * 16].copy_from_slice(record); + } + assert!(skrifa::FontRef::new(&subset).unwrap().cmap().is_err()); + let mut doc = Document::new_empty(); + let program = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), &subset))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), Object::Ref(program)); + let mut gids = vec![0; 36]; + gids[34..36].copy_from_slice(&(expected.to_u32() as u16).to_be_bytes()); + let gids = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), &gids))); + let mut descendant = Dict::new(); + descendant.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + descendant.set(b"CIDToGIDMap".to_vec(), Object::Ref(gids)); + let unicode = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), b"1 beginbfchar <0011> <03a9> endbfchar"))); + let source = font( + &mut doc, + vec![ + ("Subtype", Object::name("Type0")), + ("Encoding", Object::name("Identity-H")), + ("DescendantFonts", Object::Array(vec![Object::Dict(descendant)])), + ("ToUnicode", Object::Ref(unicode)), + ], + ); + let editor = embedded_sfnt(&doc, &source).expect("the source outlines must be usable in the editor"); + assert_eq!(skrifa::FontRef::new(&editor).unwrap().charmap().map('Ω'), Some(expected)); + let shaped = crate::source_shaping::shape(&editor, "Ω").expect("the rebuilt cmap must work in the actual shaping engine too"); + assert_eq!(shaped.glyphs[0].id, expected); + let Object::Stream(saved) = doc.get(program).as_ref().clone() else { panic!("fixture") }; + assert_eq!(saved.decoded().unwrap(), subset, "the source PDF font program stays byte-for-byte intact"); +} diff --git a/crates/fonts/src/pdf/thai.rs b/crates/fonts/src/pdf/thai.rs new file mode 100644 index 0000000000..54784c291c --- /dev/null +++ b/crates/fonts/src/pdf/thai.rs @@ -0,0 +1,372 @@ +//! Recover a narrowly identified legacy Thai exporter map, not arbitrary missing Unicode. +//! +//! Some THSarabunPSK Identity-H subsets retain original glyph IDs but write Latin/control +//! destinations for Thai in ToUnicode (including an accidental 00ff..01ff range). The intact +//! Thai anchors identify the original nominal Thai order. Only that complete fingerprint is +//! accepted; correct maps and unrelated fonts are never changed. No font artwork is bundled. + +use super::{Dict, Document, Metrics, dict, only_char, unicode_map}; +use pdfcraft_cos::{DecodeBudget, Object}; +use std::collections::BTreeMap; + +/// Recover the nominal Thai codes and the presentation marks of this damaged legacy export. +/// The caller must record the repair, and attach the map to a copy of the page's font. +pub fn recovered_thai_map(doc: &Document, font: &Dict) -> Option> { + let base = String::from_utf8_lossy(font.name(b"BaseFont")?); + let family = base.rsplit('+').next()?; + if !matches!(family, "THSarabunPSK" | "THSarabunPSK-Bold" | "THSarabunPSK-Italic" | "THSarabunPSK-BoldItalic") + || font.name(b"Subtype") != Some(b"Type0") + || font.name(b"Encoding") != Some(b"Identity-H") + { + return None; + } + let desc = dict(doc, font.get(b"DescendantFonts").and_then(|a| doc.resolve(a).as_array().and_then(|a| a.first()).cloned()).as_ref())?; + if desc.name(b"CIDToGIDMap") != Some(b"Identity") { + return None; + } + let map = unicode_map(doc, font, true); + // Five separated consonants and three contextual marks must agree, together with the + // corrupt destinations and corrupt range. This is deliberately stricter than font-name + // detection: a renamed/subsetted font with different glyph order must not be guessed. + let fingerprint = [ + (217, '\u{0e04}'), + (226, '\u{0e0d}'), + (250, '\u{0e25}'), + (256, '\u{0e2b}'), + (258, '\u{0e2d}'), + (356, '\u{0e4c}'), + (358, '\u{0e31}'), + (360, '\u{0e47}'), + (214, '\u{000f}'), + (240, ')'), + (248, '#'), + (261, '>'), + (263, '@'), + (265, '>'), + (273, 'N'), + (352, 'N'), + (353, 'O'), + ]; + let matches = |pairs: &[(u32, char)], last: u16| { + pairs.iter().all(|&(code, ch)| map.text(code).is_some_and(|t| t.chars().eq([ch]))) + && map.ranges.iter().any(|r| r.lo == 255 && r.hi == 511 && r.units.as_ref() == [last]) + }; + // The same exporter emitted three distinct subset maps. Each must match its entire + // intact-anchor and corrupt-destination fingerprint, including the oversized range. + let body = matches(&fingerprint, 0x32); + let attachment = matches( + &[ + (217, '\u{0e04}'), + (256, '\u{0e2b}'), + (258, '\u{0e2d}'), + (297, '\u{0e58}'), + (347, '\u{0e48}'), + (356, '\u{0e4c}'), + (358, '\u{0e31}'), + (360, '\u{0e47}'), + (214, '\u{000f}'), + (226, '\u{000f}'), + (240, '\u{0027}'), + (248, '/'), + (250, '1'), + (265, 'B'), + (273, 'N'), + (352, 'N'), + (353, 'O'), + ], + 0x32, + ); + let bold = matches( + &[ + (217, '\u{0e04}'), + (226, '\u{0e0d}'), + (232, '\u{0e13}'), + (234, '\u{0e15}'), + (236, '\u{0e17}'), + (243, '\u{0e1e}'), + (250, '\u{0e25}'), + (256, '\u{0e2b}'), + (258, '\u{0e2d}'), + (356, '\u{0e4c}'), + (214, '\u{000f}'), + (240, '!'), + (248, '1'), + (263, '@'), + (273, 'N'), + (352, 'N'), + (353, 'O'), + ], + 0x30, + ); + if !(body || attachment || bold) { + return None; + } + let mut out: Vec<_> = (214u16..=298) + .filter_map(|gid| { + let unicode = u32::from(gid) + if gid <= 271 { 0xd2b } else { 0xd2f }; + char::from_u32(unicode).map(|ch| (gid, ch)) + }) + .collect(); + // Contextual marks have the same character as the nominal mark, with a moved outline. + out.extend([ + (343, '\u{0e34}'), + (344, '\u{0e35}'), + (347, '\u{0e48}'), + (351, '\u{0e4c}'), + (352, '\u{0e48}'), + (353, '\u{0e49}'), + (356, '\u{0e4c}'), + (358, '\u{0e31}'), + (360, '\u{0e47}'), + ]); + Some(out) +} + +/// Source glyph positions and the original PDF character codes that draw them. Empty codes +/// represent whitespace advances: no glyph is added to the font or substituted for a space. +pub struct ExistingThaiRun { + pub shaped: crate::source_shaping::ShapedRun, + pub codes: Vec>, +} + +/// Unicode-to-glyph and glyph-to-PDF-code maps for a two-byte Identity-H TrueType font. +/// Used by the editor-only cmap adapter and by direct source-font shaping. +pub(super) fn composite_glyph_maps( + doc: &Document, + font: &Dict, + bytes: &[u8], + metrics: &Metrics, + budget: &mut DecodeBudget, +) -> Option<(BTreeMap, BTreeMap)> { + use skrifa::{MetadataProvider as _, raw::TableProvider as _}; + if font.name(b"Encoding") != Some(b"Identity-H") { + return None; + } + let descendant = dict(doc, font.get(b"DescendantFonts").and_then(|a| doc.resolve(a).as_array().and_then(|a| a.first()).cloned()).as_ref())?; + let cid_map = match descendant.get(b"CIDToGIDMap").map(|m| doc.resolve(m)).as_deref() { + None => None, + Some(Object::Name(n)) if n == b"Identity" => None, + Some(Object::Stream(s)) => Some(budget.decode_within(s, 1 << 17).ok()?), + _ => return None, + }; + let face = skrifa::FontRef::new(bytes).ok()?; + let loca = face.loca(None).ok()?; + let advances = face.glyph_metrics(skrifa::instance::Size::unscaled(), skrifa::instance::LocationRef::default()); + let outlined = |gid: u32| loca.get_raw(gid as usize).zip(loca.get_raw(gid as usize + 1)).is_some_and(|(a, b)| b > a); + let glyph_of = |cid: u16| -> Option { + match &cid_map { + None => Some(u32::from(cid)), + Some(map) => { + let at = usize::from(cid).checked_mul(2)?; + Some(u32::from(u16::from_be_bytes(map.get(at..at + 2)?.try_into().ok()?))) + } + } + }; + let mut unicode = BTreeMap::new(); + let mut codes = BTreeMap::new(); + for code in 0..=u16::MAX { + let Some(gid) = glyph_of(code) else { continue }; + if gid == 0 || gid as usize >= loca.len().saturating_sub(1) { + continue; + } + let text = metrics.text_of(u32::from(code)); + let ch = text.as_deref().and_then(only_char); + let space = ch == Some(' ') && advances.advance_width(skrifa::GlyphId::new(gid)).is_some_and(|w| w > 0.0); + if !outlined(gid) && !space { + continue; + } + codes.entry(gid).or_insert(code); + if let Some(ch) = ch.filter(|c| !c.is_control()) { + unicode.entry(u32::from(ch)).and_modify(|old: &mut u32| *old = (*old).min(gid)).or_insert(gid); + } + } + // The verified THSarabunPSK glyph order is retained by these PDF subsets. Restore the + // Windows Thai PUA alternates for HarfRust's legacy shaper, only where outlines survive. + // See https://linux.thai.net/~thep/th-otf/shaping.html (legacy PUA assignments). + let family = metrics.base_font.rsplit('+').next().unwrap_or(&metrics.base_font); + let alternate_meanings = metrics.text_of(352).is_some_and(|t| t == "่") && metrics.text_of(353).is_some_and(|t| t == "้"); + if family.starts_with("THSarabunPSK") + && unicode.get(&0x0e01) == Some(&214) + && unicode.get(&0x0e2d) == Some(&258) + && (alternate_meanings || unicode.contains_key(&0xf70a)) + { + for (index, gid) in (342u32..=368).enumerate() { + if outlined(gid) + && let Some(code) = match &cid_map { + None => u16::try_from(gid).ok(), + Some(_) => codes.get(&gid).copied(), + } + { + unicode.insert(0xf700 + index as u32, gid); + codes.entry(gid).or_insert(code); + } + } + } + Some((unicode, codes)) +} + +/// Shape legacy Thai using the PDF's own outlines and return codes for that same font. +/// No font program, family, weight or size is replaced. A subset missing a contextual glyph +/// is refused so the caller can try another subset of the same face or ask for its full font. +pub struct LegacyThaiFont { + bytes: Vec, + codes: BTreeMap, + space: f64, +} + +impl LegacyThaiFont { + /// Read and verify the source font once for all wrapping/preview measurements. + pub fn from_pdf(doc: &Document, font: &Dict, space: f64) -> Result, String> { + use skrifa::MetadataProvider as _; + let m = Metrics::from_dict(doc, font); + if !m.base_font.rsplit('+').next().unwrap_or(&m.base_font).starts_with("THSarabunPSK") { + return Ok(None); + } + if !space.is_finite() || space < 0.0 { + return Err("the source space width is invalid".into()); + } + let bytes = super::embedded_sfnt(doc, font).ok_or("the source Thai font has no readable glyph program")?; + let face = skrifa::FontRef::new(&bytes).map_err(|e| e.to_string())?; + let cmap = face.charmap(); + if cmap.map('ก').is_none() || cmap.map('อ').is_none() || cmap.map('\u{f70a}').is_none() { + return Ok(None); + } + let (_, codes) = composite_glyph_maps(doc, font, &bytes, &m, &mut DecodeBudget::new(1 << 17, 0)) + .ok_or("the Thai source glyph codes cannot be recovered safely")?; + Ok(Some(Self { bytes, codes, space })) + } + + /// Shape into glyphs that are already in this exact source face. + pub fn shape(&self, text: &str) -> Result { + use skrifa::MetadataProvider as _; + if text.len() > 262_144 || text.contains(['\n', '\r']) { + return Err("shape one bounded Thai source row at a time".into()); + } + let face = skrifa::FontRef::new(&self.bytes).map_err(|e| e.to_string())?; + require_pua(&face.charmap(), text)?; + let mut shaped = crate::source_shaping::ShapedRun { glyphs: Vec::new(), advance: 0.0 }; + let mut codes = Vec::new(); + // Legacy PDF subsets often omit the space glyph. Keep its measured PDF advance as a + // positioning gap, rather than drawing .notdef or changing to another font. + for piece in text.split_inclusive(' ') { + let word = piece.strip_suffix(' ').unwrap_or(piece); + if !word.is_empty() { + let run = crate::source_shaping::shape_legacy_thai(&self.bytes, word)?; + for mut glyph in run.glyphs { + let code = self + .codes + .get(&glyph.id.to_u32()) + .ok_or("the Thai subset lacks a required contextual glyph; load the complete original face")?; + glyph.position[0] += shaped.advance; + codes.push(code.to_be_bytes().to_vec()); + shaped.glyphs.push(glyph); + } + shaped.advance += run.advance; + } + if piece.ends_with(' ') { + shaped.glyphs.push(crate::source_shaping::ShapedGlyph { + id: skrifa::GlyphId::NOTDEF, + position: [shaped.advance, 0.0], + text: " ".into(), + }); + codes.push(Vec::new()); + shaped.advance += self.space; + } + if shaped.glyphs.len() > 65_534 || !shaped.advance.is_finite() { + return Err("Thai source row exceeds the shaping budget".into()); + } + } + Ok(ExistingThaiRun { shaped, codes }) + } +} + +/// Verify every context-dependent PUA alternate before asking the legacy shaper to use it. +/// Otherwise a subset missing an alternate would silently draw the wrong high/left mark. +fn require_pua(cmap: &skrifa::charmap::Charmap<'_>, text: &str) -> Result<(), String> { + let mut base = 0u32; + let mut above = false; + let mut chars = text.chars().peekable(); + while let Some(ch) = chars.next() { + let c = u32::from(ch); + let ascender = matches!(base, 0xe1b | 0xe1d | 0xe1f); + let pua = match c { + 0xe31 if ascender => Some(0xf710), + 0xe34..=0xe37 if ascender => Some(0xf701 + c - 0xe34), + 0xe47 if ascender => Some(0xf712), + 0xe4d if ascender => Some(0xf711), + 0xe48..=0xe4c => { + let high = above || chars.peek().is_some_and(|c| *c == 'ำ'); + if ascender { + Some((if high { 0xf713 } else { 0xf705 }) + c - 0xe48) + } else if !high { + Some(0xf70a + c - 0xe48) + } else { + None + } + } + 0xe38..=0xe3a if matches!(base, 0xe0e | 0xe0f) => Some(0xf718 + c - 0xe38), + 0xe38..=0xe3a if base == 0xe0d => Some(0xf70f), + 0xe38..=0xe3a if base == 0xe10 => Some(0xf700), + 0xe33 if ascender => Some(0xf711), + _ => None, + }; + if pua.is_some_and(|p| char::from_u32(p).is_none_or(|c| cmap.map(c).is_none())) { + return Err(format!( + "the Thai subset lacks a required contextual mark for {ch} (U+{c:04X}); load the complete original face or explicitly select a replacement font" + )); + } + if matches!(c, 0xe01..=0xe2e) { + base = c; + above = false; + } else if matches!(c, 0xe31 | 0xe34..=0xe37 | 0xe47 | 0xe4d..=0xe4e) { + above = true; + } else if !matches!(c, 0xe38..=0xe3a | 0xe48..=0xe4c) { + base = 0; + above = false; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::recovered_thai_map; + use pdfcraft_cos::{Dict, Document, Object}; + use std::collections::HashMap; + + #[test] + fn damaged_legacy_thai_map_recovers_consonants_vowels_and_contextual_tones_only_for_its_fingerprint() { + let mut doc = Document::new_empty(); + let data = b"begincmap 1 begincodespacerange <0000> endcodespacerange 18 beginbfrange \ + <00d9><00d9><0e04> <00e2><00e2><0e0d> <00fa><00fa><0e25> <0100><0100><0e2b> <0102><0102><0e2d> \ + <0164><0164><0e4c> <0166><0166><0e31> <0168><0168><0e47> \ + <00d6><00d6><000f> <00f0><00f0><0029> <00f8><00f8><0023> <0105><0105><003e> <0107><0107><0040> \ + <0109><0109><003e> <0111><0111><004e> <0160><0160><004e> <0161><0161><004f> <00ff><01ff><0032> endbfrange endcmap"; + let tu = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), data))); + let mut desc = Dict::new(); + desc.set(b"CIDToGIDMap".to_vec(), Object::name("Identity")); + let mut font = Dict::new(); + font.set(b"Subtype".to_vec(), Object::name("Type0")); + font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+THSarabunPSK")); + font.set(b"Encoding".to_vec(), Object::name("Identity-H")); + font.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(desc)])); + font.set(b"ToUnicode".to_vec(), Object::Ref(tu)); + // The corrupt range is declared before explicit anchors in the actual exporter. + let data = String::from_utf8_lossy(data) + .replace("18 beginbfrange", "1 beginbfrange <00ff><01ff><0032> endbfrange 17 beginbfrange") + .replace(" <00ff><01ff><0032> endbfrange endcmap", " endbfrange endcmap"); + doc.set(tu, Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), data.as_bytes()))); + let recovered: HashMap<_, _> = recovered_thai_map(&doc, &font).unwrap().into_iter().collect(); + let title: String = [240, 248, 261, 214, 263, 253].into_iter().map(|code| recovered[&code]).collect(); + assert_eq!(title, "ประกาศ"); + assert_eq!(recovered[&265], 'ิ'); + assert_eq!(recovered[&273], 'เ'); + assert_eq!(recovered[&352], '่'); + assert_eq!(recovered[&353], '้'); + font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+Unrelated")); + assert!(recovered_thai_map(&doc, &font).is_none()); + font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+THSarabunPSK")); + doc.set(tu, Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), b"1 beginbfchar <00d9><0e04> endbfchar"))); + assert!(recovered_thai_map(&doc, &font).is_none(), "one plausible anchor cannot authorize guessing a map"); + } +} diff --git a/crates/fonts/src/source_shaping.rs b/crates/fonts/src/source_shaping.rs index b6b28ca692..6976bd8a0e 100644 --- a/crates/fonts/src/source_shaping.rs +++ b/crates/fonts/src/source_shaping.rs @@ -22,7 +22,23 @@ pub fn required(text: &str) -> bool { .any(|c| matches!(u32::from(c), 0x0300..=0x036f | 0x0590..=0x0fff | 0x1780..=0x17ff | 0x200c..=0x200f | 0x202a..=0x202e | 0x2066..=0x2069)) } +/// Whether a Thai source program retains its mark-positioning table. +/// Character coverage alone does not show that stacked vowels and tones can be laid out. +/// OpenType Thai requires `mark`/`mkmk` (GPOS); `ccmp` substitution (GSUB) is optional. +pub fn has_thai_layout(bytes: &[u8]) -> bool { + FontRef::new(bytes).is_ok_and(|f| f.gpos().is_ok()) +} + pub fn shape(bytes: &[u8], text: &str) -> Result { + shape_inner(bytes, text, false) +} + +/// Only used after the PDF adapter has verified legacy Thai PUA glyph mappings. +pub(crate) fn shape_legacy_thai(bytes: &[u8], text: &str) -> Result { + shape_inner(bytes, text, true) +} + +fn shape_inner(bytes: &[u8], text: &str, legacy_thai: bool) -> Result { if text.len() > 262_144 || text.contains(['\n', '\r']) { return Err("shape one bounded text row at a time".into()); } @@ -30,7 +46,12 @@ pub fn shape(bytes: &[u8], text: &str) -> Result { return Ok(ShapedRun { glyphs: Vec::new(), advance: 0.0 }); } let face = FontRef::new(bytes).map_err(|e| e.to_string())?; - if text.chars().any(|c| matches!(u32::from(c), 0x0600..=0x06ff | 0x0750..=0x077f | 0x08a0..=0x0fff | 0x1780..=0x17ff)) && face.gsub().is_err() { + if !legacy_thai && text.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) && face.gpos().is_err() { + return Err("the source subset has no Thai mark-positioning table (GPOS); select its full font".into()); + } + if text.chars().any(|c| matches!(u32::from(c), 0x0600..=0x06ff | 0x0750..=0x077f | 0x08a0..=0x0dff | 0x0e80..=0x0fff | 0x1780..=0x17ff)) + && face.gsub().is_err() + { return Err("the source subset has no contextual substitution tables; select its full font".into()); } let data = ShaperData::new(&face); @@ -56,7 +77,11 @@ pub fn shape(bytes: &[u8], text: &str) -> Result { let mut seen = std::collections::HashSet::new(); for (g, p) in output.glyph_infos().iter().zip(output.glyph_positions()) { if g.glyph_id == 0 { - return Err("the full font lacks a required contextual glyph; choose the complete source face".into()); + let missing = input.get(g.cluster as usize..).and_then(|s| s.chars().next()).unwrap_or('\u{fffd}'); + return Err(format!( + "the full font lacks a required contextual glyph for {missing} (U+{:04X}); choose the complete source face", + u32::from(missing) + )); } let start = g.cluster as usize; let end = starts.get(starts.partition_point(|i| *i <= start)).copied().unwrap_or(input.len()); @@ -74,3 +99,27 @@ pub fn shape(bytes: &[u8], text: &str) -> Result { } Ok(ShapedRun { glyphs, advance: x }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn thai_positioning_does_not_require_optional_gsub() { + let Some(face) = crate::document_thai_font() else { return }; + let mut bytes = face.bytes.to_vec(); + // Keep the actual GPOS anchors and outlines while removing optional substitutions. + // Table payloads stay at their original offsets; no font fixture is committed. + let count = u16::from_be_bytes([bytes[4], bytes[5]]) as usize; + let tables: Vec<_> = bytes[12..12 + count * 16].as_chunks::<16>().0.iter().filter(|t| &t[..4] != b"GSUB").map(|t| t.to_vec()).collect(); + bytes[4..6].copy_from_slice(&(tables.len() as u16).to_be_bytes()); + for (i, table) in tables.iter().enumerate() { + bytes[12 + i * 16..28 + i * 16].copy_from_slice(table); + } + assert!(FontRef::new(&bytes).unwrap().gsub().is_err()); + assert!(has_thai_layout(&bytes)); + let shaped = shape(&bytes, "กิ่ น้ำ").unwrap(); + assert!(shaped.advance > 0.0); + assert!(shaped.glyphs.iter().all(|g| g.position.iter().all(|p| p.is_finite()))); + } +} From c7e8d127afb9ac22e6248eeebbc838bf6b2aa2d7 Mon Sep 17 00:00:00 2001 From: xullul <32987931+xullul@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:29:47 +0700 Subject: [PATCH 06/37] M7: edit grouped Thai text while preserving source formatting Add: - Prepare supported Form placements as original page operators through the existing-text UI and text_prepare, with shared-form isolation and undo/redo. - Repair narrowly identified Thai exporter maps on copied page fonts, recording repairs without changing the embedded source glyph programs. Change: - Reuse legacy Thai Regular/Bold glyph codes and same-face punctuation subsets; retain point size, spacing, horizontal scale, rise, colour and paint settings. - Keep cross-family Thai replacement explicit and write matching bundled Thai subsets through the existing bounded outline path rather than static embedding. - Keep replacement-only menu choices out of the source-font preview atlas so opening an ordinary paragraph does not wait an extra frame for unused fonts. - Refuse unsafe compositing, tagged or malformed groups, and missing contextual glyphs without losing the user's document or changing other placements. Chores: - Document the editing limits and headless workflow; keep text-edit parity partial. Builds on dekkeng's Thai fallback work in #2811 (fba0a29a) and mtlcl410's Form Ungroup implementation in #3067 (6eafca9c), adapted to the current module layout. The Ungroup menu and older UI font-priority changes are not included. Refs #3185, #2964 and #2142. Synthetic regressions cover source-font preservation, shared forms, unsafe refusals, layout, undo/redo, rendering and save/reopen. Validation: library suites 1,675 passed; automation integration 331 passed; document-font integration 2 passed; editor flows 248 passed. Existing ignored tests remain. Formatting, scoped strict Clippy, layers, assets, parity and wasm checks passed. The unchanged opacity regression failed before the choice-only atlas correction and passed afterwards. --- ROADMAP.md | 3 + crates/automation/README.md | 5 + crates/automation/src/dispatch.rs | 9 + crates/automation/src/tools.rs | 7 +- crates/automation/tests/it/automation.rs | 115 ++++ crates/edit/src/added.rs | 1 + crates/edit/src/added/draw.rs | 52 ++ crates/edit/src/images.rs | 2 +- crates/edit/src/tests.rs | 228 +++++++ crates/edit/src/text.rs | 3 +- crates/edit/src/text/blocks.rs | 2 +- crates/edit/src/text/editor.rs | 41 +- crates/edit/src/text/fallback.rs | 8 +- crates/edit/src/text/forms.rs | 192 ++++++ crates/edit/src/text/forms/expand.rs | 576 ++++++++++++++++++ crates/edit/src/text_regions.rs | 7 +- crates/edit/src/text_regions/styled.rs | 528 +++++++++++++--- crates/engine/src/document.rs | 14 + crates/engine/src/edit.rs | 4 + crates/engine/src/edit/labels.rs | 2 + crates/engine/src/edit/run.rs | 9 + crates/engine/src/history.rs | 1 + crates/ui-egui/src/canvas.rs | 3 + crates/ui-egui/src/canvas/page_input.rs | 7 + .../ui-egui/src/tools/edit_text_ui/layout.rs | 4 +- docs/editing-existing-text.md | 35 ++ docs/roadmap.md | 5 +- parity/acrobat-features.toml | 6 +- 28 files changed, 1768 insertions(+), 101 deletions(-) create mode 100644 crates/edit/src/text/forms.rs create mode 100644 crates/edit/src/text/forms/expand.rs diff --git a/ROADMAP.md b/ROADMAP.md index 820ceb0179..988be90914 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,7 @@ # PdfCraft roadmap +> **Last reviewed:** 2026-11-05 | **Last updated:** 2026-11-05 | **Change:** patch (M7 #2964: directly edit Thai source text inside supported Form XObjects; estimates unchanged) + > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged) > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850: Enter adds Fill & Sign line breaks; outside clicks commit, Escape cancels, and saved multiline appearances retain their text and height; estimates unchanged) @@ -230,6 +232,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-11-05 (M7, #2964 / #3185):** Supported Form text can be prepared for direct editing. Legacy Thai keeps the original font programs, Regular/Bold and text settings; cross-family replacement is explicit. Builds on #3067 and #2811. Unsafe groups and mixed contextual styles remain partial; estimates unchanged. - **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged. - **2026-11-03 (M11, shared JBIG2 dictionary):** With Monochrome Images set to JBIG2, Optimize has up to 16 JBIG2 images share one symbol dictionary (`/JBIG2Globals`), the letters found on more than one page stored once, when that is smaller (`pdfcraft_filters::encode_jbig2_shared`). Eight pages of 10 pt text rendered at 300 dpi take 41% less in Helvetica and 42% in Times than coded alone, 10% in Courier; groups of 16 save 45%, of 32 only 47%. Only images redaction can store again without the dictionary share one (image masks and DeviceGray named directly, at most 64 Mi pixels), and redacting one stores the rest of its group so, which the group bounds to 15. diff --git a/crates/automation/README.md b/crates/automation/README.md index db5b988394..9c31754e0d 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -204,6 +204,11 @@ undoable and survive saving/reopening. `image_hit` shares the UI's curve and hol The bounds, limits and font requirements are in [existing-text editing](../../docs/editing-existing-text.md) and [image editing](../../docs/editing-images.md). +`text_prepare` makes supported grouped Form text editable on one page, as entering Edit text +in the GUI does. It keeps the group's original glyph operators and shared source objects, +records preparation/map repairs and is undoable. List `text_paragraphs` afterwards; its indices +belong to the prepared page. Unsafe compositing and malformed/tagged groups stay intact. + ### Superscript and subscript normalization `text_script` changes characters to their Unicode superscript or subscript forms; it does not diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index ebe2a93526..0d8c6e2333 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -480,6 +480,15 @@ impl Automation { self.apply(&a, Edit::EditPageImage { page, index, change })? } } + "text_prepare" => { + let page = self.page(&a)?; + let preparation = self.doc(&a)?.text_preparation(page).map_err(ToolError::Failed)?; + let mut result = self.apply(&a, Edit::PrepareText { page })?; + result["prepared_groups"] = json!(preparation.expanded); + result["repaired_fonts"] = json!(preparation.repaired_fonts); + result["refused_groups"] = json!(preparation.refused); + result + } "text_paragraphs" => { let page = self.page(&a)?; let doc = self.doc_to_copy(&a)?; diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index af5146377a..c58d803284 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1581,9 +1581,12 @@ pub fn tools() -> Vec { t("text_font_import", "Load document font", "Load an explicitly selected, openly licensed full TrueType/OpenType face into a page. The program's family, embedding permissions and licence declaration are validated. Completes source subsets of the same face without replacing existing glyphs. Returns a font resource usable with text_edit.font_resource. Undoable; respects root confinement. No system-interface fallback is exported.") .cmd("edit.edit_text") .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}, "path":{"type":"string"}}), &["doc","page","path"])), - t("text_fonts", "List document fonts", "List validated embedded source faces and approved full preview faces for a page. Full registered resources can be selected by text_edit.font_resource; preview_only faces are not PDF resource names.") + t("text_fonts", "List document fonts", "List validated embedded source faces and approved full preview faces for a page. Full registered resources and bundled choices can be selected by text_edit.font_resource. When available, builtin:thai explicitly selects bundled Noto Sans Thai for a paragraph or range; preview_only faces cannot be selected.") .ro().cmd("edit.edit_text") .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}}), &["doc","page"])), + t("text_prepare", "Prepare grouped text", "Make existing text inside supported Form XObjects directly editable on one page. Replaces placements with their original operators and resources, preserving position, clipping and other pages. Refuses unsafe transparency, tags or malformed content. Adds no text or cover boxes. Returns prepared_groups, repaired_fonts and refused_groups. Undoable; list text_paragraphs again afterwards.") + .cmd("edit.edit_text") + .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}}), &["doc","page"])), t("text_paragraphs", "List paragraphs", "The paragraphs on a page (adjacent source runs grouped into aligned visual rows, including mixed fonts, sizes and colours): number, text, editor_text (visual rows separated by newlines), rows and source line numbers, box (top-left-origin points), font and size. Use the number with text_edit's paragraph: until you list the page again, a number names the paragraph listed here, even after edits regroup the page.") .ro() .cmd("edit.edit_text") @@ -1597,7 +1600,7 @@ pub fn tools() -> Vec { .with(schema( json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "line": { "type": "integer", "minimum": 1 }, "paragraph": { "type": "integer", "minimum": 1 }, - "font_resource": { "type": "string", "description": "Explicit source/full page font resource from text_fonts or text_font_import." }, + "font_resource": { "type": "string", "description": "Paragraphs: source/full page resource or bundled document-font choice from text_fonts (builtin:thai selects the bundled Thai face); also accepts text_font_import resources." }, "ranges": { "type": "array", "maxItems": 1024, "items": { "type": "object", "properties": {"range":{"type":"array","items":{"type":"integer","minimum":0},"minItems":2,"maxItems":2},"font":{"type":"string","enum":["helvetica","times","courier"]},"bold":{"type":"boolean"},"italic":{"type":"boolean"},"font_resource":{"type":"string"},"size":{"type":"number","exclusiveMinimum":0},"color":{"type":"string"},"underline":{"type":"boolean"},"char_spacing":{"type":"number"},"scale":{"type":"number"}},"required":["range"]}, "description":"Disjoint UTF-8 byte ranges in replacement text; unspecified source formatting survives." }, "text": { "type": "string" }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 96649c8e80..21c958b52e 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -11428,3 +11428,118 @@ fn comments_copy_and_paste_between_documents_through_tools() { assert!(matches!(a.call("comments_copy", &json!({ "doc": src, "comments": [[2]] })), Err(ToolError::InvalidArgs(_)))); assert_eq!(ok(&mut a, "comment_list", json!({ "doc": src }))["comments"].as_array().unwrap().len(), 2, "the source is untouched"); } + +#[test] +fn explicit_bundled_thai_font_choice_layout_edit_undo_and_reopen() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + std::fs::write(dir.join("source.pdf"), fixture(1)).unwrap(); + let mut a = auto(dir); + let doc = ok(&mut a, "doc_open", json!({"path":"source.pdf"}))["doc"].as_u64().unwrap(); + let before = page_text(&mut a, doc); + let fonts = ok(&mut a, "text_fonts", json!({"doc":doc,"page":1})); + let choice = fonts["fonts"].as_array().unwrap().iter().find(|f| f["resource"] == "builtin:thai"); + let args = json!({"doc":doc,"page":1,"paragraph":1,"text":"น้ำ กิ่ กี้ ผู้ใช้","font_resource":"builtin:thai"}); + if pdfcraft_fonts::document_thai_font().is_none() { + assert!(choice.is_none()); + assert!(a.call("text_edit", &args).is_err()); + assert_eq!(page_text(&mut a, doc), before); + return; + } + assert_eq!(choice.unwrap()["preview_only"], false); + ok(&mut a, "text_edit_layout", args.clone()); + assert_eq!(page_text(&mut a, doc), before); + ok(&mut a, "text_edit", args); + let edited = page_text(&mut a, doc); + assert!(edited[0].contains("น้ำ กิ่ กี้ ผู้ใช้")); + ok(&mut a, "edit_undo", json!({"doc":doc})); + assert_eq!(page_text(&mut a, doc), before); + ok(&mut a, "edit_redo", json!({"doc":doc})); + ok(&mut a, "doc_save", json!({"doc":doc,"path":"edited.pdf"})); + let reopened = ok(&mut a, "doc_open", json!({"path":"edited.pdf"}))["doc"].as_u64().unwrap(); + assert_eq!(page_text(&mut a, reopened), edited); + assert!(a.call("text_edit", &json!({"doc":reopened,"page":1,"paragraph":1,"text":"น้ำ 😀","font_resource":"builtin:thai"})).is_err()); + assert_eq!(page_text(&mut a, reopened), edited); + let image = a.call("page_render", &json!({"doc":reopened,"page":1,"dpi":144})).unwrap(); + assert!(image.iter().any(|c| matches!(c, Content::Png { .. }))); +} + +#[test] +fn grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_glyphs() { + use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; + use std::sync::Arc; + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + let mut source = Document::open(Arc::new(fixture(2))).unwrap(); + let old = "ข้อความเดิม น้ำ ผู้ใช้"; + let new = "ข้อความแก้ไข น้ำ ผู้ใช้"; + let mut cmap = String::from("begincmap 1 begincodespacerange <00> endcodespacerange\n"); + cmap.push_str(&format!("{} beginbfchar\n", old.chars().count())); + for (i, ch) in old.chars().enumerate() { + cmap.push_str(&format!("<{:02x}> <{:04x}>\n", 65 + i, u32::from(ch))); + } + cmap.push_str("endbfchar endcmap"); + let unicode = source.add(Object::Stream(Stream::flate(Dict::new(), cmap.as_bytes()))); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type1")); + font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+LegacyThai")); + font.set(b"FirstChar".to_vec(), Object::Int(65)); + font.set(b"LastChar".to_vec(), Object::Int(65 + old.chars().count() as i64 - 1)); + font.set(b"Widths".to_vec(), Object::Array(vec![Object::Int(300); old.chars().count()])); + font.set(b"ToUnicode".to_vec(), Object::Ref(unicode)); + let font = source.add(Object::Dict(font)); + let mut fonts = Dict::new(); + fonts.set(b"F1".to_vec(), Object::Ref(font)); + let mut resources = Dict::new(); + resources.set(b"Font".to_vec(), Object::Dict(fonts)); + let mut form = Dict::new(); + form.set(b"Type".to_vec(), Object::name("XObject")); + form.set(b"Subtype".to_vec(), Object::name("Form")); + form.set(b"BBox".to_vec(), Object::Array(vec![Object::Int(0), Object::Int(0), Object::Int(200), Object::Int(300)])); + form.set(b"Resources".to_vec(), Object::Dict(resources)); + let codes: String = (0..old.chars().count()).map(|i| format!("{:02x}", 65 + i)).collect(); + let form = source.add(Object::Stream(Stream::flate(form, format!("BT /F1 14 Tf 20 220 Td <{codes}> Tj ET").as_bytes()))); + for page in [ObjRef::new(4, 0), ObjRef::new(6, 0)] { + let mut xobjects = Dict::new(); + xobjects.set(b"Body".to_vec(), Object::Ref(form)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let content = source.add(Object::Stream(Stream::flate(Dict::new(), b"q /Body Do Q"))); + source + .update_dict(page, |d| { + d.set(b"Resources".to_vec(), Object::Dict(resources)); + d.set(b"Contents".to_vec(), Object::Ref(content)); + }) + .unwrap(); + } + std::fs::write(dir.join("source.pdf"), write_full(&source, &SaveOptions::default()).unwrap()).unwrap(); + let mut a = auto(dir); + let id = ok(&mut a, "doc_open", json!({"path":"source.pdf"}))["doc"].as_u64().unwrap(); + let before = a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(); + assert_eq!(ok(&mut a, "text_paragraphs", json!({"doc":id,"page":1}))["count"], 0); + ok(&mut a, "text_prepare", json!({"doc":id,"page":1})); + assert_eq!(a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(), before, "preparing must not change pixels"); + assert_eq!(ok(&mut a, "text_paragraphs", json!({"doc":id,"page":1}))["paragraphs"][0]["text"], old); + let args = json!({"doc":id,"page":1,"paragraph":1,"text":new,"font_resource":"builtin:thai"}); + if pdfcraft_fonts::document_thai_font().is_none() { + assert!(a.call("text_edit", &args).is_err()); + return; + } + ok(&mut a, "text_edit", args); + assert_eq!(ok(&mut a, "text_lines", json!({"doc":id,"page":1}))["lines"][0]["text"].as_str().unwrap().trim(), new); + assert_eq!(page_text(&mut a, id)[1], old, "a shared form's other page must not change"); + let after = a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(); + assert_ne!(after, before); + ok(&mut a, "edit_undo", json!({"doc":id})); + assert_eq!(a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(), before); + ok(&mut a, "edit_undo", json!({"doc":id})); + assert_eq!(ok(&mut a, "text_paragraphs", json!({"doc":id,"page":1}))["count"], 0); + ok(&mut a, "edit_redo", json!({"doc":id})); + ok(&mut a, "edit_redo", json!({"doc":id})); + ok(&mut a, "doc_save", json!({"doc":id,"path":"edited.pdf"})); + let reopened = ok(&mut a, "doc_open", json!({"path":"edited.pdf"}))["doc"].as_u64().unwrap(); + assert_eq!(a.call("page_render", &json!({"doc":reopened,"page":1,"dpi":72})).unwrap(), after); + assert_eq!(page_text(&mut a, reopened), [new, old]); + assert_eq!(ok(&mut a, "content_list", json!({"doc":reopened}))["count"], 0, "editing existing text must not create Add Text objects"); +} diff --git a/crates/edit/src/added.rs b/crates/edit/src/added.rs index a603a37060..4abe4a02eb 100644 --- a/crates/edit/src/added.rs +++ b/crates/edit/src/added.rs @@ -23,6 +23,7 @@ mod layout; mod store; mod types; +pub(crate) use draw::thai_glyph_resources; #[cfg(test)] use draw::{GLYPHS_PER_FONT, LaidLine, draw_laid_shaped}; #[cfg(test)] diff --git a/crates/edit/src/added/draw.rs b/crates/edit/src/added/draw.rs index 9404f939b1..03b00fe392 100644 --- a/crates/edit/src/added/draw.rs +++ b/crates/edit/src/added/draw.rs @@ -386,3 +386,55 @@ pub(super) fn params(c: &Content) -> Dict { } d } + +/// Existing Thai text uses absolute shaped glyph positions. Reuse Add Text's bounded outline +/// writer instead of trying to embed the matching craft-fonts variable program as a static face. +pub(crate) fn thai_glyph_resources( + doc: &mut Document, + fonts: &mut Dict, + shaped: &pdfcraft_fonts::source_shaping::ShapedRun, +) -> Result)>, EditError> { + if shaped.glyphs.len() > 65_534 { + return Err(EditError::Invalid("shaped row exceeds its geometry budget".into())); + } + let mut outlines = HashMap::new(); + let mut clusters = Vec::new(); + let mut index = HashMap::new(); + let mut indices = Vec::new(); + for glyph in &shaped.glyphs { + let id = glyph.id.to_u32(); + let key = (ShapedScript::Thai, id); + if let std::collections::hash_map::Entry::Vacant(entry) = outlines.entry(key) { + entry.insert(shaped_glyph(ShapedScript::Thai, id).map_err(|e| thai_error(e, &glyph.text))?); + } + let at = *index.entry((id, glyph.text.as_str())).or_insert_with(|| { + let at = clusters.len(); + let width = outlines.get(&key).map_or(0.0, |g| g.width); + clusters.push(ShapedCluster { script: ShapedScript::Thai, glyphs: vec![(id, [0.0, 0.0])], advance: width, text: glyph.text.clone() }); + at + }); + indices.push(at); + } + // Validate all outlines before adding resources. Type 3 character codes are single bytes; + // shard long rows and return the appropriate resource/code for every positioned glyph. + let mut names = Vec::new(); + for chunk in clusters.chunks(GLYPHS_PER_FONT) { + let refs: Vec<_> = chunk.iter().collect(); + let font = type3_shaped(doc, &refs, &outlines); + doc.update_dict(font, |f| f.set(b"Name".to_vec(), Object::name("PCThai")))?; + let mut name = format!("PCThai{}", font.num); + while fonts.contains(name.as_bytes()) { + name.push('_'); + } + fonts.set(name.as_bytes().to_vec(), Object::Ref(font)); + names.push(name); + } + indices + .into_iter() + .map(|at| { + let name = names.get(at / GLYPHS_PER_FONT).ok_or_else(|| EditError::Invalid("Thai font resource is missing".into()))?; + let code = u8::try_from(at % GLYPHS_PER_FONT + 1).map_err(|_| EditError::Invalid("Thai glyph limit".into()))?; + Ok((name.clone(), vec![code])) + }) + .collect() +} diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 72656fa1bb..b4ecd1f74f 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -49,7 +49,7 @@ pub struct PageImage { /// The content stream its `Do` (its first token) is in, and the `Do`'s index among the /// operators of the page's streams joined ([`Pieces`]). pub stream: usize, - op: usize, + pub(crate) op: usize, /// CTM before `Do`, excluding a Form's own Matrix. draw_matrix: [f64; 6], clips: Vec, diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 22fbd6e8ae..d8f0ca16ee 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -10696,3 +10696,231 @@ fn review_inside_scope_group_move_matches_direct_transform_under_outer_scale() { } } } + +#[test] +fn explicit_builtin_thai_replaces_an_unavailable_source_without_installing_a_font() { + let mut doc = text_page("BT /F1 16 Tf 50 700 Td (A) Tj ET BT /F2 12 Tf 50 600 Td (ab) Tj ET"); + let cmap = + doc.add(Object::Stream(Stream::flate(Dict::new(), b"1 begincodespacerange <00> endcodespacerange 1 beginbfchar <41> <0E19> endbfchar"))); + doc.update_dict(pdfcraft_cos::ObjRef::new(5, 0), |f| { + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+UnavailableThai")); + f.set(b"ToUnicode".to_vec(), Object::Ref(cmap)); + f.set(b"FirstChar".to_vec(), Object::Int(65)); + f.set(b"LastChar".to_vec(), Object::Int(65)); + f.set(b"Widths".to_vec(), Object::Array(vec![Object::Int(600)])); + }) + .unwrap(); + let before = text::text_lines(&doc, 0).unwrap(); + let choice = text::text_editor_fonts(&doc, 0).unwrap().into_iter().find(|f| f.resource == "builtin:thai"); + let style = text::BlockStyle { resource: Some("builtin:thai".into()), ..Default::default() }; + if pdfcraft_fonts::document_thai_font().is_none() { + assert!(choice.is_none()); + assert!(text::rewrite_block(&mut doc, 0, 0, Some("น้ำ"), &style).is_err()); + assert_eq!(text::text_lines(&doc, 0).unwrap()[0].text, before[0].text); + return; + } + assert!(choice.is_some(), "the document-font menu must offer a bundled Thai replacement"); + let error = text::rewrite_block(&mut doc, 0, 0, Some("นน"), &text::BlockStyle::default()).unwrap_err().to_string(); + assert!(error.contains("UnavailableThai") && error.contains("Noto Sans Thai"), "{error}"); + let draft = "น้ำ กิ่ กี้ ผู้ใช้"; + text::rewrite_block(&mut doc, 0, 0, Some(draft), &style).unwrap(); + let mut doc = reopen(&doc); + assert_eq!(text::text_blocks(&doc, 0).unwrap()[0].editor_text, draft); + assert_eq!(text::text_lines(&doc, 0).unwrap().last().unwrap().text, before[1].text); + assert_eq!(text::text_lines(&doc, 0).unwrap().last().unwrap().baseline(), before[1].baseline()); + text::rewrite_block(&mut doc, 0, 0, Some("น้ำ ปู่"), &text::BlockStyle::default()).unwrap(); + assert_eq!(text::text_blocks(&reopen(&doc), 0).unwrap()[0].editor_text, "น้ำ ปู่"); + // A selected range uses the same explicit choice; neighbouring source runs keep their face. + let mut selected = text_page("BT /F1 16 Tf 50 700 Td (Ab) Tj ET"); + let style = text::BlockStyle { + ranges: vec![text::RangeStyle { + range: 0..1, + resource: Some("builtin:thai".into()), + family: None, + size: None, + color: None, + underline: None, + char_spacing: None, + scale: None, + }], + ..Default::default() + }; + text::rewrite_block(&mut selected, 0, 0, Some("Ab"), &style).unwrap(); + let lines = text::text_lines(&reopen(&selected), 0).unwrap(); + assert!(lines.iter().any(|l| l.text == "A" && l.base_font.contains("NotoSansThai"))); + assert!(lines.iter().any(|l| l.text == "b" && l.base_font == "Helvetica")); +} + +#[test] +fn existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font() { + let Some(_) = pdfcraft_fonts::document_thai_font() else { return }; + // Synthetic source subset: only one Thai letter is encoded. No binary fixture or font + // is committed; the complete, openly licensed matching face is the optional build input. + let mut doc = text_page("BT /F1 16 Tf 50 700 Td (A) Tj ET BT /F2 12 Tf 50 600 Td (ab) Tj ET"); + let cmap = + doc.add(Object::Stream(Stream::flate(Dict::new(), b"1 begincodespacerange <00> endcodespacerange 1 beginbfchar <41> <0E19> endbfchar"))); + doc.update_dict(pdfcraft_cos::ObjRef::new(5, 0), |f| { + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+NotoSansThai-Regular")); + f.set(b"ToUnicode".to_vec(), Object::Ref(cmap)); + }) + .unwrap(); + let neighbour = text::text_lines(&doc, 0).unwrap()[1].clone(); + // More than 240 distinct logical clusters requires several single-byte Type 3 resources. + let long: String = + "กขคงจฉชซญดตถทนบปผฝพฟมยรลวศษสหฬอฮ".chars().flat_map(|c| ["ิ่", "ิ้", "ี่", "ี้", "ึ่", "ึ้", "ื่", "ื้"].map(move |marks| format!("{c}{marks}"))).collect(); + let draft = format!("{long} ผู้ใช้ ปู่ ๑๒๓"); + let style = text::BlockStyle { width: Some(14_400.0), ..Default::default() }; + text::rewrite_block(&mut doc, 0, 0, Some(&draft), &style).unwrap(); + let mut doc = reopen(&doc); + assert_eq!(text::text_blocks(&doc, 0).unwrap()[0].editor_text, draft); + assert_eq!(text::text_lines(&doc, 0).unwrap()[0].baseline(), [50.0, 700.0]); + assert_eq!(text::text_lines(&doc, 0).unwrap().last().unwrap().text, neighbour.text); + assert_eq!(text::text_lines(&doc, 0).unwrap().last().unwrap().baseline(), neighbour.baseline()); + let next = "Hello ผู้ใช้ น้ำ ๔๕๖"; + text::rewrite_block(&mut doc, 0, 0, Some(next), &text::BlockStyle::default()).unwrap(); + assert_eq!(text::text_blocks(&reopen(&doc), 0).unwrap()[0].editor_text, next); +} + +#[test] +fn grouped_source_text_preparation_edits_original_operators_without_changing_shared_forms() { + let mut doc = page_with_forms("q 1 0 0 1 12 20 cm /Body Do Q", &[("Body".into(), "BT /F1 14 Tf 40 500 Td (Original body) Tj ET".into())]); + let original_form = doc.get(pdfcraft_cos::ObjRef::new(7, 0)).clone(); + let first = pdfcraft_model::pages(&doc)[0].clone(); + let second = doc.add(Object::Dict(first.dict.clone())); + doc.update_dict(pdfcraft_cos::ObjRef::new(2, 0), |d| { + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(first.obj), Object::Ref(second)])); + d.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + assert!(!text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text.contains("Original body"))); + let prepared = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(prepared.expanded, 1); + let blocks = text::text_blocks(&doc, 0).unwrap(); + let (index, block) = blocks.iter().enumerate().find(|(_, b)| b.text == "Original body").unwrap(); + assert!((block.rect[0] - 52.0).abs() < 0.01, "placement must survive preparation: {block:?}"); + rewrite_block(&mut doc, 0, index, Some("Changed body"), &BlockStyle::default()).unwrap(); + assert!( + !text::glyph_text_lines(&doc, &pdfcraft_model::pages(&doc)[0]).unwrap().iter().any(|l| l.text.contains("Original body")), + "old glyphs must be removed, not covered" + ); + assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "Changed body")); + assert!(text::reading_blocks(&doc, 1).unwrap().iter().any(|b| b.text == "Original body")); + assert_eq!(doc.get(pdfcraft_cos::ObjRef::new(7, 0)), original_form); + assert_eq!(text::forms::prepare(&mut doc, 0).unwrap().expanded, 0); +} + +#[test] +fn grouped_text_preparation_keeps_unsafe_compositing_paths_and_malformed_forms_intact() { + for (page_ops, form_ops, extra) in [ + ("/Body Do", "BT /F1 14 Tf (body) Tj ET", "/Group << /S /Transparency /K true >>"), + ("/Body Do", "Q BT /F1 14 Tf (body) Tj ET", ""), + ("/Body Do", "BT /F1 14 Tf (body) Tj", ""), + ("10 10 m 20 20 l /Body Do S", "BT /F1 14 Tf (body) Tj ET", ""), + ("BT /F1 14 Tf /Body Do ET", "BT /F1 14 Tf (body) Tj ET", ""), + ("/Body Do", "/Body Do", "/Group << /S /Transparency >>"), + ] { + let mut doc = page_with_forms(page_ops, &[("Body".into(), form_ops.into())]); + if !extra.is_empty() { + let additions = build(&[&format!("<< {extra} >>")]); + let fields = additions.get(pdfcraft_cos::ObjRef::new(1, 0)).as_dict().unwrap().clone(); + let form = doc.get(pdfcraft_cos::ObjRef::new(7, 0)); + let Object::Stream(mut s) = form.as_ref().clone() else { panic!("fixture") }; + for (k, v) in fields.iter() { + s.dict.set(k.clone(), v.clone()); + } + doc.set(pdfcraft_cos::ObjRef::new(7, 0), Object::Stream(s)); + } + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 0, "unsafe input {page_ops}, {form_ops}, {extra}"); + assert!(!result.refused.is_empty()); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before); + } +} + +#[test] +fn editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen() { + // Remap existing OFL Inter outlines to synthetic Thai glyph slots; no proprietary font + // fixture is needed. Removing layout/cmap tables reproduces legacy PDF font subsetting. + let mut program = include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec(); + let count = u16::from_be_bytes(program[4..6].try_into().unwrap()) as usize; + let records: Vec<_> = program[12..12 + count * 16] + .as_chunks::<16>() + .0 + .iter() + .filter(|r| !matches!(&r[..4], b"cmap" | b"GSUB" | b"GPOS")) + .map(|r| r.to_vec()) + .collect(); + program[4..6].copy_from_slice(&(records.len() as u16).to_be_bytes()); + for (index, record) in records.iter().enumerate() { + program[12 + index * 16..28 + index * 16].copy_from_slice(record); + } + for bold in [false, true] { + let mut doc = text_page("BT /F1 17.5 Tf 0.42 Tc 1.1 Tw 86 Tz 1.5 Ts 2 Tr 0.6 w 0.2 0.3 0.4 rg 50 650 Td <00d601090119> Tj ET"); + let raw = doc.add(Object::Stream(Stream::flate(Dict::new(), &program))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), Object::Ref(raw)); + let mut descendant = Dict::new(); + descendant.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + descendant.set(b"CIDToGIDMap".to_vec(), Object::name("Identity")); + descendant.set(b"W".to_vec(), Object::Array(vec![Object::Int(0), Object::Int(65535), Object::Int(500)])); + let unicode = doc.add(Object::Stream(Stream::flate( + Dict::new(), + b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 2 beginbfchar <0160> <0e48> <0161> <0e49> endbfchar", + ))); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type0")); + font.set(b"BaseFont".to_vec(), Object::name(if bold { "ABCDEF+THSarabunPSK-Bold" } else { "ABCDEF+THSarabunPSK" })); + font.set(b"Encoding".to_vec(), Object::name("Identity-H")); + font.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(descendant)])); + font.set(b"ToUnicode".to_vec(), Object::Ref(unicode)); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font.clone())); + let before = text::text_lines(&doc, 0).unwrap()[0].clone(); + let fonts = page_font_programs(&doc); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กิ่ กี้ น้ำ"), &style).expect("existing source glyphs must remain editable"); + assert_eq!(edited.substituted, None, "keeping the source must never silently select Noto or another face"); + let doc = reopen(&doc); + let lines = text::text_lines(&doc, 0).unwrap(); + assert_eq!(lines[0].text, "กิ่ กี้ น้ำ"); + for after in &lines { + assert_eq!(after.font, before.font); + assert_eq!(after.base_font, before.base_font); + assert_eq!(after.bold, bold); + assert_eq!(after.size, before.size); + assert_eq!(after.color, before.color); + assert_eq!(after.baseline(), before.baseline()); + } + let mut settings = std::collections::BTreeMap::new(); + for op in pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops { + if matches!(op.op.as_slice(), b"Tc" | b"Tw" | b"Tz" | b"Ts" | b"Tr" | b"w") { + settings.insert(op.op.clone(), op.num(0).unwrap()); + } + let shows = if op.is("Tj") { + op.operands.first().and_then(Object::as_string).is_some_and(|s| !s.bytes.is_empty()) + } else if op.is("TJ") { + op.operands + .first() + .and_then(Object::as_array) + .is_some_and(|a| a.iter().any(|o| matches!(o, Object::String(s) if !s.bytes.is_empty()))) + } else { + false + }; + if shows { + break; + } + } + for (op, expected) in [("Tc", 0.42), ("Tw", 1.1), ("Tz", 86.0), ("Ts", 1.5), ("Tr", 2.0), ("w", 0.6)] { + assert_eq!(settings.get(op.as_bytes()), Some(&expected), "{op} must survive in the saved PDF"); + } + let mut explicit = doc.clone(); + let chosen = BlockStyle { resource: Some("F1".into()), width: Some(400.0), ..Default::default() }; + text::rewrite_block(&mut explicit, 0, 0, Some("กี้ น้ำ"), &chosen).expect("selecting the existing source font from the menu must work too"); + assert_eq!(text::text_lines(&reopen(&explicit), 0).unwrap()[0].base_font, before.base_font); + assert_eq!(doc.get(pdfcraft_cos::ObjRef::new(5, 0)).as_dict(), Some(&font)); + let Object::Stream(saved) = doc.get(raw).as_ref().clone() else { panic!("fixture font") }; + assert_eq!(saved.decoded().unwrap(), program); + assert_eq!(page_font_programs(&doc), fonts, "no replacement font program may be added"); + } +} diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 3d1626a576..fd6f90e79f 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -31,6 +31,7 @@ pub use session::TextSession; mod blocks; mod editor; mod fallback; +pub mod forms; mod interp; mod lines; mod rewrite; @@ -42,8 +43,8 @@ pub use blocks::{ BlockKey, BlockStyle, RangeStyle, ReadingBlocks, TextBlock, block_overlapping, clear_block, reading_blocks, reading_blocks_checked, replace_block, text_blocks, }; -use editor::layout_draft; pub use editor::{TextEditLayout, TextEditorFont, register_text_font, text_edit_layout, text_editor_fonts}; +use editor::{builtin_thai_choice, layout_draft}; #[cfg(test)] pub(crate) use fallback::SOURCE_SPACE_CALLS; use fallback::{ diff --git a/crates/edit/src/text/blocks.rs b/crates/edit/src/text/blocks.rs index 48fe956f4f..a459073345 100644 --- a/crates/edit/src/text/blocks.rs +++ b/crates/edit/src/text/blocks.rs @@ -400,7 +400,7 @@ pub fn replace_block(doc: &mut Document, page: usize, block: usize, text: &str) pub struct BlockStyle { /// Disjoint UTF-8 ranges in the replacement, each preserving unspecified source style. pub ranges: Vec, - /// Explicit page font resource (from register_text_font or the page font inventory). + /// Explicit page font resource or bundled document-font choice from the page font inventory. pub resource: Option, /// A standard font family, bold, italic. pub family: Option<(crate::added::Family, bool, bool)>, diff --git a/crates/edit/src/text/editor.rs b/crates/edit/src/text/editor.rs index f7aa1f47bd..4f4abf3a43 100644 --- a/crates/edit/src/text/editor.rs +++ b/crates/edit/src/text/editor.rs @@ -2,15 +2,30 @@ use super::blocks::{BlockStyle, TextBlock}; use super::fallback::MAX_TYPE3_GLYPHS; use super::lines::{page_dict, text_lines}; use super::rewrite::rewrite_block_from; -use super::{DecodeBudget, Document, EditError, Metrics, Object, TextLine, TextSession}; +use super::{DecodeBudget, Dict, Document, EditError, Metrics, Object, TextLine, TextSession}; -/// An embedded sfnt source face, read only for the inline editor. Bytes stay in memory and -/// are never installed, copied into assets, or used as a font for another document. +/// An embedded source font or an approved bundled document-font choice. +/// Source bytes stay in memory and are never installed or copied into another document. #[derive(Clone, Debug)] pub struct TextEditorFont { pub resource: String, pub face: String, pub bytes: Vec, + /// A selectable replacement, rather than a source/matching preview face. Merely listing + /// this choice must not reload an open editor's font atlas. + pub choice_only: bool, +} + +/// Reserved document-font choice for explicitly replacing text with the bundled Thai face. +/// This is a selector, never a font resource written into a PDF. +const BUILTIN_THAI_FONT: &str = "builtin:thai"; + +pub(super) fn builtin_thai_choice(fonts: &Dict) -> String { + let mut choice = BUILTIN_THAI_FONT.to_owned(); + while fonts.contains(choice.as_bytes()) { + choice.push('_'); + } + choice } pub fn text_editor_fonts(doc: &Document, page: usize) -> Result, EditError> { @@ -32,6 +47,7 @@ pub fn text_editor_fonts(doc: &Document, page: usize) -> Result Result bool { font.name(b"Subtype") == Some(b"Type3") - && font - .name(b"Name") - .is_some_and(|n| n == JAPANESE_FONT_NAME.as_bytes() || n == HEBREW_FONT_NAME.as_bytes() || n == LATIN_FONT_NAME.as_bytes()) + && font.name(b"Name").is_some_and(|n| { + n == JAPANESE_FONT_NAME.as_bytes() || n == HEBREW_FONT_NAME.as_bytes() || n == LATIN_FONT_NAME.as_bytes() || n == b"PCThai" + }) } /// The generated font's `/Name` and its resource prefix: the bundled Latin face, Hebrew, or the diff --git a/crates/edit/src/text/forms.rs b/crates/edit/src/text/forms.rs new file mode 100644 index 0000000000..71339af64b --- /dev/null +++ b/crates/edit/src/text/forms.rs @@ -0,0 +1,192 @@ +//! Make text in ordinary Form XObjects available to the existing paragraph editor. +//! +//! This replaces each supported placement's `Do`, never the shared form object. The same +//! source glyph operators become page content; no cover rectangle or replacement text box is +//! added. Unsupported compositing and malformed streams stay intact. + +mod expand; + +use pdfcraft_content::{Matrix, Pieces}; +use pdfcraft_cos::{Dict, Document, Object}; + +use super::{page_dict, rewritten_contents, splice}; +use crate::EditError; + +/// Result of preparing one page. Refused groups remain in the page, with an actionable reason. +#[derive(Clone, Debug, Default)] +pub struct Preparation { + pub expanded: usize, + pub refused: Vec, + pub repaired_fonts: usize, +} + +/// Prepare grouped source text atomically, within eight levels and 64 placements per page. +/// Shared forms and other pages are untouched. Preparation is a distinct undoable engine edit. +pub fn prepare(doc: &mut Document, page: usize) -> Result { + let mut next = doc.clone(); + let mut result = Preparation::default(); + let mut bytes_left = 16usize << 20; + let mut visits = 0usize; + let mut decode = pdfcraft_cos::DecodeBudget::new(16 << 20, 0); + for _ in 0..8 { + let p = page_dict(&next, page)?; + let streams = strict_streams(&next, &p.dict)?; + let pieces = Pieces::join(&streams.iter().map(|(_, b)| b.as_slice()).collect::>()); + let parsed = pieces.parse(); + if parsed.skipped > 0 { + return Err(EditError::Invalid("the page has unreadable content; grouped text was left unchanged".into())); + } + let ops = parsed.ops; + let resources = p.dict.get(b"Resources").map(|r| next.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let space = expand::page_blend_space(&next, &p.dict); + let images: std::collections::BTreeMap<_, _> = + crate::page_images(&next, page)?.into_iter().filter(|i| i.is_form).map(|i| (i.op, i)).collect(); + let mut replacements = std::collections::HashMap::new(); + let mut merged = resources; + let mut pending_path = false; + let mut in_text = false; + for (at, op) in ops.iter().enumerate() { + match op.op.as_slice() { + b"m" | b"l" | b"c" | b"v" | b"y" | b"re" | b"h" | b"W" | b"W*" => pending_path = true, + b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n" => pending_path = false, + b"BT" => in_text = true, + b"ET" => in_text = false, + _ => {} + } + let Some(image) = images.get(&at) else { continue }; + visits = visits.saturating_add(1); + if visits > 64 { + return Err(EditError::Invalid("too many grouped text placements on this page (limit 64); the page was left unchanged".into())); + } + if pending_path || in_text { + result.refused.push(format!("{}: the group is drawn inside an unfinished path or text object", image.name)); + continue; + } + let translucent = expand::translucent_at(&next, &merged, &ops, at); + match expand::ungroup_ops(&next, &merged, image.name.as_bytes(), translucent, space.as_ref(), Matrix(image.matrix), &mut decode) { + Ok((inline, mut res, patterns)) => { + // Pure vector artwork needs no preparation. Nested text groups are visited + // on the next pass, with their inherited placement and clip still in force. + if !inline.iter().any(|o| matches!(o.op.as_slice(), b"Tj" | b"TJ" | b"'" | b"\"" | b"Do")) { + continue; + } + let data_len = pdfcraft_content::serialize_ops(&inline).len(); + bytes_left = bytes_left.checked_sub(data_len).ok_or_else(|| { + EditError::Invalid("grouped page content exceeds the 16 MiB editing budget; the page was left unchanged".into()) + })?; + if !patterns.is_empty() { + let mut table = res.get(b"Pattern").map(|r| next.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + for (name, value) in patterns { + table.set(name, Object::Ref(next.add(value))); + } + res.set(b"Pattern".to_vec(), Object::Dict(table)); + } + merged = res; + replacements.insert(at, inline); + result.expanded += 1; + } + Err(error) => result.refused.push(format!("{}: {error}", image.name)), + } + } + if replacements.is_empty() { + break; + } + let changed = splice(&pieces, &ops, |at| match replacements.remove(&at) { + Some(ops) => (ops, false), + None => (Vec::new(), true), + }); + let contents = rewritten_contents(&mut next, &streams, changed); + next.update_dict(p.obj, |d| { + d.set(b"Contents".to_vec(), contents); + d.set(b"Resources".to_vec(), Object::Dict(merged)); + })?; + } + result.refused.sort(); + result.refused.dedup(); + if result.expanded > 0 { + for reason in &result.refused { + next.note_repair(format!("Grouped text left intact: {reason}")); + } + result.repaired_fonts = repair_thai_fonts(&mut next, page)?; + next.note_repair(format!( + "Prepared {} grouped text placement(s) on page {} for direct source-text editing", + result.expanded, + page.saturating_add(1) + )); + *doc = next; + } + Ok(result) +} + +/// A font is copied into this page: repairing the map must never alter a shared form/font. +fn repair_thai_fonts(doc: &mut Document, page: usize) -> Result { + let p = page_dict(doc, page)?; + let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut fonts = res.get(b"Font").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut count = 0; + let mut candidates = 0usize; + for (name, value) in fonts.clone().iter() { + let Some(mut font) = doc.resolve(value).as_dict().cloned() else { continue }; + if font.name(b"BaseFont").is_some_and(|n| String::from_utf8_lossy(n).contains("THSarabunPSK")) { + candidates += 1; + if candidates > 64 { + return Err(EditError::Invalid("too many legacy Thai font maps to repair safely; the page was left unchanged".into())); + } + } else { + continue; + } + let Some(map) = pdfcraft_fonts::pdf::recovered_thai_map(doc, &font) else { continue }; + // Append corrective bfchar entries after the original entries so valid Latin mappings + // remain. The original map's accidental oversized range is still overridden for Thai. + let Some(Object::Stream(tu)) = font.get(b"ToUnicode").map(|r| doc.resolve(r)).as_deref().cloned() else { continue }; + let old = pdfcraft_cos::DecodeBudget::new(1 << 20, 0).decode_strict(&tu)?; + let marker = b"endcmap"; + let Some(at) = old.windows(marker.len()).position(|w| w == marker) else { continue }; + let mut added = format!("\n{} beginbfchar\n", map.len()); + for (code, ch) in map { + added.push_str(&format!("<{code:04x}> <{:04x}>\n", u32::from(ch))); + } + added.push_str("endbfchar\n"); + let mut bytes = old.get(..at).unwrap_or_default().to_vec(); + bytes.extend_from_slice(added.as_bytes()); + bytes.extend_from_slice(old.get(at..).unwrap_or_default()); + let tu = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), &bytes))); + font.set(b"ToUnicode".to_vec(), Object::Ref(tu)); + let object = doc.add(Object::Dict(font)); + fonts.set(name.clone(), Object::Ref(object)); + count += 1; + } + if count > 0 { + res.set(b"Font".to_vec(), Object::Dict(fonts)); + doc.update_dict(p.obj, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + doc.note_repair(format!( + "Recovered the damaged legacy THSarabunPSK Thai character map in {count} page font(s); original glyph programs were preserved" + )); + } + Ok(count) +} + +/// Editing cannot infer the state after a damaged page stream. Read all pieces strictly before +/// expanding a placement, rather than treating an undecodable piece as empty. +fn strict_streams(doc: &Document, page: &Dict) -> Result)>, EditError> { + let mut budget = pdfcraft_cos::DecodeBudget::new(16 << 20, 0); + let list = match page.get(b"Contents") { + None => Vec::new(), + Some(value) => match doc.resolve(value).as_ref() { + Object::Null => Vec::new(), + Object::Array(a) if a.len() <= 4096 => a.clone(), + Object::Array(_) => return Err(EditError::Invalid("too many page content streams to prepare safely".into())), + _ => vec![value.clone()], + }, + }; + list.into_iter() + .map(|value| { + let resolved = doc.resolve(&value); + let Object::Stream(stream) = resolved.as_ref() else { + return Err(EditError::Invalid("the page has a missing or unreadable content stream".into())); + }; + let bytes = budget.decode_strict(stream)?; + Ok((value, bytes)) + }) + .collect() +} diff --git a/crates/edit/src/text/forms/expand.rs b/crates/edit/src/text/forms/expand.rs new file mode 100644 index 0000000000..27cbf1b48e --- /dev/null +++ b/crates/edit/src/text/forms/expand.rs @@ -0,0 +1,576 @@ +//! Existing-text preparation: draw a Form XObject's content in place of its `Do` (#2964, #3005). +//! Adapted from mtlcl410's Ungroup implementation in PR #3067, revision +//! `6eafca9caa4a4d2174a395b53914bc9e689097cc` (MIT OR Apache-2.0). +//! +//! Text inside grouped Form artwork can't be edited, because editing rewrites only the page's own +//! content streams. Ungrouping copies the form's content into the page at the same place, +//! wrapped as `q cm re W n … Q`, with the form's resources merged into a copy of +//! the page's under names that don't clash. Its text then edits like any other. The form object +//! itself is left alone, so another page or placement that draws it is unchanged. +//! +//! Drawing a form inline paints the same pixels unless the form is a transparency group whose +//! compositing differs from painting its parts one by one: a knockout group, a group that uses +//! a blend mode other than Normal inside, or a group drawn translucent, blended or under a soft +//! mask, or a group with a blending colour space (`/CS`) other than the page's: its contents +//! composite in that space and are then converted (ISO 32000-2 §11.6.6), inline they would +//! composite in the page's. Those are refused rather than changed, as is tagged form content (its structure link +//! would be lost) and content whose marked content or text objects don't balance (inlined, they +//! would change the scope of the page's own). A form in a layer (`/OC`) keeps its membership as `/OC … BDC … EMC`. +//! +//! A pattern's matrix maps pattern space to the default space of the content stream that uses it +//! (ISO 32000-2 §8.7.2): inside the form that is the form's space at the `Do`, on the page it is +//! the page's. So each pattern the form selects with `scn`/`SCN` is drawn from a copy whose +//! `/Matrix` also carries the form's placement; one that can't be copied that way is refused. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, Op, parse}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object}; + +use crate::EditError; + +/// The largest form content ungrouped (decoded bytes): the page's own content cap is far above +/// any real form, and a bomb must not be copied into the page. +const MAX_FORM_BYTES: usize = 16 << 20; + +/// Resource categories a content stream refers to by name (ISO 32000-2 §7.8.3, Table 34). +const CATEGORIES: [&[u8]; 7] = [b"Font", b"XObject", b"ExtGState", b"ColorSpace", b"Pattern", b"Shading", b"Properties"]; + +/// The most distinct patterns one ungrouped form may select; each becomes a new object. +const MAX_MOVED_PATTERNS: usize = 256; + +/// A name map for one resource category: form name → name in the page's resources. +type Renames = Vec<(Vec, Vec)>; + +/// What [`ungroup_ops`] gives back: the ops that draw the form inline, the page resources with +/// the form's merged in, and the pattern copies to add to the document, each under the name in +/// the merged `/Pattern` dictionary that the ops already use. +pub(crate) type Ungrouped = (Vec, Dict, Vec<(Vec, Object)>); + +/// Ungroup form `name` from the page's `resources`. `translucent` says whether the graphics state +/// at the `Do` is translucent, blended or soft-masked, which a transparency group composites as a +/// whole. `page_space` is the page group's blending colour space ([`page_blend_space`]). +/// `placement` maps the form's space to the page's default space at the `Do` (its `/Matrix`, +/// then the CTM there). +pub(crate) fn ungroup_ops( + doc: &Document, + resources: &Dict, + name: &[u8], + translucent: bool, + page_space: Option<&Object>, + placement: Matrix, + budget: &mut DecodeBudget, +) -> Result { + let refuse = |why: &str| EditError::Invalid(format!("this artwork can't be ungrouped: {why}")); + let xobjects = resources.get(b"XObject").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).ok_or_else(|| refuse("it is missing"))?; + let form = xobjects.get(name).ok_or_else(|| refuse("it is missing"))?; + let Object::Stream(s) = &*doc.resolve(form) else { return Err(refuse("it is not a Form XObject")) }; + if !is_name(doc, &s.dict, b"Subtype", b"Form") { + return Err(refuse("it is not a Form XObject")); + } + let numbers = |key: &[u8], n: usize| -> Option> { + let a = s.dict.get(key).map(|o| doc.resolve(o))?; + if a.as_array()?.len() != n { + return None; + } + let v: Vec = a.as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); + (v.len() == n && v.iter().all(|x| x.is_finite())).then_some(v) + }; + let bbox = numbers(b"BBox", 4).ok_or_else(|| refuse("its bounding box is missing or unreadable"))?; + let matrix = match s.dict.get(b"Matrix") { + None => None, + Some(_) => Some(Matrix( + <[f64; 6]>::try_from(numbers(b"Matrix", 6).ok_or_else(|| refuse("its matrix is unreadable"))?) + .map_err(|_| refuse("its matrix is unreadable"))?, + )), + }; + let own = s.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()); + if s.dict.get(b"Resources").is_some() && own.is_none() { + return Err(refuse("its resources cannot be read")); + } + // A form without its own resources uses the ones it is drawn with (PDF 1.1). + let form_res = own.clone().unwrap_or_else(|| resources.clone()); + + // A tagged form's content is a structure element's: drawn inline it would lose that link. + if s.dict.get(b"StructParent").is_some() || s.dict.get(b"StructParents").is_some() { + return Err(refuse("it is tagged content (Tags panel); edit its text in a copy without tags")); + } + let group = s.dict.get(b"Group").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).filter(|g| is_name(doc, g, b"S", b"Transparency")); + if s.dict.get(b"Group").is_some() && group.is_none() { + return Err(refuse("its group dictionary cannot be read")); + } + if let Some(group) = &group { + if group.get(b"K").is_some_and(|k| matches!(*doc.resolve(k), Object::Bool(true))) { + return Err(refuse("it is a knockout transparency group")); + } + if translucent { + return Err(refuse("it is a transparency group drawn translucent, blended or under a soft mask")); + } + // Its contents composite in its own colour space (ISO 32000-2 §11.6.6, Table 145 /CS); + // inline they would composite in the page's. + if let Some(cs) = group.get(b"CS") + && !same_blend_space(doc, cs, page_space) + { + return Err(refuse("it is a transparency group that blends in a colour space of its own")); + } + } + + // Strictly: damaged data must not be copied in as a shorter form. + let data = budget.decode_strict(s).map_err(|e| refuse(&format!("its content can't be read ({e})")))?; + let parsed = parse(&data); + if parsed.skipped > 0 { + return Err(refuse("its content has parts that can't be read")); + } + let mut ops = parsed.ops; + { + // Check every form, including non-group forms, for recursive/unsupported content. + // The form itself is being read: drawing itself again counts as blending. + let mut walk = BlendWalk { path: form.as_ref().into_iter().collect(), known: HashMap::new(), reads: 0, budget }; + if blends_inside(doc, &form_res, &ops, 0, &mut walk) { + return Err(refuse("it has blend modes, recursive forms, or nested content that cannot be verified")); + } + } + + // Merge the form's resources into a copy of the page's, renaming what clashes. + let mut merged = resources.clone(); + let mut renames: Vec<(&[u8], Renames)> = Vec::new(); + if own.is_some() { + for cat in CATEGORIES { + let Some(theirs) = form_res.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()) else { continue }; + if theirs.len() > 4096 { + return Err(refuse("too many resources")); + } + let mut mine = merged.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); + if mine.len() > 4096 { + return Err(refuse("too many resources")); + } + let mut map = Renames::new(); + let mut n = 0u32; + for (key, value) in theirs.iter() { + if cat == b"ColorSpace" && matches!(key.as_slice(), b"DefaultGray" | b"DefaultRGB" | b"DefaultCMYK") && mine.get(key) != Some(value) { + return Err(refuse("it overrides a default colour space")); + } + if mine.get(key) == Some(value) { + continue; + } + let fresh = loop { + n = n.saturating_add(1); + let candidate = format!("PCUg{n}").into_bytes(); + if mine.get(&candidate).is_none() && theirs.get(&candidate).is_none() { + break candidate; + } + if n == u32::MAX { + return Err(refuse("too many resources")); + } + }; + mine.set(fresh.clone(), value.clone()); + map.push((key.to_vec(), fresh)); + } + merged.set(cat.to_vec(), Object::Dict(mine)); + if !map.is_empty() { + renames.push((cat, map)); + } + } + } + let renamed = |cat: &[u8], name: &[u8]| -> Option> { + renames.iter().find(|(c, _)| *c == cat).and_then(|(_, m)| m.iter().find(|(from, _)| from == name)).map(|(_, to)| to.clone()) + }; + // A nested form, Type 3 font or tiling pattern without resources of its own looks its names + // up in the form's (the renderer chains them to whatever draws it). Drawn from the page, it + // would find the page's, which differ wherever a name was renamed (every form resource that + // isn't already the page's, the nested item's own entry included). + if !renames.is_empty() && ops.iter().any(|op| inherits_resources(doc, &form_res, op)) { + return Err(refuse("it draws a nested form, Type 3 font or pattern that uses its resources")); + } + let mut depth = 0usize; + // Marked content and text objects must balance within the form's own stream: an extra EMC + // would close one the page opened, and one left open would take in the page's later content. + let mut marked = 0usize; + let mut in_text = false; + for op in &mut ops { + match op.op.as_slice() { + b"BDC" | b"BMC" => marked = marked.saturating_add(1), + b"EMC" => marked = marked.checked_sub(1).ok_or_else(|| refuse("its content ends marked content it didn't begin"))?, + b"BT" if in_text => return Err(refuse("its content begins a text object inside another")), + b"BT" => in_text = true, + b"ET" if !in_text => return Err(refuse("its content ends a text object it didn't begin")), + b"ET" => in_text = false, + _ => {} + } + let slot: Option<(&[u8], usize)> = match op.op.as_slice() { + b"Tf" | b"Do" | b"gs" | b"sh" | b"cs" | b"CS" => { + let cat: &[u8] = match op.op.as_slice() { + b"Tf" => b"Font", + b"Do" => b"XObject", + b"gs" => b"ExtGState", + b"sh" => b"Shading", + _ => b"ColorSpace", + }; + Some((cat, 0)) + } + b"scn" | b"SCN" => op.operands.len().checked_sub(1).map(|i| (&b"Pattern"[..], i)), + b"BDC" | b"DP" => Some((&b"Properties"[..], 1)), + b"q" => { + if depth >= 4096 { + return Err(refuse("graphics state is too deeply nested")); + } + depth += 1; + None + } + b"Q" => { + depth = depth.checked_sub(1).ok_or_else(|| refuse("its content restores more graphics states than it saves"))?; + None + } + _ => None, + }; + if let Some((cat, i)) = slot + && let Some(operand) = op.operands.get_mut(i) + && let Object::Name(n) = operand + && let Some(to) = renamed(cat, n) + { + *operand = Object::Name(to); + } + if let Some((dict, _)) = &mut op.inline { + for key in [&b"CS"[..], b"ColorSpace"] { + if let Some(Object::Name(n)) = dict.get(key) + && let Some(to) = renamed(b"ColorSpace", n) + { + dict.set(key.to_vec(), Object::Name(to)); + } + } + } + } + + if marked > 0 || in_text { + return Err(refuse("its content leaves marked content or a text object open")); + } + + // Patterns keep their place relative to the shapes they fill: each one the form selects is + // drawn from a copy that also carries the placement. Nested forms and patterns inside + // patterns keep their own parents, so only this stream's selections change. + let mut patterns: Vec<(Vec, Object)> = Vec::new(); + let mut pattern_bytes = MAX_FORM_BYTES; + if placement != Matrix::IDENTITY { + let mut table = merged.get(b"Pattern").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); + let mut moved: Renames = Vec::new(); + let mut n = 0u32; + for op in &mut ops { + if !(op.is("scn") || op.is("SCN")) { + continue; + } + let Some(i) = op.operands.len().checked_sub(1) else { continue }; + let Some(Object::Name(from)) = op.operands.get(i) else { continue }; + // A name the resources don't have selects nothing, inline or not. + let Some(entry) = table.get(from).cloned() else { continue }; + let to = match moved.iter().find(|(f, _)| f == from) { + Some((_, to)) => to.clone(), + None => { + if moved.len() >= MAX_MOVED_PATTERNS { + return Err(refuse("it paints with too many patterns")); + } + let copy = + moved_pattern(doc, &entry, placement).ok_or_else(|| refuse("it paints with a pattern whose position can't be kept"))?; + if let Object::Stream(s) = © { + pattern_bytes = + pattern_bytes.checked_sub(s.raw.len()).ok_or_else(|| refuse("its pattern copies exceed the editing budget"))?; + } + let fresh = loop { + n = n.saturating_add(1); + let candidate = format!("PCUgPat{n}").into_bytes(); + if table.get(&candidate).is_none() { + break candidate; + } + if n == u32::MAX { + return Err(refuse("too many resources")); + } + }; + // Reserved here; the caller replaces it with the copy's reference. + table.set(fresh.clone(), Object::Null); + patterns.push((fresh.clone(), copy)); + moved.push((from.clone(), fresh.clone())); + fresh + } + }; + if let Some(operand) = op.operands.get_mut(i) { + *operand = Object::Name(to); + } + } + if !patterns.is_empty() { + merged.set(b"Pattern".to_vec(), Object::Dict(table)); + } + } + + // A form in a layer (/OC) stays in it: its content is marked with the same membership. + let layer = match s.dict.get(b"OC") { + None => None, + Some(oc) => { + let mut props = merged.get(b"Properties").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); + let existing = props.iter().find(|(_, v)| *v == oc).map(|(k, _)| k.clone()); + let name = match existing { + Some(k) => k, + None => { + let fresh = (1..=u32::MAX) + .map(|n| format!("PCUgOC{n}").into_bytes()) + .find(|c| props.get(c).is_none()) + .ok_or_else(|| refuse("too many resources"))?; + props.set(fresh.clone(), oc.clone()); + merged.set(b"Properties".to_vec(), Object::Dict(props)); + fresh + } + }; + Some(name) + } + }; + + let num = |v: f64| Object::Real(v); + let mut out = Vec::new(); + if let Some(name) = &layer { + out.push(Op::new("BDC", vec![Object::name("OC"), Object::Name(name.clone())])); + } + out.push(Op::new("q", Vec::new())); + if let Some(m) = matrix { + out.push(Op::new("cm", m.0.iter().copied().map(num).collect())); + } + let [a, b, c, d] = <[f64; 4]>::try_from(bbox).map_err(|_| refuse("invalid form bounds"))?; + let (x0, y0, x1, y1) = (a.min(c), b.min(d), a.max(c), b.max(d)); + out.push(Op::new("re", vec![num(x0), num(y0), num(x1 - x0), num(y1 - y0)])); + out.push(Op::new("W", Vec::new())); + out.push(Op::new("n", Vec::new())); + out.extend(ops); + // A form discards its unfinished current path when returning to its caller. + out.push(Op::new("n", Vec::new())); + // A form ends with the state it was drawn with (the Do saves and restores it). + out.extend(std::iter::repeat_with(|| Op::new("Q", Vec::new())).take(depth)); + out.push(Op::new("Q", Vec::new())); + if layer.is_some() { + out.push(Op::new("EMC", Vec::new())); + } + Ok((out, merged, patterns)) +} + +/// A copy of pattern `entry` whose `/Matrix` maps its space to the page's default space instead +/// of the form's: its own matrix, then `placement`. `None` when it isn't a readable pattern, its +/// matrix is unreadable, the result isn't finite or a tiling pattern is too large to copy. +fn moved_pattern(doc: &Document, entry: &Object, placement: Matrix) -> Option { + let resolved = doc.resolve(entry); + let (mut dict, stream) = match &*resolved { + Object::Stream(s) => (s.dict.clone(), Some(s)), + Object::Dict(d) => (d.clone(), None), + _ => return None, + }; + let own = match dict.get(b"Matrix") { + None => Matrix::IDENTITY, + Some(m) => { + let m = doc.resolve(m); + if m.as_array()?.len() != 6 { + return None; + } + let nums: Vec = m.as_array()?.iter().map(|x| doc.resolve(x).as_ref().clone()).collect(); + Matrix::from_operands(&nums)? + } + }; + let moved = own.then(&placement); + if !moved.0.iter().all(|v| v.is_finite()) { + return None; + } + dict.set(b"Matrix".to_vec(), Object::Array(moved.0.iter().map(|v| Object::Real(*v)).collect())); + match stream { + Some(s) if s.raw.len() > MAX_FORM_BYTES => None, + Some(s) => Some(Object::Stream(pdfcraft_cos::Stream { dict, raw: s.raw.clone() })), + None => Some(Object::Dict(dict)), + } +} + +/// How deep [`blends_inside`] follows nested forms; deeper ones count as blending. +const MAX_NESTING: usize = 8; + +/// The most nested forms and tiling patterns [`blends_inside`] reads in all; past that the rest +/// count as blending, so a file can't make the check run for ever. +const MAX_NESTED_READS: usize = 64; + +/// What [`blends_inside`] carries through its nested reads: the forms on the current path (a +/// form drawing itself), the answer for each form with resources of its own (its answer doesn't +/// depend on who draws it), and how many reads it has made. +struct BlendWalk<'a> { + path: Vec, + known: HashMap, + reads: usize, + budget: &'a mut DecodeBudget, +} + +/// Whether `op` draws a Form XObject (`Do`), Type 3 font (`Tf`) or tiling pattern (`scn`/`SCN`) from +/// `resources` that has no `/Resources` of its own, and so inherits the ones it is drawn with. +fn inherits_resources(doc: &Document, resources: &Dict, op: &Op) -> bool { + let (cat, i): (&[u8], usize) = match op.op.as_slice() { + b"Do" => (b"XObject", 0), + b"Tf" => (b"Font", 0), + b"scn" | b"SCN" => match op.operands.len().checked_sub(1) { + Some(i) => (b"Pattern", i), + None => return false, + }, + _ => return false, + }; + let Some(Object::Name(name)) = op.operands.get(i) else { return false }; + let Some(entry) = resources.get(cat).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().and_then(|d| d.get(name).cloned())) else { + return false; + }; + let dict = match &*doc.resolve(&entry) { + Object::Stream(s) => s.dict.clone(), + Object::Dict(d) => d.clone(), + _ => return false, + }; + let drawn = match cat { + b"XObject" => is_name(doc, &dict, b"Subtype", b"Form"), + b"Font" => is_name(doc, &dict, b"Subtype", b"Type3"), + // A shading pattern has no content of its own. + _ => int_of(doc, &dict, b"PatternType") == Some(1), + }; + drawn && dict.get(b"Resources").is_none() +} + +/// Whether `ops` blend through a used graphics state, nested form or pattern. Unreadable, +/// recursive or over-budget nested content is treated as blending, so preparation refuses it. +fn blends_inside(doc: &Document, resources: &Dict, ops: &[Op], depth: usize, walk: &mut BlendWalk<'_>) -> bool { + let category = |cat: &[u8]| resources.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); + let (states, xobjects, patterns) = (category(b"ExtGState"), category(b"XObject"), category(b"Pattern")); + let state_blends = |g: Option<&Object>| g.map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).is_some_and(|gs| blends(doc, &gs)); + for op in ops { + if op.is("gs") && state_blends(op.name(0).and_then(|n| states.get(n))) { + return true; + } + let o = if op.is("Do") { + op.name(0).and_then(|n| xobjects.get(n)) + } else if op.is("scn") || op.is("SCN") { + op.operands.len().checked_sub(1).and_then(|i| op.name(i)).and_then(|n| patterns.get(n)) + } else { + continue; + }; + let Some(o) = o else { continue }; + let resolved = doc.resolve(o); + let nested = match &*resolved { + Object::Stream(st) if op.is("Do") && is_name(doc, &st.dict, b"Subtype", b"Form") => st, + Object::Stream(st) if !op.is("Do") && int_of(doc, &st.dict, b"PatternType") == Some(1) => st, + // A shading pattern. + Object::Stream(st) if !op.is("Do") => { + if state_blends(st.dict.get(b"ExtGState")) { + return true; + } + continue; + } + Object::Dict(d) if !op.is("Do") => { + if state_blends(d.get(b"ExtGState")) { + return true; + } + continue; + } + _ => continue, + }; + let r = o.as_ref(); + if let Some(known) = r.and_then(|r| walk.known.get(&r)) { + if *known { + return true; + } + continue; + } + if depth >= MAX_NESTING || r.is_some_and(|r| walk.path.contains(&r)) || walk.reads >= MAX_NESTED_READS { + return true; + } + walk.reads += 1; + let Ok(data) = walk.budget.decode_strict(nested) else { return true }; + let own_resources = nested.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()); + let has_own = own_resources.is_some(); + let own = own_resources.unwrap_or_else(|| resources.clone()); + let inner = parse(&data); + if inner.skipped > 0 { + return true; + } + if let Some(r) = r { + walk.path.push(r); + } + let blends = blends_inside(doc, &own, &inner.ops, depth + 1, walk); + if let Some(r) = r { + walk.path.retain(|x| *x != r); + if has_own { + walk.known.insert(r, blends); + } + } + if blends { + return true; + } + } + false +} + +/// The blending colour space of the page group (ISO 32000-2 §11.4.7): the page's transparency +/// `/Group /CS`, resolved. `None` when the page names none, and the PDF processor chooses. +pub(crate) fn page_blend_space(doc: &Document, page: &Dict) -> Option { + let group = page.get(b"Group").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).filter(|g| is_name(doc, g, b"S", b"Transparency"))?; + group.get(b"CS").map(|cs| (*doc.resolve(cs)).clone()) +} + +/// Whether a group blending in `cs` composites as the page does: in the page group's own space +/// when it names one, else in DeviceRGB, the space viewers composite a page in on screen. +fn same_blend_space(doc: &Document, cs: &Object, page_space: Option<&Object>) -> bool { + let cs = doc.resolve(cs); + match page_space { + Some(page) => *cs == *page, + None => cs.as_name() == Some(b"DeviceRGB"), + } +} + +/// Whether `d`'s `key` is the name `want`, directly or through an indirect object (any value may +/// be one, ISO 32000-2 §7.3.10). +fn is_name(doc: &Document, d: &Dict, key: &[u8], want: &[u8]) -> bool { + d.get(key).is_some_and(|v| doc.resolve(v).as_name() == Some(want)) +} + +/// `d`'s integer `key`, directly or through an indirect object. +fn int_of(doc: &Document, d: &Dict, key: &[u8]) -> Option { + d.get(key).and_then(|v| doc.resolve(v).as_int()) +} + +/// Whether an ExtGState blends with something other than Normal. +fn blends(doc: &Document, gs: &Dict) -> bool { + let Some(bm) = gs.get(b"BM").map(|b| doc.resolve(b)) else { return false }; + let first = match &*bm { + Object::Array(a) => a.first().map(|x| doc.resolve(x)), + _ => Some(bm.clone()), + }; + !first.is_some_and(|b| matches!(b.as_name(), Some(b"Normal" | b"Compatible"))) +} + +/// Whether the graphics state is translucent, blended or soft-masked at op `at` of the page's +/// streams parsed as one (the indexes `page_images` gives), following `q`/`Q` and `gs`. Each `gs` +/// changes only the parameters it sets. +pub(crate) fn translucent_at(doc: &Document, resources: &Dict, ops: &[Op], at: usize) -> bool { + let states = resources.get(b"ExtGState").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); + // Fill alpha below one, stroke alpha below one, a soft mask, a blend mode other than Normal. + let mut stack: Vec<[bool; 4]> = Vec::new(); + let mut now = [false; 4]; + for op in ops.iter().take(at) { + match op.op.as_slice() { + b"q" => stack.push(now), + b"Q" => now = stack.pop().unwrap_or([false; 4]), + b"gs" => { + let Some(gs) = op.name(0).and_then(|n| states.get(n)).map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()) else { + continue; + }; + let alpha = |key: &[u8]| gs.get(key).map(|v| doc.resolve(v)).and_then(|v| v.as_f64()).map(|a| a < 1.0); + if let Some(v) = alpha(b"ca") { + now[0] = v; + } + if let Some(v) = alpha(b"CA") { + now[1] = v; + } + if let Some(m) = gs.get(b"SMask").map(|v| doc.resolve(v)) { + now[2] = m.as_name() != Some(b"None"); + } + if gs.get(b"BM").is_some() { + now[3] = blends(doc, &gs); + } + } + _ => {} + } + } + now.iter().any(|v| *v) +} diff --git a/crates/edit/src/text_regions.rs b/crates/edit/src/text_regions.rs index 83046bfbf7..98de481444 100644 --- a/crates/edit/src/text_regions.rs +++ b/crates/edit/src/text_regions.rs @@ -2,9 +2,10 @@ use super::{ BlockStyle, CraftFont, Dict, Document, EditError, HashMap, LineEdit, MAX_TYPE3_GLYPHS, Matrix, Metrics, Object, Op, PdfString, Pieces, - RangeStyle, TextBlock, TextLine, Type3Glyph, coincident_ops, content_streams, fallback_face, japanese_glyph_from, needs_type3, no_fallback_font, - no_glyph_for, page_dict, paragraph_separator, reveal_ops, rewritten_contents, same_base_font, same_font, source_encodes, source_family, - source_program, source_show, source_space, source_width, splice, strip_subset_tag, type3_faces, type3_glyph, union, write_type3_glyphs, + RangeStyle, TextBlock, TextLine, Type3Glyph, builtin_thai_choice, coincident_ops, content_streams, fallback_face, japanese_glyph_from, + needs_type3, no_fallback_font, no_glyph_for, page_dict, paragraph_separator, reveal_ops, rewritten_contents, same_base_font, same_font, + source_encodes, source_family, source_program, source_show, source_space, source_width, splice, strip_subset_tag, type3_faces, type3_glyph, + union, write_type3_glyphs, }; #[path = "text_regions/grouping.rs"] diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 7a9204a8c0..5d8075785e 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -2,9 +2,9 @@ use super::super::interp::TextState; use super::grouping::{across, is_centred, row_rect}; use super::{ BlockStyle, CraftFont, Dict, Document, EditError, HashMap, LineEdit, MAX_TYPE3_GLYPHS, Matrix, Metrics, Object, Op, PdfString, Pieces, - RangeStyle, TextBlock, TextLine, Type3Glyph, coincident_ops, content_streams, fallback_face, japanese_glyph_from, needs_type3, no_fallback_font, - no_glyph_for, page_dict, reveal_ops, rewritten_contents, source_encodes, source_family, source_program, source_show, source_space, source_width, - splice, type3_faces, type3_glyph, write_type3_glyphs, + RangeStyle, TextBlock, TextLine, Type3Glyph, builtin_thai_choice, coincident_ops, content_streams, fallback_face, japanese_glyph_from, + needs_type3, no_fallback_font, no_glyph_for, page_dict, reveal_ops, rewritten_contents, source_encodes, source_family, source_program, + source_show, source_space, source_width, splice, type3_faces, type3_glyph, write_type3_glyphs, }; // The draft is capped at 4 MiB; also bound the outline data prepared across all fallback faces. @@ -104,9 +104,16 @@ impl FontResolver<'_> { } } +struct LegacyFace { + glyphs: Option, + metrics: Metrics, +} + struct PreparedCharacters { values: Vec, substituted: Option, + outlined_thai: std::collections::HashSet, + legacy_thai: HashMap, } struct CharacterPreparation<'a> { @@ -125,12 +132,73 @@ struct CharacterPreparation<'a> { impl CharacterPreparation<'_> { fn prepare(self) -> Result { let Self { doc, first, lines, draft, spans, style, overrides, fonts, metrics, max_glyphs } = self; + let thai_choice = builtin_thai_choice(fonts); + let mut outlined_thai = std::collections::HashSet::new(); + let mut legacy_thai = HashMap::new(); let mut chars = Vec::new(); let mut substituted = None; let mut fallback = FallbackBuilder::new(max_glyphs); let mut resolver = FontResolver { doc, fonts, metrics, full_faces: HashMap::new(), programs: HashMap::new() }; for (range, index) in spans { let source = lines.get(index).unwrap_or(first); + let span = draft.get(range.clone()).unwrap_or_default(); + // Keep the exact source face. A different Thai face requires an explicit choice. + // Interface fallback fonts never enter a PDF. + let selected_resource = overrides.get(&index).and_then(|r| r.resource.as_deref()).or(style.resource.as_deref()); + let use_thai_outlines = if selected_resource == Some(thai_choice.as_str()) { + let face = pdfcraft_fonts::document_thai_font() + .ok_or_else(|| EditError::Invalid("the bundled Thai font is unavailable in this build; load a complete Thai font".into()))?; + if !pdfcraft_fonts::embedding::covers(face.bytes, span) { + return Err(EditError::Invalid(format!( + "{} cannot show every character in this selection; choose a font that covers the text", + face.family + ))); + } + substituted = Some(face.family.to_owned()); + true + } else { + selected_resource.is_none() && matching_thai_outline(resolver.doc, resolver.fonts, source, span) + }; + if use_thai_outlines { + substituted = pdfcraft_fonts::document_thai_font().map(|f| f.family.to_owned()); + } + let existing_thai = if !use_thai_outlines { + let chosen = match selected_resource { + None => Some(source.clone()), + Some(resource) => resolver.fonts.get(resource.as_bytes()).and_then(|f| resolver.doc.dict(f)).map(|font| { + let mut chosen = source.clone(); + chosen.font = resource.to_owned(); + chosen.base_font = Metrics::from_dict(resolver.doc, &font).base_font; + chosen + }), + }; + match chosen { + Some(chosen) => legacy_thai_font(resolver.doc, resolver.fonts, &chosen, span, &mut legacy_thai)?, + None => None, + } + } else { + None + }; + let outline_name = if use_thai_outlines { + // A draft-only name prevents this choice from affecting another style span that + // explicitly selects the original resource. It is never saved into the PDF. + let mut name = format!("PCOutlineThai{index}"); + while resolver.metrics.contains_key(&name) { + name.push('_'); + } + if let Some(m) = resolver + .metrics + .get(&source.font) + .or_else(|| source.origin.state.font.as_ref().and_then(|(name, _)| resolver.metrics.get(name))) + .cloned() + { + resolver.metrics.insert(name.clone(), m); + } + outlined_thai.insert(name.clone()); + Some(name) + } else { + None + }; let start = range.start; for (offset, ch) in draft.get(range).unwrap_or_default().char_indices() { if ch == '\n' { @@ -144,7 +212,11 @@ impl CharacterPreparation<'_> { } let text = ch.to_string(); let requested = overrides.get(&index).and_then(|range| range.resource.as_deref()).or(style.resource.as_deref()); - let name = if let Some(name) = resolver.encodable_font(source, &text, requested)? { + let name = if let Some(name) = &outline_name { + name.clone() + } else if let Some(name) = &existing_thai { + name.clone() + } else if let Some(name) = resolver.encodable_font(source, &text, requested)? { name } else if needs_type3(&text) { fallback.prepare(source, ch, &text, chars.len())? @@ -173,7 +245,7 @@ impl CharacterPreparation<'_> { } substituted = Some(written.family.to_owned()); } - Ok(PreparedCharacters { values: chars, substituted }) + Ok(PreparedCharacters { values: chars, substituted, outlined_thai, legacy_thai }) } } @@ -233,7 +305,16 @@ struct StyledLines<'a> { type StyledChar = (char, usize, String); type StyledRows = Vec>; type RowRanges = Vec>; -type ShapedRunCache = HashMap<(usize, usize), (Vec, pdfcraft_fonts::source_shaping::ShapedRun, String)>; +enum ShapedFont { + Existing(Vec<(String, Vec)>), + Program { bytes: Vec, family: String }, + ThaiOutlines, +} +struct ShapedText { + run: pdfcraft_fonts::source_shaping::ShapedRun, + font: ShapedFont, +} +type ShapedRunCache = HashMap<(usize, usize), ShapedText>; struct ShapedRows { runs: ShapedRunCache, @@ -243,6 +324,8 @@ struct ShapedRows { fn prepare_shaped_rows( doc: &Document, fonts: &Dict, + outlined_thai: &std::collections::HashSet, + legacy_thai: &HashMap, rows: &[Vec], kept: &[bool], advance: impl Fn(&StyledChar) -> f64, @@ -259,8 +342,8 @@ fn prepare_shaped_rows( let segment = row.get(at..end).unwrap_or_default(); advances.insert((row_index, at), segment.iter().map(&advance).sum::()); let text: String = segment.iter().map(|character| character.0).collect(); - if let Some((bytes, shaped, family)) = shape_source(doc, fonts, &item.2, &text)? { - runs.insert((row_index, at), (bytes, shaped, family)); + if let Some(shaped) = shape_source(doc, fonts, outlined_thai, legacy_thai, &item.2, &text)? { + runs.insert((row_index, at), shaped); } at = end; } @@ -268,16 +351,138 @@ fn prepare_shaped_rows( Ok(ShapedRows { runs, advances }) } -fn embed_shaped_resources(doc: &mut Document, fonts: &mut Dict, runs: &ShapedRunCache) -> Result, EditError> { +type ShapedResources = HashMap<(usize, usize), Vec<(String, Vec)>>; + +fn embed_shaped_resources(doc: &mut Document, fonts: &mut Dict, runs: &ShapedRunCache) -> Result { let mut resources = HashMap::new(); - for (key, (bytes, shaped, family)) in runs { - let mappings: Vec<_> = shaped.glyphs.iter().map(|glyph| (glyph.text.clone(), glyph.id)).collect(); - let resource = pdfcraft_fonts::embedding::embed_mapping(doc, fonts, bytes, family, &mappings).map_err(EditError::Invalid)?; - resources.insert(*key, resource); + for (key, shaped) in runs { + let glyphs = match &shaped.font { + ShapedFont::Existing(codes) => codes.clone(), + ShapedFont::ThaiOutlines => crate::added::thai_glyph_resources(doc, fonts, &shaped.run)?, + ShapedFont::Program { bytes, family } => { + let mappings: Vec<_> = shaped.run.glyphs.iter().map(|glyph| (glyph.text.clone(), glyph.id)).collect(); + let resource = pdfcraft_fonts::embedding::embed_mapping(doc, fonts, bytes, family, &mappings).map_err(EditError::Invalid)?; + (0..shaped.run.glyphs.len()) + .map(|i| { + let code = u16::try_from(i.saturating_add(1)).map_err(|_| EditError::Invalid("shaped glyph limit".into()))?; + Ok((resource.clone(), code.to_be_bytes().to_vec())) + }) + .collect::, EditError>>()? + } + }; + resources.insert(*key, glyphs); } Ok(resources) } +/// A positioned glyph run, kept in one text object across Type 3 font shards. +struct ShapedRunDrawing<'a> { + doc: &'a Document, + fonts: &'a Dict, + shaped: &'a pdfcraft_fonts::source_shaping::ShapedRun, + resources: &'a [(String, Vec)], + text: &'a str, + matrix: [f64; 6], + size: f64, + thai: bool, +} + +impl ShapedRunDrawing<'_> { + /// Draw and return the run's advance in text space, including cluster-safe spacing. + fn draw(&self, block_ops: &mut Vec, state: &mut TextState) -> Result { + let Self { doc, fonts, shaped, resources, text, matrix: tm, size, thai } = *self; + let n = pdfcraft_content::num; + let (resource, _) = self.resources.first().ok_or_else(|| EditError::Invalid("shaped run has no glyphs".into()))?; + let mut properties = Dict::new(); + properties.set(b"ActualText".to_vec(), Object::String(PdfString::text(text))); + block_ops.push(Op::new("BDC", vec![Object::name("Span"), Object::Dict(properties)])); + state.font = Some((resource.clone(), size)); + // One text object for the run: each glyph is placed where shaping put it by a TJ + // adjustment from where the glyph before it ends (its advance in the embedded + // font's widths), and raised by the text rise for a mark set above or below + // (ISO 32000-1 §9.4.3, §9.3.7). Font-program codes are separate from Unicode; + // ToUnicode and ActualText keep the logical text. + let per_unit = 1000.0 / size; + if !per_unit.is_finite() { + return Err(EditError::Invalid("the shaped text has no size".into())); + } + let mut shown_fonts = HashMap::new(); + for (name, _) in resources { + if !shown_fonts.contains_key(name) { + let font = fonts.get(name.as_bytes()).and_then(|f| doc.dict(f)).ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; + shown_fonts.insert(name.clone(), Metrics::from_dict(doc, &font)); + } + } + // Keep the original text settings. TJ compensates for the automatic PDF advance + // after each glyph, while shaping keeps marks attached to their cluster. + let (char_spacing, word_spacing, rise) = (state.char_spacing, state.word_spacing, state.rise); + block_ops.push(Op::new("BT", vec![])); + block_ops.extend(state.ops()); + block_ops.push(Op::new("Tm", tm.iter().map(|v| n(*v)).collect())); + // `pen`: where the next glyph would start, in thousandths of the font size. + let (mut pen, mut extra, mut raised) = (0.0, 0.0, rise); + let mut shown_glyphs: Vec = Vec::new(); + let mut cluster_spacing = 0.0; + let mut current_resource = resource.as_str(); + for (i, glyph) in shaped.glyphs.iter().enumerate() { + let (name, code) = resources.get(i).ok_or_else(|| EditError::Invalid("shaped glyph resource is missing".into()))?; + let shown = shown_fonts.get(name).ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; + if name != current_resource { + if !shown_glyphs.is_empty() { + block_ops.push(Op::new("TJ", vec![Object::Array(std::mem::take(&mut shown_glyphs))])); + } + block_ops.push(Op::new("Tf", vec![Object::name(name), n(size)])); + current_resource = name; + } + if thai && !glyph.text.is_empty() { + // Apply spacing only after a whole cluster, keeping marks on their base. + extra += cluster_spacing; + cluster_spacing = char_spacing * glyph.text.chars().count() as f64 + word_spacing * glyph.text.matches(' ').count() as f64; + } + let target = (glyph.position[0] * size + extra) * per_unit; + let glyph_rise = rise + glyph.position[1] * size; + if (glyph_rise - raised).abs() > 1e-9 || shown_glyphs.len() >= 4096 { + if !shown_glyphs.is_empty() { + block_ops.push(Op::new("TJ", vec![Object::Array(std::mem::take(&mut shown_glyphs))])); + } + if (glyph_rise - raised).abs() > 1e-9 { + block_ops.push(Op::new("Ts", vec![n(glyph_rise)])); + raised = glyph_rise; + } + } + if (target - pen).abs() > 1e-6 { + shown_glyphs.push(n(pen - target)); + } + shown_glyphs.push(Object::String(PdfString::literal(code.clone()))); + let cid = code.iter().fold(0_u32, |value, byte| value.saturating_mul(256).saturating_add(u32::from(*byte))); + pen = target + + if code.is_empty() { + 0.0 + } else { + shown.width(cid) * 1000.0 + (char_spacing + if shown.is_space(cid, code.len()) { word_spacing } else { 0.0 }) * per_unit + }; + if !thai { + extra += char_spacing * glyph.text.chars().count() as f64 + word_spacing * glyph.text.matches(' ').count() as f64; + } + } + let end = (shaped.advance * size + extra + cluster_spacing) * per_unit; + if (end - pen).abs() > 1e-6 { + shown_glyphs.push(n(pen - end)); + } + if !shown_glyphs.is_empty() { + block_ops.push(Op::new("TJ", vec![Object::Array(shown_glyphs)])); + } + if (raised - rise).abs() > 1e-9 { + block_ops.push(Op::new("Ts", vec![n(rise)])); + } + block_ops.push(Op::new("ET", vec![])); + (state.char_spacing, state.word_spacing) = (char_spacing, word_spacing); + extra += cluster_spacing; + block_ops.push(Op::new("EMC", vec![])); + Ok(shaped.advance * size + extra) + } +} + fn validate_contextual_rows(rows: &[Vec], kept: &[bool]) -> Result<(), EditError> { for (row, _) in rows.iter().zip(kept).filter(|(_, retained)| !**retained) { let text: String = row.iter().map(|character| character.0).collect(); @@ -578,19 +783,220 @@ fn apply_range_styles<'a>( StyledLines { lines: styled_lines, spans: styled_spans, underlines, overrides } } +/// Select one existing subset of the exact source face for a Thai span. Never replace its +/// family/weight to make an edit succeed; missing glyphs need an explicit font choice. +fn legacy_thai_font( + doc: &Document, + fonts: &Dict, + source: &TextLine, + span: &str, + cache: &mut HashMap, +) -> Result, EditError> { + if !source.base_font.rsplit('+').next().unwrap_or(&source.base_font).starts_with("THSarabunPSK") + || !span.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) + { + return Ok(None); + } + let family = pdfcraft_fonts::embedding::face_key(&source.base_font); + let mut candidates = vec![source.font.clone()]; + candidates.extend(fonts.iter().filter_map(|(name, font)| { + let font = doc.dict(font)?; + (pdfcraft_fonts::embedding::face_key(&Metrics::from_dict(doc, &font).base_font) == family).then(|| String::from_utf8_lossy(name).into_owned()) + })); + for name in candidates.into_iter().take(128) { + if let std::collections::hash_map::Entry::Vacant(entry) = cache.entry(name) { + let Some(font) = fonts.get(entry.key().as_bytes()).and_then(|f| doc.dict(f)) else { continue }; + let metrics = Metrics::from_dict(doc, &font); + let glyphs = pdfcraft_fonts::pdf::LegacyThaiFont::from_pdf(doc, &font, source_space(doc, fonts, &metrics)).ok().flatten(); + entry.insert(LegacyFace { glyphs, metrics }); + } + } + for row in span.split(['\n', '\r']).filter(|row| !row.is_empty()) { + shape_existing_thai(cache, &source.font, row)?; + } + Ok(Some(source.font.clone())) +} + +/// A visual row may use several subsets of the same exact face (often a separate subset for +/// punctuation). Shape Thai clusters intact, and retain each glyph's original PDF font/code. +fn shape_existing_thai(cache: &HashMap, primary: &str, text: &str) -> Result { + if text.len() > 262_144 { + return Err(EditError::Invalid("the Thai source row exceeds its shaping budget".into())); + } + let first = cache.get(primary).ok_or_else(|| EditError::Invalid("the Thai source face is missing".into()))?; + let family = pdfcraft_fonts::embedding::face_key(&first.metrics.base_font); + let mut candidates = vec![(primary, first)]; + candidates.extend( + cache + .iter() + .filter(|(name, face)| name.as_str() != primary && pdfcraft_fonts::embedding::face_key(&face.metrics.base_font) == family) + .map(|(name, face)| (name.as_str(), face)), + ); + // Stable source-first choice, then resource name; never depend on hash iteration order. + if let Some(tail) = candidates.get_mut(1..) { + tail.sort_by_key(|(name, _)| *name); + } + let thai = |c: char| matches!(u32::from(c), 0x0e01..=0x0e5b | 0xf700..=0xf71a); + let mut run = pdfcraft_fonts::source_shaping::ShapedRun { glyphs: Vec::new(), advance: 0.0 }; + let mut resources = Vec::new(); + let mut chars = text.char_indices().peekable(); + while let Some((at, ch)) = chars.next() { + if thai(ch) { + let mut end = at + ch.len_utf8(); + while let Some(&(next, c)) = chars.peek() { + if !thai(c) { + break; + } + chars.next(); + end = next + c.len_utf8(); + } + let segment = text.get(at..end).ok_or_else(|| EditError::Invalid("invalid Thai character boundary".into()))?; + let mut selected = None; + let mut why = String::new(); + for (name, face) in &candidates { + if let Some(font) = &face.glyphs { + match font.shape(segment) { + Ok(shaped) => { + selected = Some((*name, shaped)); + break; + } + Err(e) => { + if why.is_empty() { + why = e; + } + } + } + } + } + let (name, shaped) = selected.ok_or_else(|| { + EditError::Invalid(format!( + "{} cannot keep this edit in the original face: {}; load the complete original font or explicitly choose a replacement", + first.metrics.base_font, + if why.is_empty() { "its source glyph program is incomplete" } else { &why } + )) + })?; + for (mut glyph, code) in shaped.shaped.glyphs.into_iter().zip(shaped.codes) { + glyph.position[0] += run.advance; + resources.push((name.to_owned(), code)); + run.glyphs.push(glyph); + } + run.advance += shaped.shaped.advance; + } else { + let logical = ch.to_string(); + let found = candidates.iter().find_map(|(name, face)| { + let code = face.metrics.encode(&logical)?; + let cid = face.metrics.codes(&code).first()?.0; + Some((*name, code, face.metrics.width(cid))) + }); + let (name, code, width) = if let Some(found) = found { + found + } else if ch == ' ' { + // The source subset may omit spaces. Use a gap measured from a matching + // document subset, then the face's existing text-space fallback. + let width = candidates + .iter() + .find_map(|(_, f)| { + let code = f.metrics.encode(" ")?; + Some(f.metrics.width(f.metrics.codes(&code).first()?.0)) + }) + .unwrap_or(0.25); + (primary, Vec::new(), width) + } else { + return Err(EditError::Invalid(format!( + "{} has no source glyph for {ch} (U+{:04X}); load the complete original face or explicitly select a replacement", + first.metrics.base_font, + u32::from(ch) + ))); + }; + run.glyphs.push(pdfcraft_fonts::source_shaping::ShapedGlyph { id: Default::default(), position: [run.advance, 0.0], text: logical }); + resources.push((name.to_owned(), code)); + run.advance += width; + } + if run.glyphs.len() > 65_534 || !run.advance.is_finite() { + return Err(EditError::Invalid("the Thai source row exceeds its shaping budget".into())); + } + } + Ok(ShapedText { run, font: ShapedFont::Existing(resources) }) +} + +/// Whether the approved Thai face can replace this source, whose embedded subset cannot +/// shape the replacement. A usable complete source program always takes precedence. +fn matching_thai_outline(doc: &Document, fonts: &Dict, source: &TextLine, span: &str) -> bool { + let Some(face) = pdfcraft_fonts::document_thai_font() else { return false }; + if !span.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) + || !pdfcraft_fonts::embedding::matching_faces(&source.base_font).iter().any(|f| std::ptr::eq(*f, face)) + || !pdfcraft_fonts::embedding::covers(face.bytes, span) + { + return false; + } + // A registered complete matching program also completes the source subset. + // Do not replace it with bundled outlines merely because the original resource is a subset. + let family = pdfcraft_fonts::embedding::face_key(&source.base_font); + !fonts.iter().filter_map(|(_, font)| doc.dict(font)).any(|font| { + pdfcraft_fonts::embedding::face_key(&Metrics::from_dict(doc, &font).base_font) == family + && pdfcraft_fonts::pdf::embedded_sfnt(doc, &font).is_some_and(|bytes| { + pdfcraft_fonts::embedding::validate(&bytes).is_ok() + && pdfcraft_fonts::source_shaping::has_thai_layout(&bytes) + && pdfcraft_fonts::embedding::covers(&bytes, span) + && span.split(['\n', '\r']).all(|row| pdfcraft_fonts::source_shaping::shape(&bytes, row).is_ok()) + }) + }) +} + +/// Use the same Thai script selection and mark/cluster positioning as Add Text, including +/// rows beginning with Latin or digits (guessing from their first character would select Latin). +fn shape_thai_row(text: &str) -> Result { + if text.len() > 262_144 || text.contains(['\n', '\r']) { + return Err(EditError::Invalid("shape one bounded text row at a time".into())); + } + let clusters = + pdfcraft_fonts::shape_thai(text).map_err(|e| EditError::Invalid(format!("the matching Thai font cannot shape this text ({e:?})")))?; + let mut glyphs = Vec::new(); + let mut advance = 0.0; + for cluster in clusters { + for (i, (id, [x, y])) in cluster.glyphs.into_iter().enumerate() { + if glyphs.len() >= 65_534 { + return Err(EditError::Invalid("shaped row exceeds its geometry budget".into())); + } + glyphs.push(pdfcraft_fonts::source_shaping::ShapedGlyph { + id: id.into(), + position: [advance + x, y], + text: if i == 0 { cluster.text.clone() } else { String::new() }, + }); + } + advance += cluster.advance; + } + if !advance.is_finite() { + return Err(EditError::Invalid("shaped row exceeds its geometry budget".into())); + } + Ok(pdfcraft_fonts::source_shaping::ShapedRun { glyphs, advance }) +} + /// Shape in the source or an explicitly registered matching full face. Never use interface fonts. fn shape_source( doc: &Document, fonts: &Dict, + outlined_thai: &std::collections::HashSet, + legacy_thai: &HashMap, name: &str, text: &str, -) -> Result, pdfcraft_fonts::source_shaping::ShapedRun, String)>, EditError> { +) -> Result, EditError> { + if outlined_thai.contains(name) { + let shaped = shape_thai_row(text)?; + return Ok(Some(ShapedText { run: shaped, font: ShapedFont::ThaiOutlines })); + } if !pdfcraft_fonts::source_shaping::required(text) { return Ok(None); } let source = fonts.get(name.as_bytes()).and_then(|f| doc.dict(f)).ok_or_else(|| EditError::Invalid("the shaped text has no font resource".into()))?; - let family = Metrics::from_dict(doc, &source).base_font; + let metrics = Metrics::from_dict(doc, &source); + if metrics.base_font.rsplit('+').next().unwrap_or(&metrics.base_font).starts_with("THSarabunPSK") + && text.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) + { + return shape_existing_thai(legacy_thai, name, text).map(Some); + } + let family = metrics.base_font; let mut candidates = vec![source]; candidates.extend( fonts @@ -603,10 +1009,17 @@ fn shape_source( && pdfcraft_fonts::embedding::covers(&bytes, text) && let Ok(shaped) = pdfcraft_fonts::source_shaping::shape(&bytes, text) { - return Ok(Some((bytes, shaped, family))); + return Ok(Some(ShapedText { run: shaped, font: ShapedFont::Program { bytes, family } })); } } - Err(EditError::Invalid("complex text needs the complete source TrueType/OpenType face; load that font before editing".into())) + let fallback = pdfcraft_fonts::document_thai_font() + .filter(|_| text.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f))) + .map(|face| format!("; or select {} in the document-font menu to replace it", face.family)) + .unwrap_or_default(); + Err(EditError::Invalid(format!( + "complex text in {} needs its complete source TrueType/OpenType font; use Load font to keep it{fallback}", + family.rsplit('+').next().unwrap_or(&family) + ))) } /// For each output row (a range of the draft's characters), whether it is exactly the source row @@ -889,6 +1302,8 @@ pub(crate) fn rewrite_styled_block( max_glyphs, } .prepare()?; + let outlined_thai = prepared.outlined_thai; + let legacy_thai = prepared.legacy_thai; let chars = prepared.values; let substituted = prepared.substituted; let size_of = |source: &TextLine, index: usize| { @@ -939,7 +1354,8 @@ pub(crate) fn rewrite_styled_block( } let segment = items.get(at..end).unwrap_or_default(); let string: String = segment.iter().map(|v| v.0).collect(); - if let Some((_, shaped, _)) = shape_source(doc, &fonts, &item.2, &string)? { + if let Some(shaped) = shape_source(doc, &fonts, &outlined_thai, &legacy_thai, &item.2, &string)? { + let shaped = &shaped.run; let source = lines.get(item.1).unwrap_or(first); let scale = overrides.get(&item.1).and_then(|r| r.scale).or(style.scale).map_or(source.origin.state.scale, |s| (s / 100.0).clamp(0.1, 10.0)); @@ -990,7 +1406,7 @@ pub(crate) fn rewrite_styled_block( let at = if split { start } else { o.bt_op }; validate_contextual_rows(&rows, &kept)?; let measured: Vec<_> = rows.iter().map(|row| measure(row)).collect::>()?; - let shaped = prepare_shaped_rows(doc, &fonts, &rows, &kept, advance)?; + let shaped = prepare_shaped_rows(doc, &fonts, &outlined_thai, &legacy_thai, &rows, &kept, advance)?; let ShapedRows { runs: shaped_runs, advances: segment_advances } = shaped; let glyph_resources = embed_shaped_resources(doc, &mut fonts, &shaped_runs)?; let n = pdfcraft_content::num; @@ -1100,67 +1516,19 @@ pub(crate) fn rewrite_styled_block( // drawn in: a `gs` has no inverse, so one run's ExtGState would stay for the next. block_ops.push(Op::new("q", vec![])); block_ops.extend(state.paint_ops(drawn_from, run_ctm)); - if let Some((_, shaped, _)) = shaped_runs.get(&(row_index, pos)) { - let resource = glyph_resources.get(&(row_index, pos)).ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; - let mut properties = Dict::new(); - properties.set(b"ActualText".to_vec(), Object::String(PdfString::text(&string))); - block_ops.push(Op::new("BDC", vec![Object::name("Span"), Object::Dict(properties)])); - state.font = Some((resource.clone(), size_of(source, item.1))); - // One text object for the run: each glyph is placed where shaping put it by a TJ - // adjustment from where the glyph before it ends (its advance in the embedded - // font's widths), and raised by the text rise for a mark set above or below - // (ISO 32000-1 §9.4.3, §9.3.7). Font-program codes are separate from Unicode; - // ToUnicode and ActualText keep the logical text. - let size = size_of(source, item.1); - let per_unit = 1000.0 / size; - if !per_unit.is_finite() { - return Err(EditError::Invalid("the shaped text has no size".into())); - } - let shown = fonts - .get(resource.as_bytes()) - .and_then(|f| doc.resolve(f).as_dict().cloned()) - .map(|d| Metrics::from_dict(doc, &d)) - .ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; - // Spacing is part of each glyph's position, so the run itself has none. - let (char_spacing, word_spacing, rise) = (state.char_spacing, state.word_spacing, state.rise); - state.char_spacing = 0.0; - state.word_spacing = 0.0; - block_ops.push(Op::new("BT", vec![])); - block_ops.extend(state.ops()); - block_ops.push(Op::new("Tm", tm.iter().map(|v| n(*v)).collect())); - // `pen`: where the next glyph would start, in thousandths of the font size. - let (mut pen, mut extra, mut raised) = (0.0, 0.0, rise); - let mut shown_glyphs: Vec = Vec::new(); - for (i, glyph) in shaped.glyphs.iter().enumerate() { - let target = (glyph.position[0] * size + extra) * per_unit; - let glyph_rise = rise + glyph.position[1] * size; - if (glyph_rise - raised).abs() > 1e-9 || shown_glyphs.len() >= 4096 { - if !shown_glyphs.is_empty() { - block_ops.push(Op::new("TJ", vec![Object::Array(std::mem::take(&mut shown_glyphs))])); - } - if (glyph_rise - raised).abs() > 1e-9 { - block_ops.push(Op::new("Ts", vec![n(glyph_rise)])); - raised = glyph_rise; - } - } - if (target - pen).abs() > 1e-6 { - shown_glyphs.push(n(pen - target)); - } - let code = u16::try_from(i + 1).map_err(|_| EditError::Invalid("shaped glyph limit".into()))?; - shown_glyphs.push(Object::String(PdfString::literal(code.to_be_bytes().to_vec()))); - pen = target + shown.width(u32::from(code)) * 1000.0; - extra += char_spacing * glyph.text.chars().count() as f64 + word_spacing * glyph.text.matches(' ').count() as f64; - } - if !shown_glyphs.is_empty() { - block_ops.push(Op::new("TJ", vec![Object::Array(shown_glyphs)])); - } - if (raised - rise).abs() > 1e-9 { - block_ops.push(Op::new("Ts", vec![n(rise)])); - } - block_ops.push(Op::new("ET", vec![])); - (state.char_spacing, state.word_spacing) = (char_spacing, word_spacing); - block_ops.push(Op::new("EMC", vec![])); - segment_width = (shaped.advance * size + extra) * state.scale * source.origin.k; + if let Some(shaped) = shaped_runs.get(&(row_index, pos)) { + let resources = glyph_resources.get(&(row_index, pos)).ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; + let drawing = ShapedRunDrawing { + doc, + fonts: &fonts, + shaped: &shaped.run, + resources, + text: &string, + matrix: tm, + size: size_of(source, item.1), + thai: string.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)), + }; + segment_width = drawing.draw(&mut block_ops, &mut state)? * state.scale * source.origin.k; } else { block_ops.push(Op::new("BT", vec![])); block_ops.extend(state.ops()); diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index 279dc8d563..f3b0d75799 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -255,6 +255,20 @@ impl Document { pdfcraft_create::image_file(&editor.cos, img.object.ok_or("the image has no object")?) } + /// Whether this page has grouped source text that can be prepared without changing its appearance. + /// Used once when entering the existing-text tool, before queuing an undoable preparation. + pub fn can_prepare_text(&self, page: usize) -> bool { + self.text_preparation(page).is_ok_and(|p| p.expanded > 0) + } + + /// Preview preparation, including skipped groups and character-map repairs, without editing. + pub fn text_preparation(&self, page: usize) -> Result { + let editor = self.editor.as_ref().ok_or("the document is not editable")?; + crate::guard(|| pdfcraft_edit::text::forms::prepare(&mut editor.cos.clone(), page)) + .map_err(|e| format!("grouped text could not be prepared safely ({e})"))? + .map_err(|e| e.to_string()) + } + /// Edit a PDF ▸ Edit text: the paragraphs on `page` (0-based). pub fn text_blocks(&self, page: usize) -> Vec { self.editor.as_ref().and_then(|e| e.text_blocks(page).ok()).map(|b| b.as_ref().clone()).unwrap_or_default() diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index 1bcf24108f..f1a99a2387 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -466,6 +466,10 @@ pub enum Edit { /// Formatting changes (font, size, colour, alignment); default keeps the paragraph's. style: pdfcraft_edit::BlockStyle, }, + /// Expose supported grouped source text on one page for direct editing. + PrepareText { + page: usize, + }, /// An explicit open-licensed full font program, completing subsets or selected by a user. RegisterTextFont { page: usize, diff --git a/crates/engine/src/edit/labels.rs b/crates/engine/src/edit/labels.rs index 615ac0fea5..b66ef713f2 100644 --- a/crates/engine/src/edit/labels.rs +++ b/crates/engine/src/edit/labels.rs @@ -111,6 +111,7 @@ pub const EDIT_LABELS: &[&str] = &[ "Delete document JavaScript", "Edit document JavaScript", "Edit text", + "Prepare grouped text", "Load document font", "Move object", "Move objects", @@ -269,6 +270,7 @@ impl Edit { Edit::SetDocumentScript { script: None, .. } => Label::of("Delete document JavaScript"), Edit::SetDocumentScript { .. } => Label::of("Edit document JavaScript"), Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } => Label::of("Edit text"), + Edit::PrepareText { .. } => Label::of("Prepare grouped text"), Edit::RegisterTextFont { .. } => Label::of("Load document font"), Edit::MoveObjects { objects, .. } => plural("Move object", "Move objects", objects.len()), Edit::EditPageImage { change, .. } => Label::of(match change { diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 4719de302f..6368e9097c 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -148,6 +148,7 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re | Edit::RemoveStandardId { .. } | Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } + | Edit::PrepareText { .. } | Edit::RegisterTextFont { .. } | Edit::EditPageImage { .. } | Edit::Flatten { .. } @@ -726,6 +727,14 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::EditTextBlock { page, block, text, style } => { pdfcraft_edit::rewrite_block(doc, *page, *block, Some(text), style)?; } + Edit::PrepareText { page } => { + let result = pdfcraft_edit::text::forms::prepare(doc, *page)?; + if result.expanded == 0 { + return Err(EditError::Invalid( + result.refused.first().cloned().unwrap_or_else(|| "this page has no supported grouped text to prepare".into()), + )); + } + } Edit::RegisterTextFont { page, bytes } => { pdfcraft_edit::text::register_text_font(doc, *page, bytes)?; } diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs index ce51df5e61..4997f64efe 100644 --- a/crates/engine/src/history.rs +++ b/crates/engine/src/history.rs @@ -81,6 +81,7 @@ pub(crate) fn scope_of(edit: &Edit) -> Scope { Edit::MoveObjects { .. } | Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } + | Edit::PrepareText { .. } | Edit::EditPageImage { .. } | Edit::EnhanceScan { .. } | Edit::AddText { .. } diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 9004d88205..7959d2ae3f 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -617,6 +617,8 @@ pub struct DocView { /// Edit text: the lines per page (with the document generation they were read at), and the /// line being edited. pub(crate) edit_lines: HashMap)>, + /// Pages checked for grouped source text during this Edit text activation. + pub(crate) prepared_text_pages: std::collections::HashSet, pub line_editor: Option, /// The paragraph editor whose edit was just queued. It opens again with its typing if the /// edit fails, as a refused form field does. @@ -1006,6 +1008,7 @@ impl DocView { organize_range: Default::default(), pending_edit: None, edit_lines: HashMap::new(), + prepared_text_pages: Default::default(), line_editor: None, applied_editor: None, pending_text_open: None, diff --git a/crates/ui-egui/src/canvas/page_input.rs b/crates/ui-egui/src/canvas/page_input.rs index e1c9719668..34e1362202 100644 --- a/crates/ui-egui/src/canvas/page_input.rs +++ b/crates/ui-egui/src/canvas/page_input.rs @@ -66,6 +66,13 @@ pub(super) fn page_tool_input(ui: &mut egui::Ui, view: &mut DocView, frame: &mut let pcx = comments::PageCx { page: i, xf: &xf, ptr, info, tool, prefs, allowed, hidden: comments_hidden, redact_mark }; let (lines, images) = if matches!(tool, QuickTool::AddText | QuickTool::EditText) && can_modify { edit_inventory(view, doc, i) } else { Default::default() }; + if tool != QuickTool::EditText { + view.prepared_text_pages.clear(); + } else if can_modify && view.pending_edit.is_none() && view.line_editor.is_none() && view.prepared_text_pages.insert(i) && doc.can_prepare_text(i) + { + view.pending_edit = Some(pdfcraft_engine::Edit::PrepareText { page: i }); + ui.ctx().request_repaint(); + } let field_tool = match tool { QuickTool::Field(f) => Some(f), _ => None, diff --git a/crates/ui-egui/src/tools/edit_text_ui/layout.rs b/crates/ui-egui/src/tools/edit_text_ui/layout.rs index 1946519203..19331230a7 100644 --- a/crates/ui-egui/src/tools/edit_text_ui/layout.rs +++ b/crates/ui-egui/src/tools/edit_text_ui/layout.rs @@ -68,7 +68,9 @@ pub(crate) fn editor_fill(text: Color32) -> Color32 { /// One TextEdit lays out all source runs; its caret and selection span the whole region. pub(crate) fn install_source_fonts(ctx: &egui::Context, doc: &pdfcraft_engine::Document, page: usize) -> BTreeMap { - let programs = doc.text_editor_fonts(page); + // Bundled replacement selectors are menu choices, never source resource names. Installing + // an unused choice would make even a standard-font editor wait a frame for its font atlas. + let programs: Vec<_> = doc.text_editor_fonts(page).into_iter().filter(|font| !font.choice_only).collect(); let mut definitions = ctx.fonts(|f| f.definitions().clone()); // Only the current editor's document fonts live in the atlas. Never accumulate the // programs of previously opened files or add them to the interface's default families. diff --git a/docs/editing-existing-text.md b/docs/editing-existing-text.md index 426058f7b7..a8966ff207 100644 --- a/docs/editing-existing-text.md +++ b/docs/editing-existing-text.md @@ -1,5 +1,16 @@ # Editing existing text and source subsets +Entering **Edit text and images** prepares supported grouped Form XObjects on the visible +page in an undoable **Prepare grouped text** step. The group's original operators, clip, +placement and resources become page content; shared forms on other pages stay unchanged. +Headless callers use `text_prepare`, then list `text_paragraphs` and call `text_edit`. +This edits existing glyph operators; it adds neither a cover rectangle nor an Add Text box. +Preparation refuses knockout/blended groups, incompatible colour spaces, tagged forms, +recursive forms, unfinished paths and unreadable content. Decoding is capped at 16 MiB, +64 placements and eight expansion levels per page. Unsupported groups remain intact. +A narrowly fingerprinted legacy THSarabunPSK exporter map can also be repaired in copied page +fonts; the repair log records both preparation and character-map recovery. + Click a text region in **Edit a PDF → Edit text and images**. One editor contains its source rows and mixed styles. Embedded TrueType and OpenType/CFF faces are used in memory; source font programs, colour, size and baseline anchors stay intact. A textless preview @@ -44,6 +55,30 @@ this works only when that build input actually contains the full face. This chan add a font binary or change the release font pin. Fonts shared by releases belong in `storytold/craft-fonts` under the repository's asset rules. +**Keep source fonts** retains the existing Thai family, weight and point size. For legacy +THSarabunPSK subsets, PdfCraft reconstructs an editor-only Unicode cmap from the PDF's +Unicode and CID-to-glyph maps. HarfRust uses the retained Thai positional glyphs, and the +saved text refers to the original PDF font resources. Several subsets of the same exact +face can supply Thai and punctuation glyphs; Regular and Bold stay separate. Source font +programs are never modified or replaced. Text settings, including character/word spacing, +horizontal scale, rise, color and paint mode, remain in the PDF text state. + +A source subset that lacks a needed contextual glyph produces an error asking for the +complete original face or an explicit replacement. Cross-family replacement requires an +explicit choice. The document-font menu offers **Noto Sans Thai**: pass its `text_fonts` +resource (normally `builtin:thai`) as `text_edit.font_resource` or a range's `font_resource`. This replacement +requires the optional craft-fonts Thai input and currently uses Regular. A matching Noto +subset can still be completed by that same bundled family. + +The bundled variable Thai face is written as bounded Type 3 glyph outlines, with shaped +vowel/tone positions and logical Unicode text, rather than embedded as a static font. +Saved paragraphs can be reopened and edited again without installing a font. Modern Thai +shaping uses the source's GPOS mark positioning; GSUB substitution is optional, as specified by +[the OpenType Thai model](https://learn.microsoft.com/en-us/typography/script-development/thai). +Verified legacy Thai PUA fonts use positional substitutions instead of requiring GPOS. +Mixed contextual style runs and dictionary word segmentation remain limited. No font binary +or release pin changes are needed. + When mixed-style text needs outline fallback, characters using the same actual fallback face share Type 3 fonts, with at most 240 distinct glyphs per font. Adjacent characters with the same font and source style are written as one run. Latin characters that the source font can encode diff --git a/docs/roadmap.md b/docs/roadmap.md index 736d53bb7a..28afeea246 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,7 @@ # PdfCraft roadmap: milestones and what's next +> **Last reviewed:** 2026-11-05 | **Last updated:** 2026-11-05 | **Change:** patch (M7 #2964 / #3185: grouped source text and Thai font/style preservation; estimates unchanged) + > **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged) @@ -239,7 +241,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Refreshed against upstream `99941daa` (2026-10-15).
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions. XML/XFDF exports preserve tabs and line breaks in names and comment metadata. Plain AcroForm XML imports retain all direct text around comments and processing instructions, including CDATA and character references. CSV export and spreadsheet merging quote tabs, and import distinguishes quoted tabs from TSV delimiters without changing field identities.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Multi-select choice fields write ascending `/I` option indices while preserving `/V` selection order; synthetic forms/engine regressions cover reverse selection and save/reopen. CSV export preserves carriage returns in names and values. | -| M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Done since: Edit text preserves Arabic/Hebrew `/ActualText` typing order, with font-independent regressions; single-glyph RTL ligature spans (including lam-alef) join their surrounding word. Done since (#1116): editable text regions grouped across compatible words and subset fonts, one rich-text editor over a background with the source text removed, embedded sfnt faces reused as editor faces, and visible (clipped) bounds for cropped-image selection and resize. Exclusive crops now resize across nested states and content streams, with shared clips fixed.
Done since: #594 ordinary paragraph drags preserve overlapping source text; UI/engine regressions, native save/reopen, independent-reader checks and full CI passed in the Linux test environment. #1200 candidate: local GoToR links open in desktop tabs and through root-confined automation; appearance-only edits preserve existing actions. Named destinations remain unapplied and `/NewWindow` is a tab hint. Done since: #3187: Added-image automation normalizes extreme signed rotation angles before combining them with the current orientation; integer bounds, history, pixels, save/reopen and MCP have regressions. Done since: #2592: Styled fallback fonts batch up to 240 glyphs per actual face and retain source styles. Budget refusal and distinct same-BaseFont descriptor weights have mutation-checked regressions; coverage and complex mixed runs remain limited. Done since: #5657 page-mark operations are atomic across selected pages, including shared/inherited resources and existing unsaved edits. Missing: editing existing text and images in place (the longest pole), image/PDF watermarks PR #2155 candidate: editor-owned crops preserve original clips through group moves; Arrange preserves added-item stream ownership and refuses unsafe scope/state/tag extraction; image handles and flips follow page orientation; general transforms and Unicode script toggles have headless tools. Clipboard, mixed-size formatting, Home pins and reloadable installed-font discovery have review regressions. Latest follow-up uses explicit image-paste events, refreshes the mixed-size display, and documents script normalization with Undo. Further review covers inherited Form text and mask chronology, scoped transform composition, affine resize/rotation pivots, paste/selection ownership, no-op scripts and page-menu permissions. Done since: #5835 a right-click on an image in crop mode keeps the crop and opens no menu, also over overlapping and rotated images. | +| M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Done since: Edit text preserves Arabic/Hebrew `/ActualText` typing order, with font-independent regressions; single-glyph RTL ligature spans (including lam-alef) join their surrounding word. Done since (#1116): editable text regions grouped across compatible words and subset fonts, one rich-text editor over a background with the source text removed, embedded sfnt faces reused as editor faces, and visible (clipped) bounds for cropped-image selection and resize. Exclusive crops now resize across nested states and content streams, with shared clips fixed.
Done since: #594 ordinary paragraph drags preserve overlapping source text; UI/engine regressions, native save/reopen, independent-reader checks and full CI passed in the Linux test environment. #1200 candidate: local GoToR links open in desktop tabs and through root-confined automation; appearance-only edits preserve existing actions. Named destinations remain unapplied and `/NewWindow` is a tab hint. Done since: #3187: Added-image automation normalizes extreme signed rotation angles before combining them with the current orientation; integer bounds, history, pixels, save/reopen and MCP have regressions. Done since: #2592: Styled fallback fonts batch up to 240 glyphs per actual face and retain source styles. Budget refusal and distinct same-BaseFont descriptor weights have mutation-checked regressions; coverage and complex mixed runs remain limited. Done since: #5657 page-mark operations are atomic across selected pages, including shared/inherited resources and existing unsaved edits. Missing: editing existing text and images in place (the longest pole), image/PDF watermarks PR #2155 candidate: editor-owned crops preserve original clips through group moves; Arrange preserves added-item stream ownership and refuses unsafe scope/state/tag extraction; image handles and flips follow page orientation; general transforms and Unicode script toggles have headless tools. Clipboard, mixed-size formatting, Home pins and reloadable installed-font discovery have review regressions. Latest follow-up uses explicit image-paste events, refreshes the mixed-size display, and documents script normalization with Undo. Further review covers inherited Form text and mask chronology, scoped transform composition, affine resize/rotation pivots, paste/selection ownership, no-op scripts and page-menu permissions. Done since: #5835 a right-click on an image in crop mode keeps the crop and opens no menu, also over overlapping and rotated images. Thai editing (#2964, #3185): supported Form text is editable; legacy Thai retains source fonts and text settings. Cross-family replacement is explicit; unsafe groups and mixed contextual styles remain partial. | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Done since: Edit text preserves Arabic/Hebrew `/ActualText` typing order, with font-independent regressions. Done since (#1116): editable text regions grouped across compatible words and subset fonts, one rich-text editor over a background with the source text removed, embedded sfnt faces reused as editor faces, and visible (clipped) bounds for cropped-image selection and resize. Exclusive crops now resize across nested states and content streams, with shared clips fixed.
Done since: #594 ordinary paragraph drags preserve overlapping source text; UI/engine regressions, native save/reopen, independent-reader checks and full CI passed in the Linux test environment. #1200 candidate: local GoToR links open in desktop tabs and through root-confined automation; appearance-only edits preserve existing actions. Named destinations remain unapplied and `/NewWindow` is a tab hint. #1316: replacing a line or paragraph keeps a neighbouring line whose glyph box overlaps when the baselines differ; a second copy on the same baseline is still replaced. Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). #3212: Headers, Bates numbers, watermarks and added text stay visible after unfinished marked content, without revealing hidden content. Tracked editable text distinguishes repeated glyph spacing from word gaps, with synthetic paragraph/object regressions. Missing: editing existing text and images in place (the longest pole), watermark print/screen visibility and repeat | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | @@ -312,6 +314,7 @@ Estimates unchanged. | Date | Change | Summary | |---|---|---| +| 2026-11-05 | patch | M7 #2964 / #3185: grouped text and Thai source-font preservation; limitations and estimates unchanged. | | 2026-11-03 | minor | M11: Optimize's JBIG2 images share a symbol dictionary (`/JBIG2Globals`), up to 16 at a time and when smaller (`pdfcraft_filters::encode_jbig2_shared`): 41–46% smaller on rendered text in proportional fonts, 10% in Courier | | 2026-11-03 | patch | M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged. | | 2026-11-03 | patch | M5 #5850: Enter adds Fill & Sign line breaks; outside clicks commit, Escape cancels, and saved multiline appearances retain their text and height; estimates unchanged. | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 10b040e09f..54aa9561b5 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1894,9 +1894,9 @@ tier = "P0" milestone = "M7" status = "partial" commands = ["edit.edit_text"] -tools = ["text_lines", "text_paragraphs", "text_edit", "text_edit_layout", "text_font_files", "text_font_import", "text_fonts"] -evidence = ["crates/edit/src/tests.rs::paragraphs_are_found_and_rewrapped", "crates/edit/src/tests.rs::text_lines_are_found_and_replaced_in_place", "crates/ui-egui/tests/it/editing.rs::editing_existing_text_in_place", "crates/automation/tests/it/automation.rs::editing_existing_text_through_tools", "crates/edit/src/tests.rs::a_line_whose_operator_starts_the_next_stream_can_be_edited", "crates/automation/tests/it/automation.rs::a_line_split_across_content_streams_is_listed_and_edited", "crates/ui-egui/tests/it/editing.rs::selected_text_formatting_keeps_the_rest_of_the_paragraph_and_reopens", "crates/edit/src/tests.rs::marked_paragraph_boundaries_and_cross_stream_transforms_survive_rewrite", "crates/automation/tests/it/automation.rs::explicit_font_and_disjoint_ranges_save_reopen_through_tools", "crates/edit/src/tests.rs::disjoint_style_ranges_preserve_unselected_runs_and_validate_utf8_atomically", "crates/engine/tests/it/document_fonts.rs::opentype_cff_document_font_preserves_program_codes_and_rendering_after_reopen", "crates/ui-egui/tests/it/editing.rs::cropped_images_do_not_intercept_text_and_stay_inside_each_page", "crates/automation/tests/it/automation.rs::clipped_image_bounds_and_text_edit_save_reopen_through_tools", "crates/edit/src/tests.rs::mixed_styles_and_matrix_sizes_form_visual_regions_without_merging_columns", "crates/edit/src/tests.rs::rich_region_edits_preserve_styles_positions_and_delete_all_runs_after_reopen", "crates/ui-egui/tests/it/editing.rs::one_editor_contains_mixed_styles_and_visual_rows", "crates/ui-egui/tests/it/editing.rs::inline_editor_preserves_source_font_and_previews_live_bounds", "crates/fonts/src/pdf/tests.rs::mac_subset_cmap_uses_pdf_unicode_without_changing_source_program", "crates/automation/tests/it/automation.rs::mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools", "crates/ui-egui/tests/it/ui.rs::the_edit_tab_turns_on_edit_text_and_images", "crates/ui-egui/tests/it/editing.rs::the_caret_on_a_small_row_is_as_tall_as_that_row", "crates/edit/src/tests.rs::uzbek_and_kazakh_cyrillic_use_the_bundled_latin_face", "crates/automation/tests/it/automation.rs::editing_uzbek_and_kazakh_cyrillic_through_tools", "crates/edit/src/tests.rs::styled_paragraph_batches_cyrillic_without_changing_source_fonts", "crates/edit/src/tests.rs::styled_paragraph_over_240_fallback_glyphs_splits_and_refuses_atomically", "crates/edit/src/tests.rs::styled_fallback_batches_keep_face_choices_and_range_styles", "crates/edit/src/tests.rs::styled_fallback_shards_keep_text_advances_and_alignment", "crates/edit/src/tests.rs::styled_fallback_budget_is_enforced_before_objects_are_written", "crates/edit/src/tests.rs::styled_fallback_same_base_font_and_character_keep_both_weights", "crates/automation/tests/it/automation.rs::styled_cyrillic_fallback_is_batched_through_tools_and_reopen", "crates/ui-egui/tests/it/editing.rs::escape_in_an_open_paragraph_selects_its_box_for_delete"] -notes = "The Edit tab turns Edit text & images on at once, as in Acrobat (leaving the tab puts the tool down and applies an edit in progress); Esc in an open paragraph leaves its text and selects its box, so Delete removes it (the unapplied draft is discarded); click a paragraph box and retype it in place (font reused or Helvetica substituted, colour and spacing kept, rewrapped); a page's /Contents pieces are parsed joined, so a line whose operands and operator sit in different pieces is listed and edited (lines and paragraphs still don't join across pieces; Edit image still parses each piece alone); drag a box to move it or its right-edge handle to rewrap it (edit.text-move-resize); per-character formatting changes are next Since #1116: mixed-style visual regions retain source rows, subset encodings, colour and baseline anchors in one textless-background editor. Explicit paragraph/heading scopes constrain grouping; compatible graphics planes permit cross-stream regions. Full matching document fonts complete absent subset glyphs; installed/file selection, TrueType/OpenType-CFF embedding and disjoint UTF-8 range styles have save/reopen coverage. Single-face OpenType shaping/bidi retains logical text through ToUnicode/ActualText. Replacement text the paragraph's font can't show takes the craft-fonts Japanese faces when they draw all of it (Uzbek қ, Kazakh ә are in none of them), else the bundled Inter face, so those letters write in every build (#2593). Legacy non-SFNT preview, mixed-font/style contextual runs, full structure-tree preservation and author intent in untagged PDFs remain limited; Styled-path outline fallback batches by actual face with 240 glyphs per font and preserves source styles; adjacent equal-font/style characters form runs. At most 15360 distinct fallback glyphs per styled paragraph; missing glyphs are refused before fallback objects are written. Glyph coverage, mixed contextual shaping, single-style fallback and stale-font cleanup are unchanged; see docs/editing-existing-text.md" +tools = ["text_prepare", "text_lines", "text_paragraphs", "text_edit", "text_edit_layout", "text_font_files", "text_font_import", "text_fonts"] +evidence = ["crates/edit/src/tests.rs::editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen", "crates/fonts/src/pdf/tests.rs::composite_subset_without_cmap_uses_pdf_glyph_ids_in_editor_and_shaper_without_replacing_the_program", "crates/automation/tests/it/automation.rs::grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_glyphs", "crates/edit/src/tests.rs::grouped_text_preparation_keeps_unsafe_compositing_paths_and_malformed_forms_intact", "crates/automation/tests/it/automation.rs::explicit_bundled_thai_font_choice_layout_edit_undo_and_reopen", "crates/edit/src/tests.rs::existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font", "crates/edit/src/tests.rs::paragraphs_are_found_and_rewrapped", "crates/edit/src/tests.rs::text_lines_are_found_and_replaced_in_place", "crates/ui-egui/tests/it/editing.rs::editing_existing_text_in_place", "crates/automation/tests/it/automation.rs::editing_existing_text_through_tools", "crates/edit/src/tests.rs::a_line_whose_operator_starts_the_next_stream_can_be_edited", "crates/automation/tests/it/automation.rs::a_line_split_across_content_streams_is_listed_and_edited", "crates/ui-egui/tests/it/editing.rs::selected_text_formatting_keeps_the_rest_of_the_paragraph_and_reopens", "crates/edit/src/tests.rs::marked_paragraph_boundaries_and_cross_stream_transforms_survive_rewrite", "crates/automation/tests/it/automation.rs::explicit_font_and_disjoint_ranges_save_reopen_through_tools", "crates/edit/src/tests.rs::disjoint_style_ranges_preserve_unselected_runs_and_validate_utf8_atomically", "crates/engine/tests/it/document_fonts.rs::opentype_cff_document_font_preserves_program_codes_and_rendering_after_reopen", "crates/ui-egui/tests/it/editing.rs::cropped_images_do_not_intercept_text_and_stay_inside_each_page", "crates/automation/tests/it/automation.rs::clipped_image_bounds_and_text_edit_save_reopen_through_tools", "crates/edit/src/tests.rs::mixed_styles_and_matrix_sizes_form_visual_regions_without_merging_columns", "crates/edit/src/tests.rs::rich_region_edits_preserve_styles_positions_and_delete_all_runs_after_reopen", "crates/ui-egui/tests/it/editing.rs::one_editor_contains_mixed_styles_and_visual_rows", "crates/ui-egui/tests/it/editing.rs::inline_editor_preserves_source_font_and_previews_live_bounds", "crates/fonts/src/pdf/tests.rs::mac_subset_cmap_uses_pdf_unicode_without_changing_source_program", "crates/automation/tests/it/automation.rs::mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools", "crates/ui-egui/tests/it/ui.rs::the_edit_tab_turns_on_edit_text_and_images", "crates/ui-egui/tests/it/editing.rs::the_caret_on_a_small_row_is_as_tall_as_that_row", "crates/edit/src/tests.rs::uzbek_and_kazakh_cyrillic_use_the_bundled_latin_face", "crates/automation/tests/it/automation.rs::editing_uzbek_and_kazakh_cyrillic_through_tools", "crates/edit/src/tests.rs::styled_paragraph_batches_cyrillic_without_changing_source_fonts", "crates/edit/src/tests.rs::styled_paragraph_over_240_fallback_glyphs_splits_and_refuses_atomically", "crates/edit/src/tests.rs::styled_fallback_batches_keep_face_choices_and_range_styles", "crates/edit/src/tests.rs::styled_fallback_shards_keep_text_advances_and_alignment", "crates/edit/src/tests.rs::styled_fallback_budget_is_enforced_before_objects_are_written", "crates/edit/src/tests.rs::styled_fallback_same_base_font_and_character_keep_both_weights", "crates/automation/tests/it/automation.rs::styled_cyrillic_fallback_is_batched_through_tools_and_reopen", "crates/ui-egui/tests/it/editing.rs::escape_in_an_open_paragraph_selects_its_box_for_delete"] +notes = "Supported grouped Forms are prepared on entering Edit text, or via text_prepare, so their existing source glyphs can be edited directly. Unsafe groups remain intact. Narrow legacy THSarabunPSK character-map recovery is recorded. Legacy Thai source editing reuses original font programs, positional glyphs and same-face subsets, preserving Regular/Bold, point size, spacing and paint. A different Thai family requires explicit selection; the Noto replacement is Regular only; mixed contextual style runs remain limited. The Edit tab turns Edit text & images on at once, as in Acrobat (leaving the tab puts the tool down and applies an edit in progress); Esc in an open paragraph leaves its text and selects its box, so Delete removes it (the unapplied draft is discarded); click a paragraph box and retype it in place (font reused or Helvetica substituted, colour and spacing kept, rewrapped); a page's /Contents pieces are parsed joined, so a line whose operands and operator sit in different pieces is listed and edited (lines and paragraphs still don't join across pieces; Edit image still parses each piece alone); drag a box to move it or its right-edge handle to rewrap it (edit.text-move-resize); per-character formatting changes are next Since #1116: mixed-style visual regions retain source rows, subset encodings, colour and baseline anchors in one textless-background editor. Explicit paragraph/heading scopes constrain grouping; compatible graphics planes permit cross-stream regions. Full matching document fonts complete absent subset glyphs; installed/file selection, TrueType/OpenType-CFF embedding and disjoint UTF-8 range styles have save/reopen coverage. Single-face OpenType shaping/bidi retains logical text through ToUnicode/ActualText. Replacement text the paragraph's font can't show takes the craft-fonts Japanese faces when they draw all of it (Uzbek қ, Kazakh ә are in none of them), else the bundled Inter face, so those letters write in every build (#2593). Legacy non-SFNT preview, mixed-font/style contextual runs, full structure-tree preservation and author intent in untagged PDFs remain limited; Styled-path outline fallback batches by actual face with 240 glyphs per font and preserves source styles; adjacent equal-font/style characters form runs. At most 15360 distinct fallback glyphs per styled paragraph; missing glyphs are refused before fallback objects are written. Glyph coverage, mixed contextual shaping, single-style fallback and stale-font cleanup are unchanged; see docs/editing-existing-text.md" [[feature]] id = "edit.text-reflow" From 7634ac806457419c5a870da5f7a9ef9da709396e Mon Sep 17 00:00:00 2001 From: xullul Date: Sat, 7 Nov 2026 15:59:38 +0700 Subject: [PATCH 07/37] M7: address review of grouped Thai text preparation Add: - Report text grouped more than eight levels deep instead of leaving it silently grouped, and refuse forms whose resource-naming operators carry surplus operands, which would otherwise lose nested artwork when inlined. - Return the applied preparation's own counts from Session::prepare_text, so text_prepare no longer reports a separate preview's. - Shape a THSarabunPSK program that has its own Thai mark positioning by that layout, in its own codes. A span the subsets can't show in place is shaped whole, Latin rows included, by a complete program of the same face and style, such as a CFF face from Load font. Change: - Expand only forms that show text, directly or through nested forms; image- and vector-only forms stay grouped and uncounted. Match a Do's raw resource name bytes, and refuse RGB-blended groups on pages naming no blending space. - Check the recovered THSarabunPSK map and the Windows PUA alternates glyph by glyph against the subset's advances, leave dropped glyphs unmapped, and insert the repair after the last endcmap. - Shape a Thai run split across same-face subsets cluster by cluster and skip subset glyphs that are empty. Take the legacy path only for programs in the legacy glyph order, needing only the glyphs the new text uses. - Require GPOS mark/mkmk in the script and language system HarfRust selects. - Match same-face candidates by weight and slant from the name, descriptor weight or a non-zero ItalicAngle; the Italic flag alone is ignored, as exporters set it on upright faces. Never complete bold or italic text with the Regular bundled Thai face. - Name the missing character, or the unavailable builtin:thai choice, in errors. Chores: - Document text_prepare's fields, builtin:thai, the app's refusal reporting limits and the new routing; extend the text-edit parity evidence. Tests that need craft-fonts now say they skipped instead of returning silently. Validation: with and without craft-fonts, fonts 105 (+3 build-input), edit 465, engine 482 (+28 integration) and automation 50 + 352 passed; editor flows 248 passed. Formatting, strict Clippy, layers, assets and parity passed; the wasm check was not run locally. Each new regression failed with its fix reverted. Real-PDF edits on Royal Gazette 26288 kept their source fonts. --- ROADMAP.md | 2 +- crates/automation/README.md | 26 +- crates/automation/src/dispatch.rs | 7 +- crates/automation/src/documents.rs | 17 + crates/automation/src/tools.rs | 4 +- crates/automation/tests/it/automation.rs | 47 ++- crates/edit/src/added.rs | 2 + crates/edit/src/added/thai_tests.rs | 59 +++ crates/edit/src/tests.rs | 514 ++++++++++++++++++++++- crates/edit/src/text/forms.rs | 28 +- crates/edit/src/text/forms/expand.rs | 103 +++-- crates/edit/src/text_regions/styled.rs | 313 ++++++++++---- crates/engine/src/edit/run.rs | 5 + crates/engine/src/session/editing.rs | 17 +- crates/fonts/src/pdf.rs | 15 +- crates/fonts/src/pdf/thai.rs | 315 ++++++++++++-- crates/fonts/src/source_shaping.rs | 135 +++++- docs/editing-existing-text.md | 36 +- docs/roadmap.md | 2 +- parity/acrobat-features.toml | 4 +- 20 files changed, 1452 insertions(+), 199 deletions(-) create mode 100644 crates/edit/src/added/thai_tests.rs diff --git a/ROADMAP.md b/ROADMAP.md index 988be90914..a9087c188e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -232,7 +232,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log -- **2026-11-05 (M7, #2964 / #3185):** Supported Form text can be prepared for direct editing. Legacy Thai keeps the original font programs, Regular/Bold and text settings; cross-family replacement is explicit. Builds on #3067 and #2811. Unsafe groups and mixed contextual styles remain partial; estimates unchanged. +- **2026-11-07 (M7, #2964 / #3185):** Supported Form text can be prepared for direct editing; groups left grouped (unsafe compositing, text nested past eight levels) are reported. Legacy Thai keeps the original font programs, Regular/Bold and text settings, also across same-face subsets; the narrow THSarabunPSK map repair is checked glyph by glyph against the font program; cross-family replacement is explicit. Builds on #3067 and #2811. Unsafe groups and mixed contextual styles remain partial; estimates unchanged. - **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged. - **2026-11-03 (M11, shared JBIG2 dictionary):** With Monochrome Images set to JBIG2, Optimize has up to 16 JBIG2 images share one symbol dictionary (`/JBIG2Globals`), the letters found on more than one page stored once, when that is smaller (`pdfcraft_filters::encode_jbig2_shared`). Eight pages of 10 pt text rendered at 300 dpi take 41% less in Helvetica and 42% in Times than coded alone, 10% in Courier; groups of 16 save 45%, of 32 only 47%. Only images redaction can store again without the dictionary share one (image masks and DeviceGray named directly, at most 64 Mi pixels), and redacting one stores the rest of its group so, which the group bounds to 15. diff --git a/crates/automation/README.md b/crates/automation/README.md index 9c31754e0d..2f50b7394c 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -204,10 +204,28 @@ undoable and survive saving/reopening. `image_hit` shares the UI's curve and hol The bounds, limits and font requirements are in [existing-text editing](../../docs/editing-existing-text.md) and [image editing](../../docs/editing-images.md). -`text_prepare` makes supported grouped Form text editable on one page, as entering Edit text -in the GUI does. It keeps the group's original glyph operators and shared source objects, -records preparation/map repairs and is undoable. List `text_paragraphs` afterwards; its indices -belong to the prepared page. Unsafe compositing and malformed/tagged groups stay intact. +`text_fonts` also lists `builtin:thai` when the build includes the bundled Noto Sans Thai face +(the optional craft-fonts input). Passing it as `text_edit.font_resource` explicitly replaces a +paragraph or range with that face; the source font is never swapped for it silently. A build +without the face never lists it, and `text_edit` refuses it there. A page whose own fonts fill +the listing's 32 MiB preview budget can leave it out too; `text_edit` still accepts it then. + +### Preparing grouped text + +`text_prepare {doc, page}` makes supported grouped Form text editable on one page, as entering +Edit text in the GUI does. It keeps the group's original glyph operators and shared source +objects, records preparation and map repairs in the repair log, and is one undoable step. List +`text_paragraphs` afterwards; its indices belong to the prepared page. Besides the document +summary it returns what this edit did: + +| Field | Type | Meaning | +| --- | --- | --- | +| `prepared_groups` | integer | Form placements drawn in place, so their text is now the page's own. | +| `repaired_fonts` | integer | Page fonts whose damaged legacy THSarabunPSK Thai map was repaired on a page-owned copy. | +| `refused_groups` | array of strings | One `name: reason` per group left as it was: unsafe compositing, tags, malformed content, or text grouped more than eight levels deep. | + +It fails, changing nothing, when no group on the page can be prepared (the error gives the first +reason) or the page is out of range. ### Superscript and subscript normalization diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index 0d8c6e2333..ba2f7a2c0c 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -482,12 +482,7 @@ impl Automation { } "text_prepare" => { let page = self.page(&a)?; - let preparation = self.doc(&a)?.text_preparation(page).map_err(ToolError::Failed)?; - let mut result = self.apply(&a, Edit::PrepareText { page })?; - result["prepared_groups"] = json!(preparation.expanded); - result["repaired_fonts"] = json!(preparation.repaired_fonts); - result["refused_groups"] = json!(preparation.refused); - result + self.prepare_text(&a, page)? } "text_paragraphs" => { let page = self.page(&a)?; diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index f8c7988bbc..86fce61f33 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -444,6 +444,23 @@ impl Automation { pub(crate) fn apply(&mut self, a: &Args, edit: Edit) -> Result { let id = self.doc(a)?.id; self.session.apply(id, edit).map_err(failed)?; + self.applied(a) + } + + /// `text_prepare`: make a page's grouped text editable, with what that edit itself prepared, + /// repaired and left grouped. + pub(crate) fn prepare_text(&mut self, a: &Args, page: usize) -> Result { + let id = self.doc(a)?.id; + let prepared = self.session.prepare_text(id, page).map_err(failed)?; + let mut out = self.applied(a)?; + out["prepared_groups"] = json!(prepared.expanded); + out["repaired_fonts"] = json!(prepared.repaired_fonts); + out["refused_groups"] = json!(prepared.refused); + Ok(out) + } + + /// The document's summary after an edit, with what it stored as typed but drew with `?`. + fn applied(&mut self, a: &Args) -> Result { let doc = self.doc(a)?; let mut out = summary(doc); // Text the edit stored as typed but drew with `?` (its font can't draw it): said, not silent. diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index c58d803284..7bdefac89b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1581,10 +1581,10 @@ pub fn tools() -> Vec { t("text_font_import", "Load document font", "Load an explicitly selected, openly licensed full TrueType/OpenType face into a page. The program's family, embedding permissions and licence declaration are validated. Completes source subsets of the same face without replacing existing glyphs. Returns a font resource usable with text_edit.font_resource. Undoable; respects root confinement. No system-interface fallback is exported.") .cmd("edit.edit_text") .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}, "path":{"type":"string"}}), &["doc","page","path"])), - t("text_fonts", "List document fonts", "List validated embedded source faces and approved full preview faces for a page. Full registered resources and bundled choices can be selected by text_edit.font_resource. When available, builtin:thai explicitly selects bundled Noto Sans Thai for a paragraph or range; preview_only faces cannot be selected.") + t("text_fonts", "List document fonts", "List validated embedded source faces and approved full preview faces for a page. Full registered resources and bundled choices can be selected by text_edit.font_resource. builtin:thai explicitly replaces a paragraph or range with the bundled Noto Sans Thai; a build without that face never lists it and text_edit refuses it there (a page whose fonts fill the listing budget may also omit it, but text_edit still accepts it). preview_only faces cannot be selected.") .ro().cmd("edit.edit_text") .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}}), &["doc","page"])), - t("text_prepare", "Prepare grouped text", "Make existing text inside supported Form XObjects directly editable on one page. Replaces placements with their original operators and resources, preserving position, clipping and other pages. Refuses unsafe transparency, tags or malformed content. Adds no text or cover boxes. Returns prepared_groups, repaired_fonts and refused_groups. Undoable; list text_paragraphs again afterwards.") + t("text_prepare", "Prepare grouped text", "Make existing text inside supported Form XObjects directly editable on one page. Replaces placements with their original operators and resources, preserving position, clipping and other pages. Refuses unsafe transparency, tags or malformed content. Adds no text or cover boxes. Returns prepared_groups (placements made editable), repaired_fonts (damaged legacy Thai maps repaired) and refused_groups ('name: reason' for each group left as it was). Fails when nothing on the page can be prepared. Undoable; list text_paragraphs again afterwards.") .cmd("edit.edit_text") .with(schema(json!({"doc":doc(), "page":{"type":"integer","minimum":1}}), &["doc","page"])), t("text_paragraphs", "List paragraphs", "The paragraphs on a page (adjacent source runs grouped into aligned visual rows, including mixed fonts, sizes and colours): number, text, editor_text (visual rows separated by newlines), rows and source line numbers, box (top-left-origin points), font and size. Use the number with text_edit's paragraph: until you list the page again, a number names the paragraph listed here, even after edits regroup the page.") diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 21c958b52e..493a34899e 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -11458,12 +11458,34 @@ fn explicit_bundled_thai_font_choice_layout_edit_undo_and_reopen() { ok(&mut a, "doc_save", json!({"doc":doc,"path":"edited.pdf"})); let reopened = ok(&mut a, "doc_open", json!({"path":"edited.pdf"}))["doc"].as_u64().unwrap(); assert_eq!(page_text(&mut a, reopened), edited); - assert!(a.call("text_edit", &json!({"doc":reopened,"page":1,"paragraph":1,"text":"น้ำ 😀","font_resource":"builtin:thai"})).is_err()); + let Err(ToolError::Failed(m)) = + a.call("text_edit", &json!({"doc":reopened,"page":1,"paragraph":1,"text":"น้ำ 😀","font_resource":"builtin:thai"})) + else { + panic!("Noto Sans Thai has no emoji") + }; + assert!(m.contains("Noto Sans Thai") && m.contains("U+1F600"), "the error names the font and the character: {m}"); assert_eq!(page_text(&mut a, reopened), edited); let image = a.call("page_render", &json!({"doc":reopened,"page":1,"dpi":144})).unwrap(); assert!(image.iter().any(|c| matches!(c, Content::Png { .. }))); } +/// `text_prepare` refuses what it can't act on, changing nothing: a page with no grouped text +/// (with the reason), and a page the document doesn't have. +#[test] +fn text_prepare_refuses_pages_without_grouped_text() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + std::fs::write(dir.join("source.pdf"), fixture(1)).unwrap(); + let mut a = auto(dir); + let doc = ok(&mut a, "doc_open", json!({"path":"source.pdf"}))["doc"].as_u64().unwrap(); + let Err(ToolError::Failed(m)) = a.call("text_prepare", &json!({"doc":doc,"page":1})) else { panic!("nothing to prepare") }; + assert!(m.contains("no supported grouped text"), "{m}"); + let Err(ToolError::InvalidArgs(m)) = a.call("text_prepare", &json!({"doc":doc,"page":2})) else { panic!("no page 2") }; + assert!(m.contains("out of range"), "{m}"); + assert!(a.call("edit_undo", &json!({"doc":doc})).is_err(), "nothing was recorded to undo"); + assert_eq!(ok(&mut a, "doc_info", json!({"doc":doc}))["document"]["dirty"], false); +} + #[test] fn grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_glyphs() { use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; @@ -11518,14 +11540,25 @@ fn grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_gly let id = ok(&mut a, "doc_open", json!({"path":"source.pdf"}))["doc"].as_u64().unwrap(); let before = a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(); assert_eq!(ok(&mut a, "text_paragraphs", json!({"doc":id,"page":1}))["count"], 0); - ok(&mut a, "text_prepare", json!({"doc":id,"page":1})); + let prepared = ok(&mut a, "text_prepare", json!({"doc":id,"page":1})); + assert_eq!( + [&prepared["prepared_groups"], &prepared["repaired_fonts"], &prepared["refused_groups"]], + [&json!(1), &json!(0), &json!([])], + "{prepared}" + ); assert_eq!(a.call("page_render", &json!({"doc":id,"page":1,"dpi":72})).unwrap(), before, "preparing must not change pixels"); assert_eq!(ok(&mut a, "text_paragraphs", json!({"doc":id,"page":1}))["paragraphs"][0]["text"], old); - let args = json!({"doc":id,"page":1,"paragraph":1,"text":new,"font_resource":"builtin:thai"}); - if pdfcraft_fonts::document_thai_font().is_none() { - assert!(a.call("text_edit", &args).is_err()); - return; - } + // The bundled Thai face is an optional build input. Without it, `builtin:thai` is refused and + // the grouped edit, undo, redo and save checks run on Latin text in Helvetica instead. + let thai = json!({"doc":id,"page":1,"paragraph":1,"text":new,"font_resource":"builtin:thai"}); + let (new, args) = if pdfcraft_fonts::document_thai_font().is_some() { + (new, thai) + } else { + let Err(ToolError::Failed(m)) = a.call("text_edit", &thai) else { panic!("builtin:thai needs the bundled Thai face") }; + assert!(m.contains("builtin:thai"), "{m}"); + let latin = "Edited body"; + (latin, json!({"doc":id,"page":1,"paragraph":1,"text":latin,"font":"helvetica"})) + }; ok(&mut a, "text_edit", args); assert_eq!(ok(&mut a, "text_lines", json!({"doc":id,"page":1}))["lines"][0]["text"].as_str().unwrap().trim(), new); assert_eq!(page_text(&mut a, id)[1], old, "a shared form's other page must not change"); diff --git a/crates/edit/src/added.rs b/crates/edit/src/added.rs index 4abe4a02eb..43ca9de0bc 100644 --- a/crates/edit/src/added.rs +++ b/crates/edit/src/added.rs @@ -88,3 +88,5 @@ fn align_from(code: Option) -> Align { #[cfg(test)] mod arabic_tests; +#[cfg(test)] +mod thai_tests; diff --git a/crates/edit/src/added/thai_tests.rs b/crates/edit/src/added/thai_tests.rs new file mode 100644 index 0000000000..87d58a8e69 --- /dev/null +++ b/crates/edit/src/added/thai_tests.rs @@ -0,0 +1,59 @@ +//! Existing Thai text drawn from the bundled Thai outlines (`thai_glyph_resources`): one Type 3 +//! code per distinct glyph and cluster text, in fonts of `GLYPHS_PER_FONT` codes each. The +//! outline checks need the craft-fonts Thai face; the budget check runs without it. + +use pdfcraft_cos::{Dict, Document}; +use pdfcraft_fonts::source_shaping::{ShapedGlyph, ShapedRun}; + +use super::*; + +/// `count` copies of `glyph`, each with cluster text of its own (so each needs a code of its own). +fn distinct(glyph: &ShapedGlyph, count: usize) -> ShapedRun { + let glyphs = (0..count).map(|i| ShapedGlyph { position: [i as f64, 0.0], text: format!("{i}"), ..glyph.clone() }).collect(); + ShapedRun { glyphs, advance: count as f64 } +} + +#[test] +fn thai_glyph_resources_refuse_a_row_past_the_glyph_budget() { + let glyph = ShapedGlyph { id: Default::default(), position: [0.0, 0.0], text: String::new() }; + let over = ShapedRun { glyphs: vec![glyph; 65_535], advance: 0.0 }; + let (mut doc, mut fonts) = (Document::new_empty(), Dict::new()); + assert!(thai_glyph_resources(&mut doc, &mut fonts, &over).is_err()); + assert!(fonts.is_empty(), "a refused row adds no font"); +} + +/// Codes fill a font up to `GLYPHS_PER_FONT`, the next distinct cluster starts another font at +/// code 1, and the same glyph with the same text again reuses its code. One glyph drawn for two +/// cluster texts takes two codes, since each code's ToUnicode gives its own text. +#[test] +fn thai_glyph_resources_shard_distinct_clusters_at_the_font_boundary() { + let Some(face) = pdfcraft_fonts::document_thai_font() else { + eprintln!( + "skipping thai_glyph_resources_shard_distinct_clusters_at_the_font_boundary: built without craft-fonts (set CRAFT_FONTS_DIR to run it)" + ); + return; + }; + let ko = pdfcraft_fonts::source_shaping::shape(face.bytes, "ก").unwrap().glyphs.remove(0); + for (clusters, fonts_used) in [(GLYPHS_PER_FONT - 1, 1), (GLYPHS_PER_FONT, 1), (GLYPHS_PER_FONT + 1, 2)] { + let mut shaped = distinct(&ko, clusters); + shaped.glyphs.push(shaped.glyphs[0].clone()); + let (mut doc, mut fonts) = (Document::new_empty(), Dict::new()); + let placed = thai_glyph_resources(&mut doc, &mut fonts, &shaped).unwrap(); + assert_eq!((fonts.len(), placed.len()), (fonts_used, clusters + 1), "{clusters} clusters"); + assert_eq!(placed.last(), placed.first(), "a repeated glyph and text reuses its code"); + let codes: std::collections::HashSet<_> = placed.iter().collect(); + assert_eq!(codes.len(), clusters, "every distinct cluster has a code of its own"); + assert_eq!(placed[GLYPHS_PER_FONT - 2].1, [(GLYPHS_PER_FONT - 1) as u8]); + if clusters > GLYPHS_PER_FONT { + assert_eq!(placed[GLYPHS_PER_FONT - 1].1, [GLYPHS_PER_FONT as u8]); + assert_eq!(placed[GLYPHS_PER_FONT].1, [1], "the next font starts at code 1"); + assert_ne!(placed[GLYPHS_PER_FONT].0, placed[0].0); + } + } + // At the budget: one cluster repeated 65 534 times needs one code. + let mut row = distinct(&ko, 1); + row.glyphs = vec![row.glyphs[0].clone(); 65_534]; + let (mut doc, mut fonts) = (Document::new_empty(), Dict::new()); + let placed = thai_glyph_resources(&mut doc, &mut fonts, &row).unwrap(); + assert_eq!((fonts.len(), placed.len()), (1, 65_534)); +} diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index d8f0ca16ee..e425d00328 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -10753,7 +10753,13 @@ fn explicit_builtin_thai_replaces_an_unavailable_source_without_installing_a_fon #[test] fn existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font() { - let Some(_) = pdfcraft_fonts::document_thai_font() else { return }; + // The matching full face is the optional craft-fonts input; CI builds with it. + if pdfcraft_fonts::document_thai_font().is_none() { + eprintln!( + "skipping existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font: built without craft-fonts (set CRAFT_FONTS_DIR to run it)" + ); + return; + } // Synthetic source subset: only one Thai letter is encoded. No binary fixture or font // is committed; the complete, openly licensed matching face is the optional build input. let mut doc = text_page("BT /F1 16 Tf 50 700 Td (A) Tj ET BT /F2 12 Tf 50 600 Td (ab) Tj ET"); @@ -10781,6 +10787,35 @@ fn existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_fon assert_eq!(text::text_blocks(&reopen(&doc), 0).unwrap()[0].editor_text, next); } +/// The bundled Thai face is Regular: a subset of the same family that its descriptor marks bold +/// (a variable-font instance keeps the family name) is not completed with it silently. The +/// edit is refused, unchanged, until a replacement is chosen explicitly. +#[test] +fn a_bold_thai_subset_is_not_completed_with_the_regular_bundled_face() { + if pdfcraft_fonts::document_thai_font().is_none() { + eprintln!( + "skipping a_bold_thai_subset_is_not_completed_with_the_regular_bundled_face: built without craft-fonts (set CRAFT_FONTS_DIR to run it)" + ); + return; + } + let mut doc = text_page("BT /F1 16 Tf 50 700 Td (A) Tj ET"); + let cmap = + doc.add(Object::Stream(Stream::flate(Dict::new(), b"1 begincodespacerange <00> endcodespacerange 1 beginbfchar <41> <0E19> endbfchar"))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontWeight".to_vec(), Object::Int(700)); + doc.update_dict(pdfcraft_cos::ObjRef::new(5, 0), |f| { + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+NotoSansThai")); + f.set(b"ToUnicode".to_vec(), Object::Ref(cmap)); + f.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + }) + .unwrap(); + assert!(text::text_lines(&doc, 0).unwrap()[0].bold, "the descriptor makes the source bold"); + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let error = text::rewrite_block(&mut doc, 0, 0, Some("ผู้ใช้"), &text::BlockStyle::default()).expect_err("no silent Regular completion"); + assert!(!error.to_string().is_empty()); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before, "a refused edit changes nothing"); +} + #[test] fn grouped_source_text_preparation_edits_original_operators_without_changing_shared_forms() { let mut doc = page_with_forms("q 1 0 0 1 12 20 cm /Body Do Q", &[("Body".into(), "BT /F1 14 Tf 40 500 Td (Original body) Tj ET".into())]); @@ -10833,17 +10868,454 @@ fn grouped_text_preparation_keeps_unsafe_compositing_paths_and_malformed_forms_i let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); let result = text::forms::prepare(&mut doc, 0).unwrap(); assert_eq!(result.expanded, 0, "unsafe input {page_ops}, {form_ops}, {extra}"); - assert!(!result.refused.is_empty()); + let [refused] = result.refused.as_slice() else { panic!("one refusal for {page_ops}, {form_ops}, {extra}: {:?}", result.refused) }; + assert!(refused.starts_with("Body: "), "the refusal names the group: {refused}"); assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before); } } +/// A form whose content gives an operator a surplus operand (`0 /Inner Do`) is left grouped. The +/// renderer draws the last operand, while the inlined copy's resources are renamed at a fixed +/// position: the inner artwork would lose its resource and silently vanish. #[test] -fn editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen() { - // Remap existing OFL Inter outlines to synthetic Thai glyph slots; no proprietary font - // fixture is needed. Removing layout/cmap tables reproduces legacy PDF font subsetting. +fn grouped_text_with_surplus_resource_operands_is_left_grouped() { + let form = |body: &str| { + stream("/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> /XObject << /Inner 7 0 R >> >>", body) + }; + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> /XObject << /Body 6 0 R >> >> /Contents 4 0 R >>" + .to_string(), + stream("", "/Body Do"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + form("BT /F1 14 Tf 40 500 Td (body) Tj ET 0 /Inner Do"), + stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10] /Resources << >>", "0 0 10 10 re f"), + ]; + let mut doc = build(&objs.iter().map(String::as_str).collect::>()); + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 0); + assert!(result.refused.iter().any(|r| r.starts_with("Body: ")), "{:?}", result.refused); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before); + // The same form with well-formed operands is prepared, its inner artwork renamed with it. + let objs: Vec = + objs.iter().enumerate().map(|(i, o)| if i == 5 { form("BT /F1 14 Tf 40 500 Td (body) Tj ET /Inner Do") } else { o.clone() }).collect(); + let mut doc = build(&objs.iter().map(String::as_str).collect::>()); + assert_eq!(text::forms::prepare(&mut doc, 0).unwrap().expanded, 1); + let page = &pdfcraft_model::pages(&doc)[0]; + let xobjects = page.dict.get(b"Resources").and_then(|r| doc.dict(r)).and_then(|r| r.get(b"XObject").and_then(|x| doc.dict(x))).unwrap(); + for op in pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops.iter().filter(|op| op.is("Do")) { + assert!(xobjects.contains(op.last_name().unwrap()), "every drawn name resolves: {op:?}"); + } +} + +/// Text grouped deeper than preparation opens is reported, not passed over: of nine nested +/// forms, eight are opened and the ninth, which holds the text, is named in the refusals. +#[test] +fn grouped_text_nested_past_the_level_limit_is_reported() { + let forms: Vec<(String, String)> = (1..=9) + .map(|i| (format!("Body{i}"), if i < 9 { format!("/Body{} Do", i + 1) } else { "BT /F1 14 Tf 40 500 Td (deep) Tj ET".into() })) + .collect(); + let mut doc = page_with_forms("/Body1 Do", &forms); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 8); + assert_eq!(result.refused, ["Body9: its text is grouped more than 8 levels deep"]); + // Eight levels are still opened completely: the ninth, one level less deep, is prepared. + let mut doc = page_with_forms("/Body2 Do", &forms); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!((result.expanded, result.refused.len()), (8, 0)); + assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "deep")); +} + +/// A form that draws only an image (itself, or through a form it draws) is artwork: it is +/// neither expanded nor counted. Text reached through a nested form still is. +#[test] +fn grouped_image_artwork_is_not_prepared() { + let image = "q 10 0 0 10 0 0 cm /Im Do Q"; + for forms in [vec![("Logo", image)], vec![("Logo", "/Inner Do"), ("Inner", image)]] { + let forms: Vec<(String, String)> = forms.into_iter().map(|(n, b)| (n.to_string(), b.to_string())).collect(); + let mut doc = page_with_forms("/Logo Do", &forms); + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!((result.expanded, result.refused.len()), (0, 0), "{forms:?}"); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before, "{forms:?}"); + } + let forms = [("Logo".to_string(), "/Inner Do".to_string()), ("Inner".to_string(), "BT /F1 14 Tf 40 500 Td (inner) Tj ET".to_string())]; + let mut doc = page_with_forms("/Logo Do", &forms); + assert_eq!(text::forms::prepare(&mut doc, 0).unwrap().expanded, 2); + assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "inner")); +} + +/// The `Do` is matched to its resource by the name's bytes as written, which need not be UTF-8. +#[test] +fn grouped_text_under_a_non_utf8_resource_name_is_prepared() { + let mut doc = page_with_forms("/F#FF Do", &[("F#FF".into(), "BT /F1 14 Tf 40 500 Td (named) Tj ET".into())]); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!((result.expanded, result.refused.len()), (1, 0), "{:?}", result.refused); + assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "named")); +} + +/// A transparency group that blends in DeviceRGB composites as the page does only on a page +/// whose own group says DeviceRGB too. On a page that names no blending space the processor +/// chooses one (often CMYK in print), so inlining could change colours: that group is refused. +#[test] +fn grouped_text_in_an_rgb_group_needs_a_page_that_blends_in_rgb() { + for page_group in [false, true] { + let mut doc = page_with_forms("/Body Do", &[("Body".into(), "0 0 0 1 k BT /F1 14 Tf 40 500 Td (body) Tj ET".into())]); + let group = || build(&["<< /S /Transparency /CS /DeviceRGB >>"]).get(pdfcraft_cos::ObjRef::new(1, 0)).as_dict().unwrap().clone(); + let Object::Stream(mut s) = doc.get(pdfcraft_cos::ObjRef::new(7, 0)).as_ref().clone() else { panic!("fixture") }; + s.dict.set(b"Group".to_vec(), Object::Dict(group())); + doc.set(pdfcraft_cos::ObjRef::new(7, 0), Object::Stream(s)); + if page_group { + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |d| d.set(b"Group".to_vec(), Object::Dict(group()))).unwrap(); + } + let result = text::forms::prepare(&mut doc, 0).unwrap(); + if page_group { + assert_eq!((result.expanded, result.refused.len()), (1, 0), "{:?}", result.refused); + } else { + assert_eq!(result.expanded, 0); + assert!(result.refused.iter().any(|r| r.contains("colour space of its own")), "{:?}", result.refused); + } + } +} + +/// The damaged THSarabunPSK map is repaired with entries after the last `endcmap`: a stream with +/// a second block that repeats corrupt destinations must not override the corrective entries. +#[test] +fn grouped_legacy_thai_map_repair_follows_every_block_of_the_damaged_map() { + let damaged = b"begincmap 1 begincodespacerange <0000> endcodespacerange \ + 1 beginbfrange <00ff><01ff><0032> endbfrange 17 beginbfrange \ + <00d9><00d9><0e04> <00e2><00e2><0e0d> <00fa><00fa><0e25> <0100><0100><0e2b> <0102><0102><0e2d> \ + <0164><0164><0e4c> <0166><0166><0e31> <0168><0168><0e47> \ + <00d6><00d6><000f> <00f0><00f0><0029> <00f8><00f8><0023> <0105><0105><003e> <0107><0107><0040> \ + <0109><0109><003e> <0111><0111><004e> <0160><0160><004e> <0161><0161><004f> endbfrange endcmap\n\ + begincmap 2 beginbfchar <0103> <0036> <0107> <0040> endbfchar endcmap"; + // ฮา: codes 259 and 263, which the second block maps to "6@" again. + let mut doc = page_with_forms("/Body Do", &[("Body".into(), "BT /F1 14 Tf 40 500 Td <01030107> Tj ET".into())]); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &legacy_thai_program(), damaged); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!((result.expanded, result.repaired_fonts), (1, 1)); + let lines = text::text_lines(&doc, 0).unwrap(); + assert!(lines.iter().any(|l| l.text == "ฮา"), "{:?}", lines.iter().map(|l| &l.text).collect::>()); +} + +/// A Thai run whose glyphs a subsetter split across two subsets of the face is kept in that +/// face: each cluster comes from a subset that has it. A cluster no subset has is refused with +/// the reason, and the document is unchanged. +#[test] +fn editing_legacy_thai_shapes_a_run_across_subsets_of_the_same_face() { + // F1 lacks ค (glyph 217), F2 lacks ข (215); "ขค" needs both. + let mut without_kho_khwai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khwai, 217); + let mut without_kho_khai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khai, 215); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let first = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &without_kho_khwai, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(first)); + let second = legacy_thai_font(&mut doc, "GHIJKL+THSarabunPSK", &without_kho_khai, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(second)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).expect("the face's subsets together have every glyph"); + assert_eq!(edited.substituted, None); + let doc = reopen(&doc); + let lines = text::text_lines(&doc, 0).unwrap(); + assert_eq!(lines.iter().map(|l| l.text.as_str()).collect::(), "กขค"); + let shown = shown_codes(&doc); + assert!(shown.contains(&("F1".into(), vec![0x00, 0xd6])) && shown.contains(&("F1".into(), vec![0x00, 0xd7])), "ก and ข from F1: {shown:?}"); + assert!(shown.contains(&("F2".into(), vec![0x00, 0xd9])), "ค from the subset that has it, F2: {shown:?}"); + assert_eq!(page_font_programs(&doc).iter().filter(|(_, program)| program.is_some()).count(), 2, "no font program is added"); + // ฆ (glyph 219) is in no subset of the face. + let mut neither = legacy_thai_program(); + drop_sfnt_glyph(&mut neither, 219); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let only = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &neither, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(only)); + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let error = text::rewrite_block(&mut doc, 0, 0, Some("กฆ"), &style).unwrap_err().to_string(); + assert!(error.contains("THSarabunPSK") && error.contains("load the complete original font"), "{error}"); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before, "a refused edit changes nothing"); +} + +/// A THSarabunPSK-named subset that has OpenType Thai mark positioning is a modern face: it is +/// shaped with its own layout like any source font, not by the legacy shaper (which would ask +/// for Windows PUA alternates such a face doesn't need). +#[test] +fn a_thai_subset_with_mark_positioning_is_shaped_with_its_own_layout() { let mut program = include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec(); let count = u16::from_be_bytes(program[4..6].try_into().unwrap()) as usize; + let records: Vec<_> = program[12..12 + count * 16].as_chunks::<16>().0.iter().filter(|r| &r[..4] != b"cmap").map(|r| r.to_vec()).collect(); + program[4..6].copy_from_slice(&(records.len() as u16).to_be_bytes()); + for (index, record) in records.iter().enumerate() { + program[12 + index * 16..28 + index * 16].copy_from_slice(record); + } + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &program, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("ปู่"), &style).expect("its own layout positions the marks"); + assert_eq!(edited.substituted, None); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().iter().map(|l| l.text.as_str()).collect::(), "ปู่"); + assert!(shown_codes(&doc).iter().all(|(font, _)| font == "F1"), "drawn in the source resource's own codes: {:?}", shown_codes(&doc)); +} + +/// A legacy subset beside a complete face of the same family that has its own Thai mark +/// positioning (as Load font adds): what the subset lacks comes from the complete face, in its +/// codes, rather than the edit being refused. +#[test] +fn a_legacy_subset_is_completed_by_a_positioned_face_of_the_same_family() { + let mut without_kho_khwai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khwai, 217); + let mut complete = include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec(); + let count = u16::from_be_bytes(complete[4..6].try_into().unwrap()) as usize; + let records: Vec<_> = complete[12..12 + count * 16].as_chunks::<16>().0.iter().filter(|r| &r[..4] != b"cmap").map(|r| r.to_vec()).collect(); + complete[4..6].copy_from_slice(&(records.len() as u16).to_be_bytes()); + for (index, record) in records.iter().enumerate() { + complete[12 + index * 16..28 + index * 16].copy_from_slice(record); + } + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let legacy = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &without_kho_khwai, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(legacy)); + let full = legacy_thai_font(&mut doc, "THSarabunPSK", &complete, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(full)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).expect("the complete face has ค"); + assert_eq!(edited.substituted, None); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().iter().map(|l| l.text.as_str()).collect::(), "กขค"); + assert!(shown_codes(&doc).contains(&("F2".into(), vec![0x00, 0xd9])), "ค from the complete face: {:?}", shown_codes(&doc)); +} + +/// `program` with its `cmap` replaced by one mapping Thai to the legacy THSarabunPSK glyph order +/// (U+0E01–U+0E3A to glyphs 214–271, U+0E3F–U+0E5B to 272–298), as a complete face's own does, +/// and A–Z to glyphs 36–61 (any outlines do: only what the program covers matters here). +fn with_legacy_thai_cmap(mut program: Vec) -> Vec { + let groups = [(0x41u32, 0x5au32, 36u32), (0x0e01, 0x0e3a, 214), (0x0e3f, 0x0e5b, 272)]; + let mut cmap: Vec = [0u16, 1, 3, 10].iter().flat_map(|v| v.to_be_bytes()).collect(); + cmap.extend(12u32.to_be_bytes()); + cmap.extend(12u16.to_be_bytes()); + cmap.extend(0u16.to_be_bytes()); + for value in [16 + 12 * groups.len() as u32, 0, groups.len() as u32] { + cmap.extend(value.to_be_bytes()); + } + for (first, last, glyph) in groups { + cmap.extend([first, last, glyph].iter().flat_map(|v| v.to_be_bytes())); + } + program.resize(program.len().next_multiple_of(4), 0); + let count = u16::from_be_bytes(program[4..6].try_into().unwrap()) as usize; + let record = (0..count).map(|i| 12 + i * 16).find(|&r| &program[r..r + 4] == b"cmap").unwrap(); + let at = program.len() as u32; + program[record + 8..record + 12].copy_from_slice(&at.to_be_bytes()); + program[record + 12..record + 16].copy_from_slice(&(cmap.len() as u32).to_be_bytes()); + program.extend(cmap); + program +} + +/// A legacy subset beside a complete face of the same family that can't be reused in its +/// existing codes (its encoding is a CMap stream, as Load font writes for a CFF face): the span +/// the subset can't show is shaped by the complete program with its own layout, all of it in +/// that one face, instead of being refused. +#[test] +fn a_legacy_subset_is_completed_by_a_complete_program_it_cannot_reuse_in_place() { + let mut without_kho_khwai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khwai, 217); + let complete = with_legacy_thai_cmap(include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec()); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let legacy = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &without_kho_khwai, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(legacy)); + let mut full = legacy_thai_font(&mut doc, "THSarabunPSK", &complete, LEGACY_THAI_UNICODE); + let encoding = doc.add(Object::Stream(Stream::flate( + Dict::new(), + b"begincmap 1 begincodespacerange <0000> endcodespacerange 1 begincidrange <0000> 0 endcidrange endcmap", + ))); + full.set(b"Encoding".to_vec(), Object::Ref(encoding)); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(full)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).expect("the complete program shapes what the subset lacks"); + assert_eq!(edited.substituted, None); + let doc = reopen(&doc); + let lines = text::text_lines(&doc, 0).unwrap(); + assert_eq!(lines.iter().map(|l| l.text.as_str()).collect::(), "กขค"); + assert!( + lines.iter().all(|l| pdfcraft_fonts::embedding::face_key(&l.base_font) == "thsarabunpsk"), + "the same face throughout: {:?}", + lines.iter().map(|l| &l.base_font).collect::>() + ); + assert!(shown_codes(&doc).iter().all(|(font, _)| font != "F1"), "no cluster split from the subset: {:?}", shown_codes(&doc)); +} + +/// The complete program that takes over a span shapes all of its rows, a Latin-only one too, +/// even where its existing PDF encoding has no Latin; and it must have the source's own style: +/// a Regular program never completes text that the subset's descriptor marks bold. +#[test] +fn a_complete_program_takes_the_whole_span_only_in_the_source_style() { + let fixture = |bold: bool| { + let mut without_kho_khwai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khwai, 217); + let complete = with_legacy_thai_cmap(include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec()); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let mut legacy = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &without_kho_khwai, LEGACY_THAI_UNICODE); + if bold { + let Some(Object::Array(mut descendants)) = legacy.get(b"DescendantFonts").cloned() else { panic!("fixture") }; + let Some(Object::Dict(descendant)) = descendants.first_mut() else { panic!("fixture") }; + let Some(Object::Dict(mut descriptor)) = descendant.get(b"FontDescriptor").cloned() else { panic!("fixture") }; + descriptor.set(b"FontWeight".to_vec(), Object::Int(700)); + descendant.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + legacy.set(b"DescendantFonts".to_vec(), Object::Array(descendants)); + } + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(legacy)); + let mut full = legacy_thai_font(&mut doc, "THSarabunPSK", &complete, LEGACY_THAI_UNICODE); + let encoding = doc.add(Object::Stream(Stream::flate( + Dict::new(), + b"begincmap 1 begincodespacerange <0000> endcodespacerange 1 begincidrange <0000> 0 endcidrange endcmap", + ))); + full.set(b"Encoding".to_vec(), Object::Ref(encoding)); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(full)); + doc + }; + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let mut doc = fixture(false); + text::rewrite_block(&mut doc, 0, 0, Some("กขค\nABC"), &style).expect("every row shaped by the complete program"); + let doc = reopen(&doc); + assert_eq!(text::text_blocks(&doc, 0).unwrap()[0].editor_text, "กขค\nABC"); + assert!(shown_codes(&doc).iter().all(|(font, _)| font != "F1" && font != "F2"), "{:?}", shown_codes(&doc)); + let mut doc = fixture(true); + assert!(text::text_lines(&doc, 0).unwrap()[0].bold, "the descriptor makes the subset bold"); + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + assert!(text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).is_err(), "no Regular program for bold text"); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before); +} + +/// Exporters set the descriptor's Italic flag on upright faces (seen on a THSarabunPSK header +/// subset: `/Flags 96`, `/ItalicAngle 0`). Such a subset is still the same style as its upright +/// siblings, so they complete what it lacks; a real slant (`/ItalicAngle -12`) is not. +#[test] +fn an_italic_flag_alone_does_not_separate_subsets_of_one_face() { + for (angle, completes) in [(0, true), (-12, false)] { + let mut without_kho_khwai = legacy_thai_program(); + drop_sfnt_glyph(&mut without_kho_khwai, 217); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let mut flagged = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &without_kho_khwai, LEGACY_THAI_UNICODE); + let Some(Object::Array(mut descendants)) = flagged.get(b"DescendantFonts").cloned() else { panic!("fixture") }; + let Some(Object::Dict(descendant)) = descendants.first_mut() else { panic!("fixture") }; + let Some(Object::Dict(mut descriptor)) = descendant.get(b"FontDescriptor").cloned() else { panic!("fixture") }; + descriptor.set(b"Flags".to_vec(), Object::Int(96)); + descriptor.set(b"ItalicAngle".to_vec(), Object::Int(angle)); + descendant.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + flagged.set(b"DescendantFonts".to_vec(), Object::Array(descendants)); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(flagged)); + let upright = legacy_thai_font(&mut doc, "GHIJKL+THSarabunPSK", &legacy_thai_program(), LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(upright)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style); + assert_eq!(edited.is_ok(), completes, "ItalicAngle {angle}: {:?}", edited.err()); + if completes { + assert!(shown_codes(&reopen(&doc)).contains(&("F2".into(), vec![0x00, 0xd9])), "ค from the upright sibling"); + } + } +} + +/// A subset keeps only what its document used. One without อ or any tone alternate still takes +/// text that needs neither, in the source face; text that needs an alternate it lacks is refused. +#[test] +fn editing_legacy_thai_needs_only_the_glyphs_the_text_uses() { + let mut sparse = legacy_thai_program(); + for gid in [258, 347, 352, 353] { + drop_sfnt_glyph(&mut sparse, gid); + } + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &sparse, LEGACY_THAI_UNICODE); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let before = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let error = text::rewrite_block(&mut doc, 0, 0, Some("ปู่"), &style).unwrap_err().to_string(); + assert!(error.contains("contextual mark") && error.contains("U+0E48"), "{error}"); + assert_eq!(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(), before); + let edited = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).expect("every glyph this text needs is in the subset"); + assert_eq!(edited.substituted, None); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().iter().map(|l| l.text.as_str()).collect::(), "กขค"); + assert!(shown_codes(&doc).iter().all(|(font, _)| font == "F1"), "{:?}", shown_codes(&doc)); +} + +/// A subset may give a character a code whose glyph it emptied (its `/W` still lists it). The +/// edit then uses another subset of the face that draws it, rather than drawing nothing. +#[test] +fn editing_legacy_thai_skips_a_subset_whose_glyph_is_empty() { + // Both subsets map A (glyph 36 in Inter) through ToUnicode; F1 has emptied it. + let unicode = [LEGACY_THAI_UNICODE, b" 1 beginbfchar <0024> <0041> endbfchar"].concat(); + let mut empty_a = legacy_thai_program(); + drop_sfnt_glyph(&mut empty_a, 36); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let first = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &empty_a, &unicode); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(first)); + let second = legacy_thai_font(&mut doc, "GHIJKL+THSarabunPSK", &legacy_thai_program(), &unicode); + doc.set(pdfcraft_cos::ObjRef::new(6, 0), Object::Dict(second)); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + text::rewrite_block(&mut doc, 0, 0, Some("กA"), &style).unwrap(); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().iter().map(|l| l.text.as_str()).collect::(), "กA"); + let shown = shown_codes(&doc); + assert!(shown.contains(&("F2".into(), vec![0x00, 0x24])), "A from the subset that has its outline: {shown:?}"); + assert!(!shown.contains(&("F1".into(), vec![0x00, 0x24])), "not from the one that emptied it: {shown:?}"); +} + +/// Each two-byte code a page's content shows, with the font resource it is shown in. +fn shown_codes(doc: &Document) -> Vec<(String, Vec)> { + let mut font = String::new(); + let mut shown = Vec::new(); + for op in pdfcraft_content::parse(&page_content_bytes(doc, 0)).ops { + if op.is("Tf") { + font = String::from_utf8_lossy(op.name(0).unwrap_or_default()).into_owned(); + } + let strings: Vec> = match op.op.as_slice() { + b"Tj" | b"'" => op.operands.last().and_then(Object::as_string).map(|s| s.bytes.clone()).into_iter().collect(), + b"TJ" => { + op.operands.first().and_then(Object::as_array).into_iter().flatten().filter_map(Object::as_string).map(|s| s.bytes.clone()).collect() + } + _ => Vec::new(), + }; + for bytes in strings { + shown.extend(bytes.chunks(2).map(|c| (font.clone(), c.to_vec()))); + } + } + shown +} + +/// Where table `tag` starts in an sfnt. +fn sfnt_table(program: &[u8], tag: &[u8; 4]) -> usize { + let count = u16::from_be_bytes(program[4..6].try_into().unwrap()) as usize; + let record = (0..count).map(|i| 12 + i * 16).find(|&r| &program[r..r + 4] == tag).unwrap(); + u32::from_be_bytes(program[record + 8..record + 12].try_into().unwrap()) as usize +} + +/// Empty glyph `gid` of an sfnt as a subsetter drops it: its data stays as the previous glyph's +/// padding, and every other glyph keeps its outline. +fn drop_sfnt_glyph(program: &mut [u8], gid: usize) { + let long = program[sfnt_table(program, b"head") + 51] == 1; + let (loca, size) = (sfnt_table(program, b"loca"), if long { 4 } else { 2 }); + let next = program[loca + (gid + 1) * size..][..size].to_vec(); + program[loca + gid * size..][..size].copy_from_slice(&next); +} + +/// Inter's outlines in a legacy THSarabunPSK subset's glyph order; no proprietary font fixture is +/// needed. The slots where that order has a Thai vowel or tone mark, or a mark-like Windows PUA +/// alternate, take no advance (the source font is checked for that before its glyphs are trusted) +/// and the others 500 units. Removing the layout and `cmap` tables reproduces legacy subsetting. +fn legacy_thai_program() -> Vec { + let mut program = include_bytes!("../../../assets/fonts/Inter-Regular.ttf").to_vec(); + let hmtx = sfnt_table(&program, b"hmtx"); + for gid in (214..=298usize).chain(342..=368) { + let mark = match gid { + 342..=368 => !matches!(gid, 342 | 357), + _ => matches!(gid + if gid <= 271 { 0xd2b } else { 0xd2f }, 0x0e31 | 0x0e34..=0x0e3a | 0x0e47..=0x0e4e), + }; + let advance: u16 = if mark { 0 } else { 500 }; + program[hmtx + gid * 4..hmtx + gid * 4 + 2].copy_from_slice(&advance.to_be_bytes()); + } + let count = u16::from_be_bytes(program[4..6].try_into().unwrap()) as usize; let records: Vec<_> = program[12..12 + count * 16] .as_chunks::<16>() .0 @@ -10855,6 +11327,38 @@ fn editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_aft for (index, record) in records.iter().enumerate() { program[12 + index * 16..28 + index * 16].copy_from_slice(record); } + program +} + +/// A THSarabunPSK Identity-H font over `program` (a [`legacy_thai_program`]), mapping `unicode`. +fn legacy_thai_font(doc: &mut Document, base_font: &str, program: &[u8], unicode: &[u8]) -> Dict { + let raw = doc.add(Object::Stream(Stream::flate(Dict::new(), program))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), Object::Ref(raw)); + let mut descendant = Dict::new(); + descendant.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + descendant.set(b"CIDToGIDMap".to_vec(), Object::name("Identity")); + descendant.set(b"W".to_vec(), Object::Array(vec![Object::Int(0), Object::Int(65535), Object::Int(500)])); + let unicode = doc.add(Object::Stream(Stream::flate(Dict::new(), unicode))); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type0")); + font.set(b"BaseFont".to_vec(), Object::name(base_font)); + font.set(b"Encoding".to_vec(), Object::name("Identity-H")); + font.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(descendant)])); + font.set(b"ToUnicode".to_vec(), Object::Ref(unicode)); + font +} + +/// The intact ToUnicode of a legacy THSarabunPSK subset: nominal Thai and two tone alternates. +const LEGACY_THAI_UNICODE: &[u8] = + b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 2 beginbfchar <0160> <0e48> <0161> <0e49> endbfchar"; + +#[test] +fn editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen() { + // Remap existing OFL Inter outlines to synthetic Thai glyph slots; no proprietary font + // fixture is needed. Removing layout/cmap tables reproduces legacy PDF font subsetting. + let program = legacy_thai_program(); for bold in [false, true] { let mut doc = text_page("BT /F1 17.5 Tf 0.42 Tc 1.1 Tw 86 Tz 1.5 Ts 2 Tr 0.6 w 0.2 0.3 0.4 rg 50 650 Td <00d601090119> Tj ET"); let raw = doc.add(Object::Stream(Stream::flate(Dict::new(), &program))); diff --git a/crates/edit/src/text/forms.rs b/crates/edit/src/text/forms.rs index 71339af64b..dff7440145 100644 --- a/crates/edit/src/text/forms.rs +++ b/crates/edit/src/text/forms.rs @@ -20,6 +20,10 @@ pub struct Preparation { pub repaired_fonts: usize, } +/// How many levels of nested groups preparation opens on one page. Text nested deeper stays +/// grouped and is reported in [`Preparation::refused`]. +const MAX_LEVELS: usize = 8; + /// Prepare grouped source text atomically, within eight levels and 64 placements per page. /// Shared forms and other pages are untouched. Preparation is a distinct undoable engine edit. pub fn prepare(doc: &mut Document, page: usize) -> Result { @@ -28,7 +32,8 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result>()); @@ -63,11 +68,18 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result { - // Pure vector artwork needs no preparation. Nested text groups are visited - // on the next pass, with their inherited placement and clip still in force. - if !inline.iter().any(|o| matches!(o.op.as_slice(), b"Tj" | b"TJ" | b"'" | b"\"" | b"Do")) { + // The resource name exactly as the `Do` writes it: a name need not be UTF-8, and + // `image.name` is only its display form. + let name = op.last_name().map_or_else(|| image.name.as_bytes().to_vec(), <[u8]>::to_vec); + match expand::ungroup_ops(&next, &merged, &name, translucent, space.as_ref(), Matrix(image.matrix), &mut decode) { + Ok(expand::Ungrouped { ops: inline, resources: mut res, patterns, shows_text }) => { + // Vector and image artwork needs no preparation. Nested text groups are + // visited on the next pass, with their inherited placement and clip in force. + if !shows_text { + continue; + } + if pass == MAX_LEVELS { + result.refused.push(format!("{}: its text is grouped more than {MAX_LEVELS} levels deep", image.name)); continue; } let data_len = pdfcraft_content::serialize_ops(&inline).len(); @@ -140,8 +152,10 @@ fn repair_thai_fonts(doc: &mut Document, page: usize) -> Result <{:04x}>\n", u32::from(ch))); diff --git a/crates/edit/src/text/forms/expand.rs b/crates/edit/src/text/forms/expand.rs index 27cbf1b48e..bf01e201d1 100644 --- a/crates/edit/src/text/forms/expand.rs +++ b/crates/edit/src/text/forms/expand.rs @@ -11,11 +11,13 @@ //! Drawing a form inline paints the same pixels unless the form is a transparency group whose //! compositing differs from painting its parts one by one: a knockout group, a group that uses //! a blend mode other than Normal inside, or a group drawn translucent, blended or under a soft -//! mask, or a group with a blending colour space (`/CS`) other than the page's: its contents -//! composite in that space and are then converted (ISO 32000-2 §11.6.6), inline they would -//! composite in the page's. Those are refused rather than changed, as is tagged form content (its structure link -//! would be lost) and content whose marked content or text objects don't balance (inlined, they -//! would change the scope of the page's own). A form in a layer (`/OC`) keeps its membership as `/OC … BDC … EMC`. +//! mask, or a group with a blending colour space (`/CS`) the page group doesn't name too: its +//! contents composite in that space and are then converted (ISO 32000-2 §11.6.6), inline they +//! would composite in the page's, which a page without one leaves to the PDF processor. Those +//! are refused rather than changed, as is tagged form content (its structure link would be lost) +//! and content whose marked content or text objects don't balance (inlined, they would change +//! the scope of the page's own). A form in a layer (`/OC`) keeps its membership as +//! `/OC … BDC … EMC`. //! //! A pattern's matrix maps pattern space to the default space of the content stream that uses it //! (ISO 32000-2 §8.7.2): inside the form that is the form's space at the `Do`, on the page it is @@ -42,10 +44,20 @@ const MAX_MOVED_PATTERNS: usize = 256; /// A name map for one resource category: form name → name in the page's resources. type Renames = Vec<(Vec, Vec)>; -/// What [`ungroup_ops`] gives back: the ops that draw the form inline, the page resources with -/// the form's merged in, and the pattern copies to add to the document, each under the name in -/// the merged `/Pattern` dictionary that the ops already use. -pub(crate) type Ungrouped = (Vec, Dict, Vec<(Vec, Object)>); +/// What [`ungroup_ops`] gives back. +pub(crate) struct Ungrouped { + /// The ops that draw the form inline. + pub(crate) ops: Vec, + /// The page resources with the form's merged in. + pub(crate) resources: Dict, + /// The pattern copies to add to the document, each under the name in the merged `/Pattern` + /// dictionary that the ops already use. + pub(crate) patterns: Vec<(Vec, Object)>, + /// Whether the form shows text, in its own content or in a Form XObject it draws (at any + /// depth [`MAX_NESTING`] allows). Text inside a tiling pattern or Type 3 glyph doesn't count: + /// inlining the form doesn't make it editable. Image-only and vector-only forms are artwork. + pub(crate) shows_text: bool, +} /// Ungroup form `name` from the page's `resources`. `translucent` says whether the graphics state /// at the `Do` is translucent, blended or soft-masked, which a transparency group composites as a @@ -122,14 +134,15 @@ pub(crate) fn ungroup_ops( return Err(refuse("its content has parts that can't be read")); } let mut ops = parsed.ops; - { + let shows_text = { // Check every form, including non-group forms, for recursive/unsupported content. // The form itself is being read: drawing itself again counts as blending. - let mut walk = BlendWalk { path: form.as_ref().into_iter().collect(), known: HashMap::new(), reads: 0, budget }; + let mut walk = BlendWalk { path: form.as_ref().into_iter().collect(), known: HashMap::new(), reads: 0, budget, text: false }; if blends_inside(doc, &form_res, &ops, 0, &mut walk) { return Err(refuse("it has blend modes, recursive forms, or nested content that cannot be verified")); } - } + walk.text + }; // Merge the form's resources into a copy of the page's, renaming what clashes. let mut merged = resources.clone(); @@ -197,6 +210,13 @@ pub(crate) fn ungroup_ops( b"ET" => in_text = false, _ => {} } + // A name is renamed at the operand position below, but the renderer takes operands from + // the end and ignores surplus ones before them: with a surplus operand the drawn name + // would keep its old spelling while its resource moved, and draw nothing or the wrong + // thing. The parser keeps such operands without counting them as skipped. + if takes_operands(&op.op).is_some_and(|n| op.operands.len() > n) { + return Err(refuse("its content gives an operator more operands than it takes")); + } let slot: Option<(&[u8], usize)> = match op.op.as_slice() { b"Tf" | b"Do" | b"gs" | b"sh" | b"cs" | b"CS" => { let cat: &[u8] = match op.op.as_slice() { @@ -345,7 +365,7 @@ pub(crate) fn ungroup_ops( if layer.is_some() { out.push(Op::new("EMC", Vec::new())); } - Ok((out, merged, patterns)) + Ok(Ungrouped { ops: out, resources: merged, patterns, shows_text }) } /// A copy of pattern `entry` whose `/Matrix` maps its space to the page's default space instead @@ -389,13 +409,30 @@ const MAX_NESTING: usize = 8; const MAX_NESTED_READS: usize = 64; /// What [`blends_inside`] carries through its nested reads: the forms on the current path (a -/// form drawing itself), the answer for each form with resources of its own (its answer doesn't -/// depend on who draws it), and how many reads it has made. +/// form drawing itself), the answers for each form with resources of its own (whether it blends +/// and whether it shows text; neither depends on who draws it), how many reads it has made, and +/// whether the content being walked shows text itself or through a Form XObject it draws. struct BlendWalk<'a> { path: Vec, - known: HashMap, + known: HashMap, reads: usize, budget: &'a mut DecodeBudget, + text: bool, +} + +/// Whether `op` shows text (ISO 32000-2 §9.4.3, Table 107). +fn shows_text(op: &Op) -> bool { + matches!(op.op.as_slice(), b"Tj" | b"TJ" | b"'" | b"\"") +} + +/// How many operands an operator that names a resource takes (ISO 32000-2 Annex A), its name +/// being the first of them: only for those whose name is read at a fixed position here. +fn takes_operands(op: &[u8]) -> Option { + match op { + b"Do" | b"gs" | b"sh" | b"cs" | b"CS" => Some(1), + b"Tf" | b"BDC" | b"DP" => Some(2), + _ => None, + } } /// Whether `op` draws a Form XObject (`Do`), Type 3 font (`Tf`) or tiling pattern (`scn`/`SCN`) from @@ -430,11 +467,21 @@ fn inherits_resources(doc: &Document, resources: &Dict, op: &Op) -> bool { /// Whether `ops` blend through a used graphics state, nested form or pattern. Unreadable, /// recursive or over-budget nested content is treated as blending, so preparation refuses it. +/// Sets `walk.text` when `ops` show text, or a Form XObject they draw does; text in a tiling +/// pattern's cell is not the page's text and is not counted. fn blends_inside(doc: &Document, resources: &Dict, ops: &[Op], depth: usize, walk: &mut BlendWalk<'_>) -> bool { let category = |cat: &[u8]| resources.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); let (states, xobjects, patterns) = (category(b"ExtGState"), category(b"XObject"), category(b"Pattern")); let state_blends = |g: Option<&Object>| g.map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).is_some_and(|gs| blends(doc, &gs)); + if ops.iter().any(shows_text) { + walk.text = true; + } for op in ops { + // Read at operand 0 below; with a surplus operand the renderer draws another name, which + // this walk can't vouch for. + if takes_operands(&op.op).is_some_and(|n| op.operands.len() > n) { + return true; + } if op.is("gs") && state_blends(op.name(0).and_then(|n| states.get(n))) { return true; } @@ -466,10 +513,12 @@ fn blends_inside(doc: &Document, resources: &Dict, ops: &[Op], depth: usize, wal _ => continue, }; let r = o.as_ref(); - if let Some(known) = r.and_then(|r| walk.known.get(&r)) { - if *known { + let drawn_form = op.is("Do"); + if let Some((known, text)) = r.and_then(|r| walk.known.get(&r)).copied() { + if known { return true; } + walk.text |= drawn_form && text; continue; } if depth >= MAX_NESTING || r.is_some_and(|r| walk.path.contains(&r)) || walk.reads >= MAX_NESTED_READS { @@ -487,11 +536,14 @@ fn blends_inside(doc: &Document, resources: &Dict, ops: &[Op], depth: usize, wal if let Some(r) = r { walk.path.push(r); } + let outer_text = std::mem::replace(&mut walk.text, false); let blends = blends_inside(doc, &own, &inner.ops, depth + 1, walk); + let inner_text = walk.text; + walk.text = outer_text || (drawn_form && inner_text); if let Some(r) = r { walk.path.retain(|x| *x != r); if has_own { - walk.known.insert(r, blends); + walk.known.insert(r, (blends, inner_text)); } } if blends { @@ -508,14 +560,13 @@ pub(crate) fn page_blend_space(doc: &Document, page: &Dict) -> Option { group.get(b"CS").map(|cs| (*doc.resolve(cs)).clone()) } -/// Whether a group blending in `cs` composites as the page does: in the page group's own space -/// when it names one, else in DeviceRGB, the space viewers composite a page in on screen. +/// Whether a group blending in `cs` composites as the page does: only when the page group names +/// the same space. A page that names none composites in a space the PDF processor chooses +/// (ISO 32000-2 §11.4.7): DeviceRGB on screen, but often DeviceCMYK when printed, where a group +/// that blends in DeviceRGB converts its CMYK and gray colours through RGB and inline they would +/// not be. Such a group is refused rather than changed. fn same_blend_space(doc: &Document, cs: &Object, page_space: Option<&Object>) -> bool { - let cs = doc.resolve(cs); - match page_space { - Some(page) => *cs == *page, - None => cs.as_name() == Some(b"DeviceRGB"), - } + page_space.is_some_and(|page| *doc.resolve(cs) == *page) } /// Whether `d`'s `key` is the name `want`, directly or through an indirect object (any value may diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 5d8075785e..84ec8c75cc 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -107,13 +107,34 @@ impl FontResolver<'_> { struct LegacyFace { glyphs: Option, metrics: Metrics, + /// Weight and slant ([`FontStyle::of`]). + style: FontStyle, } struct PreparedCharacters { values: Vec, substituted: Option, - outlined_thai: std::collections::HashSet, - legacy_thai: HashMap, + thai: ThaiShaping, +} + +/// How the Thai spans of an edit are shaped, by the font name their characters carry. +#[derive(Default)] +struct ThaiShaping { + /// Draft-only names drawn with the bundled Thai outlines. + outlined: std::collections::HashSet, + /// Source faces shaped in their existing codes, by resource name. + legacy: HashMap, + /// Draft-only names whose rows, Latin ones included, are shaped and embedded with the + /// complete program of this same-face resource: a span its subsets can't show in place. + program: HashMap, +} + +/// How a Thai span keeps its source face ([`legacy_thai_font`]). +enum ThaiSpan { + /// In this resource's existing codes. + InPlace(String), + /// Shaped by this same-face resource's complete program with its own layout. + Program(String), } struct CharacterPreparation<'a> { @@ -133,8 +154,7 @@ impl CharacterPreparation<'_> { fn prepare(self) -> Result { let Self { doc, first, lines, draft, spans, style, overrides, fonts, metrics, max_glyphs } = self; let thai_choice = builtin_thai_choice(fonts); - let mut outlined_thai = std::collections::HashSet::new(); - let mut legacy_thai = HashMap::new(); + let mut thai = ThaiShaping::default(); let mut chars = Vec::new(); let mut substituted = None; let mut fallback = FallbackBuilder::new(max_glyphs); @@ -146,13 +166,17 @@ impl CharacterPreparation<'_> { // Interface fallback fonts never enter a PDF. let selected_resource = overrides.get(&index).and_then(|r| r.resource.as_deref()).or(style.resource.as_deref()); let use_thai_outlines = if selected_resource == Some(thai_choice.as_str()) { - let face = pdfcraft_fonts::document_thai_font() - .ok_or_else(|| EditError::Invalid("the bundled Thai font is unavailable in this build; load a complete Thai font".into()))?; + let face = pdfcraft_fonts::document_thai_font().ok_or_else(|| { + EditError::Invalid(format!( + "the bundled Thai font ({thai_choice}) is not in this build, which has no craft-fonts Thai face; load a complete Thai font or choose one the page lists" + )) + })?; if !pdfcraft_fonts::embedding::covers(face.bytes, span) { - return Err(EditError::Invalid(format!( - "{} cannot show every character in this selection; choose a font that covers the text", - face.family - ))); + let missing = span.chars().find(|c| !pdfcraft_fonts::embedding::covers(face.bytes, c.encode_utf8(&mut [0; 4]))); + return Err(EditError::Invalid(match missing { + Some(c) => format!("{} has no glyph for {c} (U+{:04X}); choose a font that covers the text", face.family, u32::from(c)), + None => format!("{} cannot show every character in this selection; choose a font that covers the text", face.family), + })); } substituted = Some(face.family.to_owned()); true @@ -173,7 +197,7 @@ impl CharacterPreparation<'_> { }), }; match chosen { - Some(chosen) => legacy_thai_font(resolver.doc, resolver.fonts, &chosen, span, &mut legacy_thai)?, + Some(chosen) => legacy_thai_font(resolver.doc, resolver.fonts, &chosen, span, &mut thai.legacy)?, None => None, } } else { @@ -194,11 +218,27 @@ impl CharacterPreparation<'_> { { resolver.metrics.insert(name.clone(), m); } - outlined_thai.insert(name.clone()); + thai.outlined.insert(name.clone()); Some(name) } else { None }; + let (existing_thai, program_name) = match existing_thai { + Some(ThaiSpan::InPlace(name)) => (Some(name), None), + Some(ThaiSpan::Program(resource)) => { + // Draft-only too: another span may still show this resource's existing codes. + let mut name = format!("PCProgramThai{index}"); + while resolver.metrics.contains_key(&name) { + name.push('_'); + } + if let Some(m) = resolver.metrics.get(&source.font).cloned() { + resolver.metrics.insert(name.clone(), m); + } + thai.program.insert(name.clone(), resource); + (None, Some(name)) + } + None => (None, None), + }; let start = range.start; for (offset, ch) in draft.get(range).unwrap_or_default().char_indices() { if ch == '\n' { @@ -212,7 +252,7 @@ impl CharacterPreparation<'_> { } let text = ch.to_string(); let requested = overrides.get(&index).and_then(|range| range.resource.as_deref()).or(style.resource.as_deref()); - let name = if let Some(name) = &outline_name { + let name = if let Some(name) = outline_name.as_ref().or(program_name.as_ref()) { name.clone() } else if let Some(name) = &existing_thai { name.clone() @@ -245,7 +285,7 @@ impl CharacterPreparation<'_> { } substituted = Some(written.family.to_owned()); } - Ok(PreparedCharacters { values: chars, substituted, outlined_thai, legacy_thai }) + Ok(PreparedCharacters { values: chars, substituted, thai }) } } @@ -324,8 +364,7 @@ struct ShapedRows { fn prepare_shaped_rows( doc: &Document, fonts: &Dict, - outlined_thai: &std::collections::HashSet, - legacy_thai: &HashMap, + thai: &ThaiShaping, rows: &[Vec], kept: &[bool], advance: impl Fn(&StyledChar) -> f64, @@ -342,7 +381,7 @@ fn prepare_shaped_rows( let segment = row.get(at..end).unwrap_or_default(); advances.insert((row_index, at), segment.iter().map(&advance).sum::()); let text: String = segment.iter().map(|character| character.0).collect(); - if let Some(shaped) = shape_source(doc, fonts, outlined_thai, legacy_thai, &item.2, &text)? { + if let Some(shaped) = shape_source(doc, fonts, thai, &item.2, &text)? { runs.insert((row_index, at), shaped); } at = end; @@ -791,7 +830,7 @@ fn legacy_thai_font( source: &TextLine, span: &str, cache: &mut HashMap, -) -> Result, EditError> { +) -> Result, EditError> { if !source.base_font.rsplit('+').next().unwrap_or(&source.base_font).starts_with("THSarabunPSK") || !span.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) { @@ -808,13 +847,39 @@ fn legacy_thai_font( let Some(font) = fonts.get(entry.key().as_bytes()).and_then(|f| doc.dict(f)) else { continue }; let metrics = Metrics::from_dict(doc, &font); let glyphs = pdfcraft_fonts::pdf::LegacyThaiFont::from_pdf(doc, &font, source_space(doc, fonts, &metrics)).ok().flatten(); - entry.insert(LegacyFace { glyphs, metrics }); + let style = FontStyle::of(doc, &font, &metrics); + entry.insert(LegacyFace { glyphs, metrics, style }); } } - for row in span.split(['\n', '\r']).filter(|row| !row.is_empty()) { - shape_existing_thai(cache, &source.font, row)?; + // A face none of whose subsets can be shaped in place (no Thai, or neither positioned nor in + // the legacy glyph order) is shaped like any other source font (`shape_source`). + if !has_thai_source_face(cache, &family) { + return Ok(None); } - Ok(Some(source.font.clone())) + let rows = || span.split(['\n', '\r']).filter(|row| !row.is_empty()); + for row in rows() { + if let Err(error) = shape_existing_thai(cache, &source.font, row) { + // Not in the face's existing codes. A complete program of the face that shapes every + // row with its own layout (one Load font added as CFF, or whose contextual glyphs it + // gave no code yet) takes the whole span, so no cluster is split between fonts. + let Some(source_font) = fonts.get(source.font.as_bytes()).and_then(|f| doc.dict(f)) else { return Err(error) }; + let complete = program_candidates(doc, fonts, &source.font, source_font, &source.base_font) + .into_iter() + .find(|(_, font)| rows().all(|row| shape_with_program(doc, font, row).is_some())); + return match complete { + Some((name, _)) => Ok(Some(ThaiSpan::Program(name))), + None => Err(error), + }; + } + } + Ok(Some(ThaiSpan::InPlace(source.font.clone()))) +} + +/// Whether a subset of the face `family` (a [`face_key`](pdfcraft_fonts::embedding::face_key)) +/// in `cache` can be shaped in place: a THSarabunPSK program with its own Thai mark positioning, +/// or a verified legacy positional one. +fn has_thai_source_face(cache: &HashMap, family: &str) -> bool { + cache.values().any(|face| face.glyphs.is_some() && pdfcraft_fonts::embedding::face_key(&face.metrics.base_font) == family) } /// A visual row may use several subsets of the same exact face (often a separate subset for @@ -829,7 +894,9 @@ fn shape_existing_thai(cache: &HashMap, primary: &str, text: candidates.extend( cache .iter() - .filter(|(name, face)| name.as_str() != primary && pdfcraft_fonts::embedding::face_key(&face.metrics.base_font) == family) + .filter(|(name, face)| { + name.as_str() != primary && face.style == first.style && pdfcraft_fonts::embedding::face_key(&face.metrics.base_font) == family + }) .map(|(name, face)| (name.as_str(), face)), ); // Stable source-first choice, then resource name; never depend on hash iteration order. @@ -851,42 +918,42 @@ fn shape_existing_thai(cache: &HashMap, primary: &str, text: end = next + c.len_utf8(); } let segment = text.get(at..end).ok_or_else(|| EditError::Invalid("invalid Thai character boundary".into()))?; - let mut selected = None; - let mut why = String::new(); - for (name, face) in &candidates { - if let Some(font) = &face.glyphs { - match font.shape(segment) { - Ok(shaped) => { - selected = Some((*name, shaped)); - break; - } - Err(e) => { - if why.is_empty() { - why = e; - } - } - } - } - } - let (name, shaped) = selected.ok_or_else(|| { + let unshaped = |why: String| { EditError::Invalid(format!( "{} cannot keep this edit in the original face: {}; load the complete original font or explicitly choose a replacement", first.metrics.base_font, if why.is_empty() { "its source glyph program is incomplete" } else { &why } )) - })?; - for (mut glyph, code) in shaped.shaped.glyphs.into_iter().zip(shaped.codes) { - glyph.position[0] += run.advance; - resources.push((name.to_owned(), code)); - run.glyphs.push(glyph); + }; + // The whole run in one subset keeps any kerning between its clusters. A subsetter may + // instead have split the run's glyphs across subsets of the face: then each cluster + // (a base and the marks on it, which shape together) comes from a subset that has it. + let pieces = match shape_in_one_subset(&candidates, segment) { + Ok(whole) => vec![whole], + Err(_) => thai_clusters(segment) + .into_iter() + .map(|cluster| shape_in_one_subset(&candidates, cluster).map_err(unshaped)) + .collect::, _>>()?, + }; + for (name, shaped) in pieces { + for (mut glyph, code) in shaped.shaped.glyphs.into_iter().zip(shaped.codes) { + glyph.position[0] += run.advance; + resources.push((name.to_owned(), code)); + run.glyphs.push(glyph); + } + run.advance += shaped.shaped.advance; } - run.advance += shaped.shaped.advance; } else { let logical = ch.to_string(); let found = candidates.iter().find_map(|(name, face)| { let code = face.metrics.encode(&logical)?; - let cid = face.metrics.codes(&code).first()?.0; - Some((*name, code, face.metrics.width(cid))) + let first = face.metrics.codes(&code).first()?.0; + // `encode` accepts a CID the subset emptied (#1443): drawing it would show nothing. + // Try the next subset; a space then falls back to its measured gap below. + if face.metrics.has_outline(first) == Some(false) { + return None; + } + Some((*name, code, face.metrics.width(first))) }); let (name, code, width) = if let Some(found) = found { found @@ -919,10 +986,45 @@ fn shape_existing_thai(cache: &HashMap, primary: &str, text: Ok(ShapedText { run, font: ShapedFont::Existing(resources) }) } +/// Shape `text` in the first candidate subset that has every glyph it needs, and name it. The +/// error is the first subset's reason (empty when no candidate has a usable glyph program). +fn shape_in_one_subset<'a>(candidates: &[(&'a str, &LegacyFace)], text: &str) -> Result<(&'a str, pdfcraft_fonts::pdf::ExistingThaiRun), String> { + let mut why = String::new(); + for (name, face) in candidates { + let Some(font) = &face.glyphs else { continue }; + match font.shape(text) { + Ok(shaped) => return Ok((name, shaped)), + Err(e) if why.is_empty() => why = e, + Err(_) => {} + } + } + Err(why) +} + +/// `text` (a run of Thai characters) cut before each character that doesn't attach to the one +/// before it: a cluster is a base or leading vowel with the vowels and tone marks above and below +/// it, and Sara Am, whose Nikhahit sits on the base. Legacy presentation forms (U+F700–U+F71A) +/// are kept with what precedes them. +fn thai_clusters(text: &str) -> Vec<&str> { + let attaches = |c: char| matches!(u32::from(c), 0x0e31 | 0x0e33..=0x0e3a | 0x0e47..=0x0e4e | 0xf700..=0xf71a); + let mut starts: Vec = text.char_indices().filter(|(at, c)| *at == 0 || !attaches(*c)).map(|(at, _)| at).collect(); + starts.push(text.len()); + // Each boundary is a `char_indices` position or the end, strictly increasing: every `get` + // succeeds and no cluster is empty. + starts.windows(2).filter_map(|w| text.get(*w.first()?..*w.get(1)?)).collect() +} + /// Whether the approved Thai face can replace this source, whose embedded subset cannot /// shape the replacement. A usable complete source program always takes precedence. fn matching_thai_outline(doc: &Document, fonts: &Dict, source: &TextLine, span: &str) -> bool { let Some(face) = pdfcraft_fonts::document_thai_font() else { return false }; + // The bundled face is Regular and upright. A source its descriptor or name marks bold or + // italic keeps that style, so completing it with these outlines would be a silent change: + // it needs an explicit choice instead. (Variable-font instances often keep the family name.) + let style = fonts.get(source.font.as_bytes()).and_then(|f| doc.dict(f)).map(|f| FontStyle::of(doc, &f, &Metrics::from_dict(doc, &f))); + if style.map_or(source.bold || source.italic, |s| s.bold || s.italic) { + return false; + } if !span.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) || !pdfcraft_fonts::embedding::matching_faces(&source.base_font).iter().any(|f| std::ptr::eq(*f, face)) || !pdfcraft_fonts::embedding::covers(face.bytes, span) @@ -972,46 +1074,98 @@ fn shape_thai_row(text: &str) -> Result Vec<(String, Dict)> { + let key = pdfcraft_fonts::embedding::face_key(base_font); + let style = FontStyle::of(doc, &source, &Metrics::from_dict(doc, &source)); + let mut candidates = vec![(name.to_owned(), source)]; + candidates.extend( + fonts + .iter() + .filter_map(|(n, f)| Some((String::from_utf8_lossy(n).into_owned(), doc.dict(f)?))) + .filter(|(_, f)| { + let m = Metrics::from_dict(doc, f); + pdfcraft_fonts::embedding::face_key(&m.base_font) == key && FontStyle::of(doc, f, &m) == style + }) + .take(128), + ); + candidates +} + +/// A font's weight and slant, for matching subsets and complete programs of one face: a Regular +/// program never completes text that a descriptor marks bold, say. +#[derive(Clone, Copy, Debug, PartialEq)] +struct FontStyle { + bold: bool, + italic: bool, +} + +impl FontStyle { + /// Bold from the name or the descriptor's weight ([`Metrics`]); italic from the name or a + /// non-zero `/ItalicAngle`. The descriptor's Italic flag alone isn't trusted: exporters set + /// it on upright faces (a THSarabunPSK subset with `/Flags 96` and `/ItalicAngle 0`). + fn of(doc: &Document, font: &Dict, metrics: &Metrics) -> Self { + let descendant = + font.get(b"DescendantFonts").and_then(|d| doc.resolve(d).as_array().and_then(|a| a.first()).cloned()).and_then(|d| doc.dict(&d)); + let descriptor = descendant.as_ref().unwrap_or(font).get(b"FontDescriptor").and_then(|d| doc.dict(d)); + let angle = descriptor.as_ref().and_then(|d| d.get(b"ItalicAngle")).and_then(|a| doc.resolve(a).as_f64()).unwrap_or(0.0); + let name = metrics.base_font.to_ascii_lowercase(); + Self { bold: metrics.bold, italic: name.contains("italic") || name.contains("oblique") || (angle.is_finite() && angle.abs() > 0.1) } + } +} + +/// `text` shaped by `font`'s embedded program with its own layout, when the program covers it. +fn shape_with_program(doc: &Document, font: &Dict, text: &str) -> Option<(Vec, pdfcraft_fonts::source_shaping::ShapedRun)> { + let bytes = pdfcraft_fonts::pdf::embedded_sfnt(doc, font)?; + if !pdfcraft_fonts::embedding::covers(&bytes, text) { + return None; + } + let shaped = pdfcraft_fonts::source_shaping::shape(&bytes, text).ok()?; + Some((bytes, shaped)) +} + /// Shape in the source or an explicitly registered matching full face. Never use interface fonts. -fn shape_source( - doc: &Document, - fonts: &Dict, - outlined_thai: &std::collections::HashSet, - legacy_thai: &HashMap, - name: &str, - text: &str, -) -> Result, EditError> { - if outlined_thai.contains(name) { +fn shape_source(doc: &Document, fonts: &Dict, thai: &ThaiShaping, name: &str, text: &str) -> Result, EditError> { + if thai.outlined.contains(name) { let shaped = shape_thai_row(text)?; return Ok(Some(ShapedText { run: shaped, font: ShapedFont::ThaiOutlines })); } + if let Some(resource) = thai.program.get(name) { + let font = fonts + .get(resource.as_bytes()) + .and_then(|f| doc.dict(f)) + .ok_or_else(|| EditError::Invalid("the complete font resource is missing".into()))?; + let family = Metrics::from_dict(doc, &font).base_font; + let (bytes, shaped) = shape_with_program(doc, &font, text) + .ok_or_else(|| EditError::Invalid(format!("{family} cannot shape this text with its own layout; choose a font that covers it")))?; + return Ok(Some(ShapedText { run: shaped, font: ShapedFont::Program { bytes, family } })); + } if !pdfcraft_fonts::source_shaping::required(text) { return Ok(None); } let source = fonts.get(name.as_bytes()).and_then(|f| doc.dict(f)).ok_or_else(|| EditError::Invalid("the shaped text has no font resource".into()))?; - let metrics = Metrics::from_dict(doc, &source); - if metrics.base_font.rsplit('+').next().unwrap_or(&metrics.base_font).starts_with("THSarabunPSK") + let family = Metrics::from_dict(doc, &source).base_font; + // In the face's existing codes first; what they can't show, a complete program of the face + // may (`legacy_thai_font` already gave the span to one that can). + let in_place = (family.rsplit('+').next().unwrap_or(&family).starts_with("THSarabunPSK") && text.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f)) - { - return shape_existing_thai(legacy_thai, name, text).map(Some); - } - let family = metrics.base_font; - let mut candidates = vec![source]; - candidates.extend( - fonts - .iter() - .filter_map(|(_, f)| doc.dict(f)) - .filter(|f| pdfcraft_fonts::embedding::face_key(&Metrics::from_dict(doc, f).base_font) == pdfcraft_fonts::embedding::face_key(&family)), - ); - for font in candidates { - if let Some(bytes) = pdfcraft_fonts::pdf::embedded_sfnt(doc, &font) - && pdfcraft_fonts::embedding::covers(&bytes, text) - && let Ok(shaped) = pdfcraft_fonts::source_shaping::shape(&bytes, text) - { + && has_thai_source_face(&thai.legacy, &pdfcraft_fonts::embedding::face_key(&family))) + .then(|| shape_existing_thai(&thai.legacy, name, text)); + let in_place = match in_place { + Some(Ok(shaped)) => return Ok(Some(shaped)), + Some(Err(error)) => Some(error), + None => None, + }; + for (_, font) in program_candidates(doc, fonts, name, source, &family) { + if let Some((bytes, shaped)) = shape_with_program(doc, &font, text) { return Ok(Some(ShapedText { run: shaped, font: ShapedFont::Program { bytes, family } })); } } + if let Some(error) = in_place { + return Err(error); + } let fallback = pdfcraft_fonts::document_thai_font() .filter(|_| text.chars().any(|c| matches!(u32::from(c), 0x0e00..=0x0e7f))) .map(|face| format!("; or select {} in the document-font menu to replace it", face.family)) @@ -1302,8 +1456,7 @@ pub(crate) fn rewrite_styled_block( max_glyphs, } .prepare()?; - let outlined_thai = prepared.outlined_thai; - let legacy_thai = prepared.legacy_thai; + let thai = prepared.thai; let chars = prepared.values; let substituted = prepared.substituted; let size_of = |source: &TextLine, index: usize| { @@ -1354,7 +1507,7 @@ pub(crate) fn rewrite_styled_block( } let segment = items.get(at..end).unwrap_or_default(); let string: String = segment.iter().map(|v| v.0).collect(); - if let Some(shaped) = shape_source(doc, &fonts, &outlined_thai, &legacy_thai, &item.2, &string)? { + if let Some(shaped) = shape_source(doc, &fonts, &thai, &item.2, &string)? { let shaped = &shaped.run; let source = lines.get(item.1).unwrap_or(first); let scale = @@ -1406,7 +1559,7 @@ pub(crate) fn rewrite_styled_block( let at = if split { start } else { o.bt_op }; validate_contextual_rows(&rows, &kept)?; let measured: Vec<_> = rows.iter().map(|row| measure(row)).collect::>()?; - let shaped = prepare_shaped_rows(doc, &fonts, &outlined_thai, &legacy_thai, &rows, &kept, advance)?; + let shaped = prepare_shaped_rows(doc, &fonts, &thai, &rows, &kept, advance)?; let ShapedRows { runs: shaped_runs, advances: segment_advances } = shaped; let glyph_resources = embed_shaped_resources(doc, &mut fonts, &shaped_runs)?; let n = pdfcraft_content::num; diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 6368e9097c..93bedb6645 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -194,6 +194,9 @@ pub(crate) struct EditCtx { /// The overlay streams the edit's applied redactions added, which alone the renderer's check /// leaves out. pub(crate) redaction_overlays: Vec, + /// What [`Edit::PrepareText`] prepared, repaired and left grouped, for + /// [`Session::prepare_text`](crate::Session::prepare_text). + pub(crate) prepared: Option, } impl EditCtx { @@ -223,6 +226,7 @@ impl EditCtx { rng: getrandom::fill, renamed_fields: Vec::new(), redaction_overlays: Vec::new(), + prepared: None, } } @@ -734,6 +738,7 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E result.refused.first().cloned().unwrap_or_else(|| "this page has no supported grouped text to prepare".into()), )); } + cx.prepared = Some(result); } Edit::RegisterTextFont { page, bytes } => { pdfcraft_edit::text::register_text_font(doc, *page, bytes)?; diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index be7fd5aca5..eaf2583ca6 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -20,6 +20,19 @@ fn deletes_pages(edit: &Edit) -> bool { impl Session { /// Apply an edit. On success the previous state is undoable and the view data is refreshed. pub fn apply(&mut self, id: DocId, edit: Edit) -> Result<(), EditError> { + self.apply_reporting(id, edit).map(|_| ()) + } + + /// Edit a PDF ▸ Edit text: make the grouped text on `page` (0-based) editable, as one undoable + /// [`Edit::PrepareText`], and say what it prepared, which fonts' maps it repaired and which + /// groups it left as they were (and why). The counts are this edit's own, not a preview's. + pub fn prepare_text(&mut self, id: DocId, page: usize) -> Result { + self.apply_reporting(id, Edit::PrepareText { page })? + .ok_or_else(|| EditError::Invalid("preparing grouped text gave no result; the document was not changed".into())) + } + + /// [`Session::apply`], giving back what an [`Edit::PrepareText`] did (`None` for other edits). + fn apply_reporting(&mut self, id: DocId, edit: Edit) -> Result, EditError> { let now = self.now(); let today = self.today(); let js_off = self.js_off; @@ -61,7 +74,7 @@ impl Session { }; // Nothing to bake: leave undo history and the dirty flag alone. if matches!(&edit, Edit::FlattenFillSign) && !pdfcraft_annot::has_visible_fill_sign(&editor.cos) { - return Ok(()); + return Ok(None); } let mut next = editor.cos.clone(); if !js_off && uses_scripts(&edit) { @@ -228,7 +241,7 @@ impl Session { doc.xfa_scripts_off = true; doc.js_output.errors.push(XFA_SCRIPTS_OFF.into()); } - Ok(()) + Ok(cx.prepared) } pub fn undo(&mut self, id: DocId) -> Result { diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 4fab8a0313..75c56625c8 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -938,6 +938,19 @@ impl Metrics { } } + /// Whether the embedded program draws an outline for `code`. Known (`Some`) only for a subset + /// Type 0 font over a readable TrueType program, where a dropped glyph is left empty; `None` + /// when the program can't tell, for example a simple or CFF font. [`Metrics::encode`] accepts + /// any code with a `/W` entry (#1443), so a caller that would rather use another subset of the + /// same face than draw nothing asks this too. An empty space glyph is normal. + pub fn has_outline(&self, code: u32) -> Option { + if !self.composite || self.outlined_cids.is_empty() { + return None; + } + let cid = self.cid(code); + Some(self.outlined_cids.get(cid as usize / 64).is_some_and(|w| (*w >> (cid % 64)) & 1 == 1)) + } + /// Whether `code` shows exactly `c` and has a glyph for it. fn shows(&self, code: u32, c: char) -> bool { self.unicode.text(code).is_some_and(|t| t.chars().eq(std::iter::once(c))) && self.has_glyph(code) @@ -1561,4 +1574,4 @@ mod encoding_tests { mod thai; use thai::composite_glyph_maps; -pub use thai::{LegacyThaiFont, recovered_thai_map}; +pub use thai::{ExistingThaiRun, LegacyThaiFont, recovered_thai_map}; diff --git a/crates/fonts/src/pdf/thai.rs b/crates/fonts/src/pdf/thai.rs index 54784c291c..8596d58734 100644 --- a/crates/fonts/src/pdf/thai.rs +++ b/crates/fonts/src/pdf/thai.rs @@ -3,7 +3,9 @@ //! Some THSarabunPSK Identity-H subsets retain original glyph IDs but write Latin/control //! destinations for Thai in ToUnicode (including an accidental 00ff..01ff range). The intact //! Thai anchors identify the original nominal Thai order. Only that complete fingerprint is -//! accepted; correct maps and unrelated fonts are never changed. No font artwork is bundled. +//! accepted, and then only when every glyph the subset kept advances as its assumed character +//! does (a mark takes none); correct maps and unrelated fonts are never changed. No font +//! artwork is bundled. use super::{Dict, Document, Metrics, dict, only_char, unicode_map}; use pdfcraft_cos::{DecodeBudget, Object}; @@ -12,6 +14,7 @@ use std::collections::BTreeMap; /// Recover the nominal Thai codes and the presentation marks of this damaged legacy export. /// The caller must record the repair, and attach the map to a copy of the page's font. pub fn recovered_thai_map(doc: &Document, font: &Dict) -> Option> { + use skrifa::{MetadataProvider as _, raw::TableProvider as _}; let base = String::from_utf8_lossy(font.name(b"BaseFont")?); let family = base.rsplit('+').next()?; if !matches!(family, "THSarabunPSK" | "THSarabunPSK-Bold" | "THSarabunPSK-Italic" | "THSarabunPSK-BoldItalic") @@ -101,14 +104,12 @@ pub fn recovered_thai_map(doc: &Document, font: &Dict) -> Option = (214u16..=298) - .filter_map(|gid| { - let unicode = u32::from(gid) + if gid <= 271 { 0xd2b } else { 0xd2f }; - char::from_u32(unicode).map(|ch| (gid, ch)) - }) - .collect(); + let nominal = (214u16..=298).filter_map(|gid| { + let unicode = u32::from(gid) + if gid <= 271 { 0xd2b } else { 0xd2f }; + char::from_u32(unicode).map(|ch| (gid, ch)) + }); // Contextual marks have the same character as the nominal mark, with a moved outline. - out.extend([ + let contextual = [ (343, '\u{0e34}'), (344, '\u{0e35}'), (347, '\u{0e48}'), @@ -118,10 +119,54 @@ pub fn recovered_thai_map(doc: &Document, font: &Dict) -> Option start) { + continue; + } + if advances.advance_width(skrifa::GlyphId::new(u32::from(gid))).is_none_or(|w| (w == 0.0) != is_thai_mark(ch)) { + return None; + } + out.push((gid, ch)); + } Some(out) } +/// Whether a Thai character is a vowel or tone mark set above or below its base, with no advance +/// of its own (U+0E31, U+0E34–U+0E3A, U+0E47–U+0E4E). Sara Am and the other vowels advance. +fn is_thai_mark(ch: char) -> bool { + matches!(u32::from(ch), 0x0e31 | 0x0e34..=0x0e3a | 0x0e47..=0x0e4e) +} + +/// The glyph a legacy THSarabunPSK program draws Thai character `ch` with: U+0E01–U+0E3A are +/// glyphs 214–271 and U+0E3F–U+0E5B glyphs 272–298 (the order `recovered_thai_map` assumes). +fn legacy_glyph(ch: u32) -> Option { + match ch { + 0x0e01..=0x0e3a => ch.checked_sub(0xd2b), + 0x0e3f..=0x0e5b => ch.checked_sub(0xd2f), + _ => None, + } +} + +/// The TrueType program (`/FontFile2`) of a CIDFont, decoded within the editor's size cap. +fn embedded_truetype(doc: &Document, descendant: &Dict) -> Option> { + let descriptor = dict(doc, descendant.get(b"FontDescriptor"))?; + let program = doc.resolve(descriptor.get(b"FontFile2")?); + let Object::Stream(program) = program.as_ref() else { return None }; + DecodeBudget::new(super::MAX_SFNT, 0).decode_within(program, super::MAX_SFNT).ok() +} + /// Source glyph positions and the original PDF character codes that draw them. Empty codes /// represent whitespace advances: no glyph is added to the font or substituted for a space. pub struct ExistingThaiRun { @@ -185,18 +230,23 @@ pub(super) fn composite_glyph_maps( // See https://linux.thai.net/~thep/th-otf/shaping.html (legacy PUA assignments). let family = metrics.base_font.rsplit('+').next().unwrap_or(&metrics.base_font); let alternate_meanings = metrics.text_of(352).is_some_and(|t| t == "่") && metrics.text_of(353).is_some_and(|t| t == "้"); + // Each alternate the subset kept must advance as its form does: U+F700 and U+F70F are ฐ and + // ญ without their descenders, the rest marks with no advance of their own. Any that doesn't + // means another glyph order, and then none is used: the wrong mark would draw silently. + let kept: Vec<(usize, u32)> = (342u32..=368).enumerate().filter(|&(_, gid)| outlined(gid)).collect(); + let in_order = + kept.iter().all(|&(index, gid)| advances.advance_width(skrifa::GlyphId::new(gid)).is_some_and(|w| (w > 0.0) == matches!(index, 0 | 15))); if family.starts_with("THSarabunPSK") && unicode.get(&0x0e01) == Some(&214) && unicode.get(&0x0e2d) == Some(&258) && (alternate_meanings || unicode.contains_key(&0xf70a)) + && in_order { - for (index, gid) in (342u32..=368).enumerate() { - if outlined(gid) - && let Some(code) = match &cid_map { - None => u16::try_from(gid).ok(), - Some(_) => codes.get(&gid).copied(), - } - { + for (index, gid) in kept { + if let Some(code) = match &cid_map { + None => u16::try_from(gid).ok(), + Some(_) => codes.get(&gid).copied(), + } { unicode.insert(0xf700 + index as u32, gid); codes.entry(gid).or_insert(code); } @@ -205,19 +255,26 @@ pub(super) fn composite_glyph_maps( Some((unicode, codes)) } -/// Shape legacy Thai using the PDF's own outlines and return codes for that same font. -/// No font program, family, weight or size is replaced. A subset missing a contextual glyph -/// is refused so the caller can try another subset of the same face or ask for its full font. +/// Shape Thai in a THSarabunPSK source font's own outlines and return codes for that same font. +/// No font program, family, weight or size is replaced. A program with OpenType Thai mark +/// positioning (a complete face loaded with Load font, say) is shaped by that layout; one +/// without it must be a verified legacy positional program, shaped with its nominal glyphs and +/// Windows PUA alternates. A subset missing a glyph the text needs is refused, so the caller can +/// try another subset of the same face or ask for its full font. pub struct LegacyThaiFont { bytes: Vec, codes: BTreeMap, space: f64, + /// The program positions Thai marks itself (GPOS `mark`/`mkmk`). + positioned: bool, } impl LegacyThaiFont { - /// Read and verify the source font once for all wrapping/preview measurements. + /// Read and verify the source font once for all wrapping/preview measurements. `None` for + /// another family, a font with no Thai, and a program without Thai mark positioning whose + /// Thai glyphs are not in the legacy order: the nominal and alternate marks this shaper + /// assumes can't be trusted there, so it is shaped as an ordinary source font. pub fn from_pdf(doc: &Document, font: &Dict, space: f64) -> Result, String> { - use skrifa::MetadataProvider as _; let m = Metrics::from_dict(doc, font); if !m.base_font.rsplit('+').next().unwrap_or(&m.base_font).starts_with("THSarabunPSK") { return Ok(None); @@ -226,14 +283,24 @@ impl LegacyThaiFont { return Err("the source space width is invalid".into()); } let bytes = super::embedded_sfnt(doc, font).ok_or("the source Thai font has no readable glyph program")?; - let face = skrifa::FontRef::new(&bytes).map_err(|e| e.to_string())?; - let cmap = face.charmap(); - if cmap.map('ก').is_none() || cmap.map('อ').is_none() || cmap.map('\u{f70a}').is_none() { + let positioned = crate::source_shaping::has_thai_layout(&bytes); + let (unicode, codes) = composite_glyph_maps(doc, font, &bytes, &m, &mut DecodeBudget::new(1 << 17, 0)) + .ok_or("the Thai source glyph codes cannot be recovered safely")?; + // A subset keeps only the glyphs its document used, so no particular letter or alternate + // is required here: [`Self::shape`] refuses a text whose glyphs, or required alternates + // ([`require_pua`]), the subset lacks. Without its own positioning, every Thai glyph it + // does keep must be where the legacy order puts that character, or one of the PUA + // alternates for a mark (or for ฐ and ญ without their descenders). + let mut thai = unicode.range(0x0e01..=0x0e5b).peekable(); + if thai.peek().is_none() { return Ok(None); } - let (_, codes) = composite_glyph_maps(doc, font, &bytes, &m, &mut DecodeBudget::new(1 << 17, 0)) - .ok_or("the Thai source glyph codes cannot be recovered safely")?; - Ok(Some(Self { bytes, codes, space })) + let usable = positioned + || thai.all(|(&ch, &gid)| { + legacy_glyph(ch) == Some(gid) + || ((342..=368).contains(&gid) && (char::from_u32(ch).is_some_and(is_thai_mark) || matches!(ch, 0x0e0d | 0x0e10))) + }); + Ok(usable.then_some(Self { bytes, codes, space, positioned })) } /// Shape into glyphs that are already in this exact source face. @@ -242,8 +309,11 @@ impl LegacyThaiFont { if text.len() > 262_144 || text.contains(['\n', '\r']) { return Err("shape one bounded Thai source row at a time".into()); } - let face = skrifa::FontRef::new(&self.bytes).map_err(|e| e.to_string())?; - require_pua(&face.charmap(), text)?; + // Only the legacy shaper needs the PUA alternates; a positioned program moves its marks. + if !self.positioned { + let face = skrifa::FontRef::new(&self.bytes).map_err(|e| e.to_string())?; + require_pua(&face.charmap(), text)?; + } let mut shaped = crate::source_shaping::ShapedRun { glyphs: Vec::new(), advance: 0.0 }; let mut codes = Vec::new(); // Legacy PDF subsets often omit the space glyph. Keep its measured PDF advance as a @@ -251,7 +321,11 @@ impl LegacyThaiFont { for piece in text.split_inclusive(' ') { let word = piece.strip_suffix(' ').unwrap_or(piece); if !word.is_empty() { - let run = crate::source_shaping::shape_legacy_thai(&self.bytes, word)?; + let run = if self.positioned { + crate::source_shaping::shape(&self.bytes, word)? + } else { + crate::source_shaping::shape_legacy_thai(&self.bytes, word)? + }; for mut glyph in run.glyphs { let code = self .codes @@ -330,10 +404,176 @@ fn require_pua(cmap: &skrifa::charmap::Charmap<'_>, text: &str) -> Result<(), St #[cfg(test)] mod tests { - use super::recovered_thai_map; - use pdfcraft_cos::{Dict, Document, Object}; + use super::{LegacyThaiFont, Metrics, composite_glyph_maps, is_thai_mark, recovered_thai_map}; + use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; use std::collections::HashMap; + /// Where table `tag` starts in an sfnt. + fn table(program: &[u8], tag: &[u8; 4]) -> usize { + let count = u16::from_be_bytes([program[4], program[5]]) as usize; + let record = (0..count).map(|i| 12 + i * 16).find(|&r| &program[r..r + 4] == tag).unwrap(); + u32::from_be_bytes(program[record + 8..record + 12].try_into().unwrap()) as usize + } + + /// Set glyph `gid`'s advance (a long `hmtx` entry). + fn set_advance(program: &mut [u8], gid: usize, advance: u16) { + let long = u16::from_be_bytes(program[table(program, b"hhea") + 34..][..2].try_into().unwrap()) as usize; + assert!(gid < long, "a long hmtx entry"); + let at = table(program, b"hmtx") + gid * 4; + program[at..at + 2].copy_from_slice(&advance.to_be_bytes()); + } + + /// Empty glyph `gid`, as a subsetter drops it: its data is left as the previous glyph's padding. + fn drop_glyph(program: &mut [u8], gid: usize) { + let long = program[table(program, b"head") + 51] == 1; + let loca = table(program, b"loca"); + if long { + let next: [u8; 4] = program[loca + (gid + 1) * 4..][..4].try_into().unwrap(); + program[loca + gid * 4..][..4].copy_from_slice(&next); + } else { + let next: [u8; 2] = program[loca + (gid + 1) * 2..][..2].try_into().unwrap(); + program[loca + gid * 2..][..2].copy_from_slice(&next); + } + } + + /// Inter's outlines standing in for a THSarabunPSK subset (no proprietary font is used): the + /// glyphs where the exporter's order has a Thai mark or a mark-like alternate take no + /// advance, the others 500 units. + fn thai_ordered_program() -> Vec { + let mut program = crate::script::INTER_REGULAR.to_vec(); + for gid in 214..=368usize { + let mark = match gid { + 214..=298 => is_thai_mark(char::from_u32(gid as u32 + if gid <= 271 { 0xd2b } else { 0xd2f }).unwrap()), + 342..=368 => !matches!(gid, 342 | 357), + _ => continue, + }; + set_advance(&mut program, gid, if mark { 0 } else { 500 }); + } + program + } + + /// A THSarabunPSK Identity-H font over `program`, with `to_unicode`. + fn thai_font(doc: &mut Document, program: &[u8], to_unicode: &[u8]) -> Dict { + let tu = doc.add(Object::Stream(Stream::flate(Dict::new(), to_unicode))); + let file = doc.add(Object::Stream(Stream::flate(Dict::new(), program))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), Object::Ref(file)); + let mut desc = Dict::new(); + desc.set(b"CIDToGIDMap".to_vec(), Object::name("Identity")); + desc.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + let mut font = Dict::new(); + font.set(b"Subtype".to_vec(), Object::name("Type0")); + font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+THSarabunPSK")); + font.set(b"Encoding".to_vec(), Object::name("Identity-H")); + font.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(desc)])); + font.set(b"ToUnicode".to_vec(), Object::Ref(tu)); + font + } + + /// The exporter's damaged map: intact anchors, Latin/control destinations for the other Thai + /// codes, and the oversized range first. + const DAMAGED: &[u8] = b"begincmap 1 begincodespacerange <0000> endcodespacerange \ + 1 beginbfrange <00ff><01ff><0032> endbfrange 17 beginbfrange \ + <00d9><00d9><0e04> <00e2><00e2><0e0d> <00fa><00fa><0e25> <0100><0100><0e2b> <0102><0102><0e2d> \ + <0164><0164><0e4c> <0166><0166><0e31> <0168><0168><0e47> \ + <00d6><00d6><000f> <00f0><00f0><0029> <00f8><00f8><0023> <0105><0105><003e> <0107><0107><0040> \ + <0109><0109><003e> <0111><0111><004e> <0160><0160><004e> <0161><0161><004f> endbfrange endcmap"; + + /// The anchors identify the exporter, but the glyph order is still checked glyph by glyph: a + /// mark where a letter belongs (or the reverse) anywhere in the kept glyphs refuses the whole + /// map, and a glyph the subset dropped is left unmapped whatever its filler advance says. + #[test] + fn recovered_thai_map_checks_every_kept_glyph_against_the_font_program() { + let mut doc = Document::new_empty(); + let font = thai_font(&mut doc, &thai_ordered_program(), DAMAGED); + let recovered: HashMap<_, _> = recovered_thai_map(&doc, &font).unwrap().into_iter().collect(); + assert_eq!(recovered[&262], 'ั'); + assert_eq!(recovered[&297], '๘'); + // 259–261 are ฮ, ฯ and ะ, between the last anchor (อ, 258) and the first mark: a font that + // put Mai Han-akat one glyph earlier passes every anchor and only this check sees it. + let mut shifted = thai_ordered_program(); + set_advance(&mut shifted, 261, 0); + set_advance(&mut shifted, 262, 500); + let shifted = thai_font(&mut doc, &shifted, DAMAGED); + assert!(recovered_thai_map(&doc, &shifted).is_none(), "a kept glyph that advances unlike its character refuses the map"); + let mut dropped = thai_ordered_program(); + drop_glyph(&mut dropped, 262); + set_advance(&mut dropped, 262, 500); + let dropped = thai_font(&mut doc, &dropped, DAMAGED); + let recovered = recovered_thai_map(&doc, &dropped).unwrap(); + assert!(recovered.iter().all(|(gid, _)| *gid != 262), "a dropped glyph is not mapped"); + assert!(recovered.iter().any(|(gid, _)| *gid == 263)); + let mut unreadable = font.clone(); + let Some(Object::Array(desc)) = unreadable.get(b"DescendantFonts").cloned() else { panic!("fixture") }; + let Some(Object::Dict(mut desc)) = desc.first().cloned() else { panic!("fixture") }; + desc.remove(b"FontDescriptor"); + unreadable.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(desc)])); + assert!(recovered_thai_map(&doc, &unreadable).is_none(), "without the program nothing is verified, so nothing is mapped"); + } + + /// `program` without the tables a legacy subsetter leaves out. + fn without_tables(program: &[u8], tags: &[&[u8; 4]]) -> Vec { + let mut program = program.to_vec(); + let count = u16::from_be_bytes([program[4], program[5]]) as usize; + let records: Vec> = + program[12..12 + count * 16].chunks(16).filter(|r| !tags.iter().any(|t| &r[..4] == t.as_slice())).map(<[u8]>::to_vec).collect(); + program[4..6].copy_from_slice(&(records.len() as u16).to_be_bytes()); + for (i, record) in records.iter().enumerate() { + program[12 + i * 16..28 + i * 16].copy_from_slice(record); + } + program + } + + /// A THSarabunPSK program with its own Thai mark positioning is shaped by that layout; one + /// without it only if every Thai glyph it keeps is where the legacy order has that character + /// (the legacy shaper skips the positioning check). A subset needs no particular glyph. + #[test] + fn thai_source_programs_are_shaped_by_their_own_layout_or_as_verified_legacy() { + let unicode = b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 2 beginbfchar <0160> <0e48> <0161> <0e49> endbfchar"; + // `Some(positioned)` when the program can be shaped in place. + let mode = |program: &[u8], unicode: &[u8]| { + let mut doc = Document::new_empty(); + let font = thai_font(&mut doc, program, unicode); + LegacyThaiFont::from_pdf(&doc, &font, 0.25).unwrap().map(|f| f.positioned) + }; + let stripped = without_tables(&thai_ordered_program(), &[b"cmap", b"GSUB", b"GPOS"]); + assert_eq!(mode(&stripped, unicode), Some(false), "a legacy positional subset"); + let positioned = without_tables(&thai_ordered_program(), &[b"cmap", b"GSUB"]); + assert_eq!(mode(&positioned, unicode), Some(true), "a face with Thai mark positioning uses it"); + // ก at another glyph than the legacy order's. + let moved = b"2 beginbfrange <00d7> <010f> <0e02> <0110> <012a> <0e3f> endbfrange 1 beginbfchar <0024> <0e01> endbfchar"; + assert_eq!(mode(&stripped, moved), None, "out of the legacy order and without its own positioning"); + assert_eq!(mode(&positioned, moved), Some(true), "its own layout doesn't depend on the legacy order"); + // Some of the legacy glyphs, no ก or อ among them. + assert_eq!(mode(&stripped, b"1 beginbfrange <00d7> <00e0> <0e02> endbfrange"), Some(false)); + assert_eq!(mode(&stripped, b"1 beginbfchar <0024> <0041> endbfchar"), None, "no Thai at all"); + } + + /// The Windows PUA alternates are used only when each one the subset kept advances as its + /// form does: one mark out of place withdraws them all, so the shaper refuses instead of + /// drawing another mark. + #[test] + fn legacy_thai_alternates_need_their_glyph_order_confirmed() { + let unicode = b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 2 beginbfchar <0160> <0e48> <0161> <0e49> endbfchar"; + let alternates = |program: &[u8]| { + let mut doc = Document::new_empty(); + let font = thai_font(&mut doc, program, unicode); + let metrics = Metrics::from_dict(&doc, &font); + let (map, _) = composite_glyph_maps(&doc, &font, program, &metrics, &mut DecodeBudget::new(1 << 20, 0)).unwrap(); + map.range(0xf700..=0xf71a).map(|(u, g)| (*u, *g)).collect::>() + }; + let ordered = alternates(&thai_ordered_program()); + assert!(ordered.contains(&(0xf70a, 352)) && ordered.contains(&(0xf710, 358)), "{ordered:?}"); + let mut misplaced = thai_ordered_program(); + set_advance(&mut misplaced, 352, 500); + assert!(alternates(&misplaced).is_empty(), "one misplaced alternate withdraws them all"); + // A dropped alternate is simply absent; the others still come from the kept glyphs. + let mut dropped = thai_ordered_program(); + drop_glyph(&mut dropped, 353); + let kept = alternates(&dropped); + assert!(kept.contains(&(0xf70a, 352)) && !kept.iter().any(|(_, g)| *g == 353), "{kept:?}"); + } + #[test] fn damaged_legacy_thai_map_recovers_consonants_vowels_and_contextual_tones_only_for_its_fingerprint() { let mut doc = Document::new_empty(); @@ -342,15 +582,8 @@ mod tests { <0164><0164><0e4c> <0166><0166><0e31> <0168><0168><0e47> \ <00d6><00d6><000f> <00f0><00f0><0029> <00f8><00f8><0023> <0105><0105><003e> <0107><0107><0040> \ <0109><0109><003e> <0111><0111><004e> <0160><0160><004e> <0161><0161><004f> <00ff><01ff><0032> endbfrange endcmap"; - let tu = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), data))); - let mut desc = Dict::new(); - desc.set(b"CIDToGIDMap".to_vec(), Object::name("Identity")); - let mut font = Dict::new(); - font.set(b"Subtype".to_vec(), Object::name("Type0")); - font.set(b"BaseFont".to_vec(), Object::name("ABCDEF+THSarabunPSK")); - font.set(b"Encoding".to_vec(), Object::name("Identity-H")); - font.set(b"DescendantFonts".to_vec(), Object::Array(vec![Object::Dict(desc)])); - font.set(b"ToUnicode".to_vec(), Object::Ref(tu)); + let mut font = thai_font(&mut doc, &thai_ordered_program(), data); + let tu = font.get(b"ToUnicode").and_then(Object::as_ref).unwrap(); // The corrupt range is declared before explicit anchors in the actual exporter. let data = String::from_utf8_lossy(data) .replace("18 beginbfrange", "1 beginbfrange <00ff><01ff><0032> endbfrange 17 beginbfrange") diff --git a/crates/fonts/src/source_shaping.rs b/crates/fonts/src/source_shaping.rs index 6976bd8a0e..b1043ad994 100644 --- a/crates/fonts/src/source_shaping.rs +++ b/crates/fonts/src/source_shaping.rs @@ -22,11 +22,33 @@ pub fn required(text: &str) -> bool { .any(|c| matches!(u32::from(c), 0x0300..=0x036f | 0x0590..=0x0fff | 0x1780..=0x17ff | 0x200c..=0x200f | 0x202a..=0x202e | 0x2066..=0x2069)) } -/// Whether a Thai source program retains its mark-positioning table. +/// Whether a Thai source program retains its mark positioning. /// Character coverage alone does not show that stacked vowels and tones can be laid out. /// OpenType Thai requires `mark`/`mkmk` (GPOS); `ccmp` substitution (GSUB) is optional. pub fn has_thai_layout(bytes: &[u8]) -> bool { - FontRef::new(bytes).is_ok_and(|f| f.gpos().is_ok()) + FontRef::new(bytes).is_ok_and(|f| positions_thai_marks(&f)) +} + +/// Whether the GPOS features the shaper uses for Thai include `mark` or `mkmk`. The selection is +/// HarfRust's, through the same read-fonts calls: the `thai` script, else `DFLT`, `dflt` or +/// `latn` (`ScriptList::select`); then, given no language, a language system tagged `dflt`, +/// else the script's default one. Marks under a script or language it doesn't select don't +/// count, nor does a GPOS with only kerning. +fn positions_thai_marks(face: &FontRef) -> bool { + use skrifa::Tag; + let Ok(gpos) = face.gpos() else { return false }; + let (Ok(scripts), Ok(features)) = (gpos.script_list(), gpos.feature_list()) else { return false }; + let records = features.feature_records(); + let marks = |index: u16| records.get(usize::from(index)).is_some_and(|r| r.feature_tag() == *b"mark" || r.feature_tag() == *b"mkmk"); + let Some(selected) = scripts.select(&[Tag::new(b"thai")]) else { return false }; + let Ok(script) = scripts.get(selected.index) else { return false }; + let language = match script.lang_sys_index_for_tag(Tag::new(b"dflt")) { + Some(index) => script.lang_sys(index).ok().map(|l| l.element), + None => script.default_lang_sys().transpose().ok().flatten(), + }; + let Some(language) = language else { return false }; + language.feature_indices().iter().any(|i| marks(i.get())) + || (language.required_feature_index() != 0xFFFF && marks(language.required_feature_index())) } pub fn shape(bytes: &[u8], text: &str) -> Result { @@ -46,8 +68,8 @@ fn shape_inner(bytes: &[u8], text: &str, legacy_thai: bool) -> Result Result usize { + let count = u16::from_be_bytes([bytes[4], bytes[5]]) as usize; + let record = (0..count).map(|i| 12 + i * 16).find(|&r| &bytes[r..r + 4] == b"GPOS").unwrap(); + u32::from_be_bytes(bytes[record + 8..record + 12].try_into().unwrap()) as usize + } + + fn u16_at(bytes: &[u8], at: usize) -> usize { + u16::from_be_bytes([bytes[at], bytes[at + 1]]) as usize + } + + /// Where the GPOS ScriptList starts. + fn script_list(bytes: &[u8]) -> usize { + gpos(bytes) + u16_at(bytes, gpos(bytes) + 4) + } + + /// Where script `tag`'s table starts. + fn script(bytes: &[u8], tag: &[u8; 4]) -> usize { + let list = script_list(bytes); + let record = (0..u16_at(bytes, list)).map(|i| list + 2 + i * 6).find(|&r| &bytes[r..r + 4] == tag).unwrap(); + list + u16_at(bytes, record + 4) + } + + /// Retag script `from` as `to`, keeping the records sorted by tag as the shaper's binary + /// search needs. + fn retag_script(bytes: &mut [u8], from: &[u8; 4], to: &[u8; 4]) { + let list = script_list(bytes); + let count = u16_at(bytes, list); + let mut records: Vec<[u8; 6]> = (0..count).map(|i| bytes[list + 2 + i * 6..list + 8 + i * 6].try_into().unwrap()).collect(); + records.iter_mut().find(|r| &r[..4] == from).unwrap()[..4].copy_from_slice(to); + records.sort_by(|a, b| a[..4].cmp(&b[..4])); + for (i, record) in records.iter().enumerate() { + bytes[list + 2 + i * 6..list + 8 + i * 6].copy_from_slice(record); + } + } + + /// A shaper uses a font's `thai` script when it has one: marks the font registers only for + /// Latin don't position Thai. + #[test] + fn thai_layout_counts_only_the_script_the_shaper_selects() { + let mut bytes = crate::script::INTER_REGULAR.to_vec(); + // Inter's DFLT becomes a `thai` script whose default language system has no features; + // its latn script keeps `mark` and `mkmk`. + retag_script(&mut bytes, b"DFLT", b"thai"); + let thai = script(&bytes, b"thai"); + let language = thai + u16_at(&bytes, thai); + assert_ne!(language, thai, "a default language system"); + bytes[language + 2..language + 6].copy_from_slice(&[0xFF, 0xFF, 0, 0]); + assert!(!has_thai_layout(&bytes), "the latn script's marks are not the thai script's"); + } + + /// Given no language, the shaper takes a language system tagged `dflt` before the script's + /// default one, in both directions. + #[test] + fn thai_layout_follows_a_language_system_tagged_dflt() { + // Without DFLT the shaper falls back to latn, whose default language system has marks and + // is shared by its CAT, MOL and ROM records. Cyrillic's default one has only `kern`. + let mut base = crate::script::INTER_REGULAR.to_vec(); + retag_script(&mut base, b"DFLT", b"zzzz"); + assert!(has_thai_layout(&base)); + let latn = script(&base, b"latn"); + let kern_only = script(&base, b"cyrl") + u16_at(&base, script(&base, b"cyrl")); + let kern_only = u16::try_from(kern_only - latn).unwrap().to_be_bytes(); + let rom = (0..u16_at(&base, latn + 2)).map(|j| latn + 4 + j * 6).find(|&r| &base[r..r + 4] == b"ROM ").unwrap(); + // ROM becomes `dflt` (still sorted last) and has only kerning: it wins over the default. + let mut named_without = base.clone(); + named_without[rom..rom + 4].copy_from_slice(b"dflt"); + named_without[rom + 4..rom + 6].copy_from_slice(&kern_only); + assert!(!has_thai_layout(&named_without), "the dflt language system has no marks"); + // The default has only kerning and the `dflt` one keeps the marks: it wins again. + let mut default_without = base.clone(); + default_without[rom..rom + 4].copy_from_slice(b"dflt"); + default_without[latn..latn + 2].copy_from_slice(&kern_only); + assert!(has_thai_layout(&default_without), "the dflt language system has the marks"); + } + + /// A GPOS table alone is not mark positioning: with its `mark` and `mkmk` features renamed, + /// as in a kerning-only subset, a face can't stack Thai vowels and tones. + #[test] + fn thai_layout_needs_mark_positioning_features() { + let inter = crate::script::INTER_REGULAR; + assert!(has_thai_layout(inter), "Inter positions marks under its default script"); + let mut bytes = inter.to_vec(); + let table = gpos(&bytes); + let features = table + u16_at(&bytes, table + 6); + let mut renamed = 0; + for i in 0..u16_at(&bytes, features) { + let tag = features + 2 + i * 6; + if matches!(&bytes[tag..tag + 4], b"mark" | b"mkmk") { + bytes[tag..tag + 4].copy_from_slice(b"kern"); + renamed += 1; + } + } + assert!(renamed > 0); + assert!(FontRef::new(&bytes).unwrap().gpos().is_ok(), "the GPOS table still parses"); + assert!(!has_thai_layout(&bytes)); + let error = shape(&bytes, "กิ่").unwrap_err(); + assert!(error.contains("mark positioning"), "{error}"); + } + #[test] fn thai_positioning_does_not_require_optional_gsub() { - let Some(face) = crate::document_thai_font() else { return }; + let Some(face) = crate::document_thai_font() else { + eprintln!("skipping thai_positioning_does_not_require_optional_gsub: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + }; let mut bytes = face.bytes.to_vec(); // Keep the actual GPOS anchors and outlines while removing optional substitutions. // Table payloads stay at their original offsets; no font fixture is committed. diff --git a/docs/editing-existing-text.md b/docs/editing-existing-text.md index a8966ff207..b7716cf4d9 100644 --- a/docs/editing-existing-text.md +++ b/docs/editing-existing-text.md @@ -5,11 +5,16 @@ page in an undoable **Prepare grouped text** step. The group's original operator placement and resources become page content; shared forms on other pages stay unchanged. Headless callers use `text_prepare`, then list `text_paragraphs` and call `text_edit`. This edits existing glyph operators; it adds neither a cover rectangle nor an Add Text box. -Preparation refuses knockout/blended groups, incompatible colour spaces, tagged forms, -recursive forms, unfinished paths and unreadable content. Decoding is capped at 16 MiB, -64 placements and eight expansion levels per page. Unsupported groups remain intact. -A narrowly fingerprinted legacy THSarabunPSK exporter map can also be repaired in copied page -fonts; the repair log records both preparation and character-map recovery. +Preparation refuses knockout/blended groups, groups that blend in a colour space the page +group does not also name (including DeviceRGB on a page that names none), tagged forms, +recursive forms, unfinished paths and unreadable content. Forms that draw only images or +vector artwork are left grouped. Decoding is capped at 16 MiB, 64 placements and eight +expansion levels per page; text grouped deeper is reported. Unsupported groups remain intact: +`text_prepare` lists each with its reason, and the app records them in the repair log when it +prepares other groups on the page. When no group on a page can be prepared, the app says +nothing; the text stays grouped. A narrowly fingerprinted legacy THSarabunPSK exporter map +can also be repaired in copied page fonts, only when every glyph the subset kept advances as +its assumed Thai character does; the repair log records both preparation and map recovery. Click a text region in **Edit a PDF → Edit text and images**. One editor contains its source rows and mixed styles. Embedded TrueType and OpenType/CFF faces are used in memory; @@ -59,7 +64,9 @@ add a font binary or change the release font pin. Fonts shared by releases belon THSarabunPSK subsets, PdfCraft reconstructs an editor-only Unicode cmap from the PDF's Unicode and CID-to-glyph maps. HarfRust uses the retained Thai positional glyphs, and the saved text refers to the original PDF font resources. Several subsets of the same exact -face can supply Thai and punctuation glyphs; Regular and Bold stay separate. Source font +face can supply Thai and punctuation glyphs, including a Thai run whose clusters are split +across subsets, and a subset whose glyph for a character is empty is skipped; Regular and +Bold stay separate. Source font programs are never modified or replaced. Text settings, including character/word spacing, horizontal scale, rise, color and paint mode, remain in the PDF text state. @@ -68,14 +75,25 @@ complete original face or an explicit replacement. Cross-family replacement requ explicit choice. The document-font menu offers **Noto Sans Thai**: pass its `text_fonts` resource (normally `builtin:thai`) as `text_edit.font_resource` or a range's `font_resource`. This replacement requires the optional craft-fonts Thai input and currently uses Regular. A matching Noto -subset can still be completed by that same bundled family. +subset can still be completed by that same bundled family when it is Regular and upright; a +subset its name or descriptor marks bold or italic needs an explicit replacement. The bundled variable Thai face is written as bounded Type 3 glyph outlines, with shaped vowel/tone positions and logical Unicode text, rather than embedded as a static font. Saved paragraphs can be reopened and edited again without installing a font. Modern Thai -shaping uses the source's GPOS mark positioning; GSUB substitution is optional, as specified by +shaping uses the source's GPOS mark positioning (a `mark` or `mkmk` feature in the script and language system the shaper +selects for Thai, not just a GPOS table); GSUB substitution is optional, as specified by [the OpenType Thai model](https://learn.microsoft.com/en-us/typography/script-development/thai). -Verified legacy Thai PUA fonts use positional substitutions instead of requiring GPOS. +Verified legacy Thai PUA fonts use positional substitutions instead of requiring GPOS; their +alternates are used only when each kept one has a mark's zero advance (or, for the two +consonant forms, an advance). A THSarabunPSK program with its own Thai mark +positioning, such as a complete face loaded with Load font, is shaped by that layout instead, +still in its own codes. One without it takes the legacy path only when every Thai glyph it keeps +is where the legacy glyph order puts it, and is otherwise shaped as an ordinary source font. A +subset needs only the glyphs its new text uses. When no subset can show the new text in its +existing codes, a complete program of the same face and style (one added with Load font, CFF +included) shapes the whole span with its own layout, Latin rows too, rather than the edit being +refused; a Regular program never completes text the source marks bold or italic. Mixed contextual style runs and dictionary word segmentation remain limited. No font binary or release pin changes are needed. diff --git a/docs/roadmap.md b/docs/roadmap.md index 28afeea246..2b42b9ba9c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # PdfCraft roadmap: milestones and what's next -> **Last reviewed:** 2026-11-05 | **Last updated:** 2026-11-05 | **Change:** patch (M7 #2964 / #3185: grouped source text and Thai font/style preservation; estimates unchanged) +> **Last reviewed:** 2026-11-07 | **Last updated:** 2026-11-07 | **Change:** patch (M7 #2964 / #3185: grouped source text and Thai font/style preservation, with reported deep or unsafe groups and a per-glyph check of the Thai map repair; estimates unchanged) > **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged) diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 54aa9561b5..b752fe76b5 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1895,8 +1895,8 @@ milestone = "M7" status = "partial" commands = ["edit.edit_text"] tools = ["text_prepare", "text_lines", "text_paragraphs", "text_edit", "text_edit_layout", "text_font_files", "text_font_import", "text_fonts"] -evidence = ["crates/edit/src/tests.rs::editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen", "crates/fonts/src/pdf/tests.rs::composite_subset_without_cmap_uses_pdf_glyph_ids_in_editor_and_shaper_without_replacing_the_program", "crates/automation/tests/it/automation.rs::grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_glyphs", "crates/edit/src/tests.rs::grouped_text_preparation_keeps_unsafe_compositing_paths_and_malformed_forms_intact", "crates/automation/tests/it/automation.rs::explicit_bundled_thai_font_choice_layout_edit_undo_and_reopen", "crates/edit/src/tests.rs::existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font", "crates/edit/src/tests.rs::paragraphs_are_found_and_rewrapped", "crates/edit/src/tests.rs::text_lines_are_found_and_replaced_in_place", "crates/ui-egui/tests/it/editing.rs::editing_existing_text_in_place", "crates/automation/tests/it/automation.rs::editing_existing_text_through_tools", "crates/edit/src/tests.rs::a_line_whose_operator_starts_the_next_stream_can_be_edited", "crates/automation/tests/it/automation.rs::a_line_split_across_content_streams_is_listed_and_edited", "crates/ui-egui/tests/it/editing.rs::selected_text_formatting_keeps_the_rest_of_the_paragraph_and_reopens", "crates/edit/src/tests.rs::marked_paragraph_boundaries_and_cross_stream_transforms_survive_rewrite", "crates/automation/tests/it/automation.rs::explicit_font_and_disjoint_ranges_save_reopen_through_tools", "crates/edit/src/tests.rs::disjoint_style_ranges_preserve_unselected_runs_and_validate_utf8_atomically", "crates/engine/tests/it/document_fonts.rs::opentype_cff_document_font_preserves_program_codes_and_rendering_after_reopen", "crates/ui-egui/tests/it/editing.rs::cropped_images_do_not_intercept_text_and_stay_inside_each_page", "crates/automation/tests/it/automation.rs::clipped_image_bounds_and_text_edit_save_reopen_through_tools", "crates/edit/src/tests.rs::mixed_styles_and_matrix_sizes_form_visual_regions_without_merging_columns", "crates/edit/src/tests.rs::rich_region_edits_preserve_styles_positions_and_delete_all_runs_after_reopen", "crates/ui-egui/tests/it/editing.rs::one_editor_contains_mixed_styles_and_visual_rows", "crates/ui-egui/tests/it/editing.rs::inline_editor_preserves_source_font_and_previews_live_bounds", "crates/fonts/src/pdf/tests.rs::mac_subset_cmap_uses_pdf_unicode_without_changing_source_program", "crates/automation/tests/it/automation.rs::mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools", "crates/ui-egui/tests/it/ui.rs::the_edit_tab_turns_on_edit_text_and_images", "crates/ui-egui/tests/it/editing.rs::the_caret_on_a_small_row_is_as_tall_as_that_row", "crates/edit/src/tests.rs::uzbek_and_kazakh_cyrillic_use_the_bundled_latin_face", "crates/automation/tests/it/automation.rs::editing_uzbek_and_kazakh_cyrillic_through_tools", "crates/edit/src/tests.rs::styled_paragraph_batches_cyrillic_without_changing_source_fonts", "crates/edit/src/tests.rs::styled_paragraph_over_240_fallback_glyphs_splits_and_refuses_atomically", "crates/edit/src/tests.rs::styled_fallback_batches_keep_face_choices_and_range_styles", "crates/edit/src/tests.rs::styled_fallback_shards_keep_text_advances_and_alignment", "crates/edit/src/tests.rs::styled_fallback_budget_is_enforced_before_objects_are_written", "crates/edit/src/tests.rs::styled_fallback_same_base_font_and_character_keep_both_weights", "crates/automation/tests/it/automation.rs::styled_cyrillic_fallback_is_batched_through_tools_and_reopen", "crates/ui-egui/tests/it/editing.rs::escape_in_an_open_paragraph_selects_its_box_for_delete"] -notes = "Supported grouped Forms are prepared on entering Edit text, or via text_prepare, so their existing source glyphs can be edited directly. Unsafe groups remain intact. Narrow legacy THSarabunPSK character-map recovery is recorded. Legacy Thai source editing reuses original font programs, positional glyphs and same-face subsets, preserving Regular/Bold, point size, spacing and paint. A different Thai family requires explicit selection; the Noto replacement is Regular only; mixed contextual style runs remain limited. The Edit tab turns Edit text & images on at once, as in Acrobat (leaving the tab puts the tool down and applies an edit in progress); Esc in an open paragraph leaves its text and selects its box, so Delete removes it (the unapplied draft is discarded); click a paragraph box and retype it in place (font reused or Helvetica substituted, colour and spacing kept, rewrapped); a page's /Contents pieces are parsed joined, so a line whose operands and operator sit in different pieces is listed and edited (lines and paragraphs still don't join across pieces; Edit image still parses each piece alone); drag a box to move it or its right-edge handle to rewrap it (edit.text-move-resize); per-character formatting changes are next Since #1116: mixed-style visual regions retain source rows, subset encodings, colour and baseline anchors in one textless-background editor. Explicit paragraph/heading scopes constrain grouping; compatible graphics planes permit cross-stream regions. Full matching document fonts complete absent subset glyphs; installed/file selection, TrueType/OpenType-CFF embedding and disjoint UTF-8 range styles have save/reopen coverage. Single-face OpenType shaping/bidi retains logical text through ToUnicode/ActualText. Replacement text the paragraph's font can't show takes the craft-fonts Japanese faces when they draw all of it (Uzbek қ, Kazakh ә are in none of them), else the bundled Inter face, so those letters write in every build (#2593). Legacy non-SFNT preview, mixed-font/style contextual runs, full structure-tree preservation and author intent in untagged PDFs remain limited; Styled-path outline fallback batches by actual face with 240 glyphs per font and preserves source styles; adjacent equal-font/style characters form runs. At most 15360 distinct fallback glyphs per styled paragraph; missing glyphs are refused before fallback objects are written. Glyph coverage, mixed contextual shaping, single-style fallback and stale-font cleanup are unchanged; see docs/editing-existing-text.md" +evidence = ["crates/edit/src/tests.rs::editing_legacy_thai_keeps_the_embedded_face_weight_size_spacing_and_paint_after_reopen", "crates/edit/src/tests.rs::editing_legacy_thai_shapes_a_run_across_subsets_of_the_same_face", "crates/edit/src/tests.rs::grouped_text_nested_past_the_level_limit_is_reported", "crates/edit/src/tests.rs::grouped_image_artwork_is_not_prepared", "crates/edit/src/tests.rs::grouped_text_in_an_rgb_group_needs_a_page_that_blends_in_rgb", "crates/fonts/src/pdf/thai.rs::recovered_thai_map_checks_every_kept_glyph_against_the_font_program", "crates/automation/tests/it/automation.rs::text_prepare_refuses_pages_without_grouped_text", "crates/fonts/src/pdf/tests.rs::composite_subset_without_cmap_uses_pdf_glyph_ids_in_editor_and_shaper_without_replacing_the_program", "crates/automation/tests/it/automation.rs::grouped_existing_thai_body_edits_saves_reopens_and_undo_restores_original_glyphs", "crates/edit/src/tests.rs::grouped_text_preparation_keeps_unsafe_compositing_paths_and_malformed_forms_intact", "crates/automation/tests/it/automation.rs::explicit_bundled_thai_font_choice_layout_edit_undo_and_reopen", "crates/edit/src/tests.rs::existing_thai_subset_uses_shaped_matching_face_without_embedding_variable_font", "crates/edit/src/tests.rs::paragraphs_are_found_and_rewrapped", "crates/edit/src/tests.rs::text_lines_are_found_and_replaced_in_place", "crates/ui-egui/tests/it/editing.rs::editing_existing_text_in_place", "crates/automation/tests/it/automation.rs::editing_existing_text_through_tools", "crates/edit/src/tests.rs::a_line_whose_operator_starts_the_next_stream_can_be_edited", "crates/automation/tests/it/automation.rs::a_line_split_across_content_streams_is_listed_and_edited", "crates/ui-egui/tests/it/editing.rs::selected_text_formatting_keeps_the_rest_of_the_paragraph_and_reopens", "crates/edit/src/tests.rs::marked_paragraph_boundaries_and_cross_stream_transforms_survive_rewrite", "crates/automation/tests/it/automation.rs::explicit_font_and_disjoint_ranges_save_reopen_through_tools", "crates/edit/src/tests.rs::disjoint_style_ranges_preserve_unselected_runs_and_validate_utf8_atomically", "crates/engine/tests/it/document_fonts.rs::opentype_cff_document_font_preserves_program_codes_and_rendering_after_reopen", "crates/ui-egui/tests/it/editing.rs::cropped_images_do_not_intercept_text_and_stay_inside_each_page", "crates/automation/tests/it/automation.rs::clipped_image_bounds_and_text_edit_save_reopen_through_tools", "crates/edit/src/tests.rs::mixed_styles_and_matrix_sizes_form_visual_regions_without_merging_columns", "crates/edit/src/tests.rs::rich_region_edits_preserve_styles_positions_and_delete_all_runs_after_reopen", "crates/ui-egui/tests/it/editing.rs::one_editor_contains_mixed_styles_and_visual_rows", "crates/ui-egui/tests/it/editing.rs::inline_editor_preserves_source_font_and_previews_live_bounds", "crates/fonts/src/pdf/tests.rs::mac_subset_cmap_uses_pdf_unicode_without_changing_source_program", "crates/automation/tests/it/automation.rs::mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools", "crates/ui-egui/tests/it/ui.rs::the_edit_tab_turns_on_edit_text_and_images", "crates/ui-egui/tests/it/editing.rs::the_caret_on_a_small_row_is_as_tall_as_that_row", "crates/edit/src/tests.rs::uzbek_and_kazakh_cyrillic_use_the_bundled_latin_face", "crates/automation/tests/it/automation.rs::editing_uzbek_and_kazakh_cyrillic_through_tools", "crates/edit/src/tests.rs::styled_paragraph_batches_cyrillic_without_changing_source_fonts", "crates/edit/src/tests.rs::styled_paragraph_over_240_fallback_glyphs_splits_and_refuses_atomically", "crates/edit/src/tests.rs::styled_fallback_batches_keep_face_choices_and_range_styles", "crates/edit/src/tests.rs::styled_fallback_shards_keep_text_advances_and_alignment", "crates/edit/src/tests.rs::styled_fallback_budget_is_enforced_before_objects_are_written", "crates/edit/src/tests.rs::styled_fallback_same_base_font_and_character_keep_both_weights", "crates/automation/tests/it/automation.rs::styled_cyrillic_fallback_is_batched_through_tools_and_reopen", "crates/ui-egui/tests/it/editing.rs::escape_in_an_open_paragraph_selects_its_box_for_delete"] +notes = "Supported grouped Forms are prepared on entering Edit text, or via text_prepare, so their existing source glyphs can be edited directly. Unsafe groups (including RGB-blended groups on a page that names no blending space) remain intact and are reported by text_prepare (the app records them in the repair log only when it prepares other groups on the page, and says nothing when it can prepare none), as is text grouped more than eight levels deep; image-only forms are left grouped. Narrow legacy THSarabunPSK character-map recovery, checked glyph by glyph against the subset's own advances, is recorded. Legacy Thai source editing reuses original font programs, positional glyphs and same-face subsets (a Thai run split across subsets is shaped cluster by cluster; what no subset can show in its codes is shaped whole by a complete program of the face, such as one from Load font), preserving Regular/Bold, point size, spacing and paint. A different Thai family requires explicit selection; the Noto replacement is Regular only; mixed contextual style runs remain limited. The Edit tab turns Edit text & images on at once, as in Acrobat (leaving the tab puts the tool down and applies an edit in progress); Esc in an open paragraph leaves its text and selects its box, so Delete removes it (the unapplied draft is discarded); click a paragraph box and retype it in place (font reused or Helvetica substituted, colour and spacing kept, rewrapped); a page's /Contents pieces are parsed joined, so a line whose operands and operator sit in different pieces is listed and edited (lines and paragraphs still don't join across pieces; Edit image still parses each piece alone); drag a box to move it or its right-edge handle to rewrap it (edit.text-move-resize); per-character formatting changes are next Since #1116: mixed-style visual regions retain source rows, subset encodings, colour and baseline anchors in one textless-background editor. Explicit paragraph/heading scopes constrain grouping; compatible graphics planes permit cross-stream regions. Full matching document fonts complete absent subset glyphs; installed/file selection, TrueType/OpenType-CFF embedding and disjoint UTF-8 range styles have save/reopen coverage. Single-face OpenType shaping/bidi retains logical text through ToUnicode/ActualText. Replacement text the paragraph's font can't show takes the craft-fonts Japanese faces when they draw all of it (Uzbek қ, Kazakh ә are in none of them), else the bundled Inter face, so those letters write in every build (#2593). Legacy non-SFNT preview, mixed-font/style contextual runs, full structure-tree preservation and author intent in untagged PDFs remain limited; Styled-path outline fallback batches by actual face with 240 glyphs per font and preserves source styles; adjacent equal-font/style characters form runs. At most 15360 distinct fallback glyphs per styled paragraph; missing glyphs are refused before fallback objects are written. Glyph coverage, mixed contextual shaping, single-style fallback and stale-font cleanup are unchanged; see docs/editing-existing-text.md" [[feature]] id = "edit.text-reflow" From a453112151c6ef5dc44fdcb80573dcb9c09586bd Mon Sep 17 00:00:00 2001 From: xullul Date: Sat, 7 Nov 2026 17:00:48 +0700 Subject: [PATCH 08/37] M7: read each grouped form once and check PUA alternate meanings Change: - Leave a form found to be artwork, or refused for a reason in the form itself, out of later preparation passes, so a page with many logos or unsupported groups no longer reaches the 64-placement limit and prepares nothing. ungroup_ops now marks each refusal as intrinsic or contextual; contextual ones (translucency, the resources it would merge into, pattern placement, the decode budget) are re-evaluated where the form appears, so a form refused in one place is still prepared where a parent draws it. - Refuse a page-level Do with an operand too many for that placement only: the form found by its first operand may not be the one drawn. Check a form's own operand counts before the nested walk, as an intrinsic refusal. - Use the Windows PUA alternates only when every kept alternate whose code the PDF's map names a Thai character names that alternate's base character, so alternates in another order that the map reveals are refused. Chores: - Say that the font program confirms each recovered glyph's advance class, not its identity, and document the alternate check. Validation: with and without craft-fonts, fonts 105 (+3 build-input), edit 468, engine 482 (+28 integration) and automation 50 + 352 passed; editor flows 248 passed. Formatting, strict Clippy, layers, assets and parity passed; the wasm check was not run locally. Each new regression failed with its fix reverted. Real-PDF edits on Royal Gazette 26288 kept their source fonts. --- crates/edit/src/tests.rs | 74 ++++++++++++++++++++++++++++ crates/edit/src/text/forms.rs | 35 ++++++++++--- crates/edit/src/text/forms/expand.rs | 67 +++++++++++++++---------- crates/fonts/src/pdf/thai.rs | 34 +++++++++++-- docs/editing-existing-text.md | 3 +- 5 files changed, 176 insertions(+), 37 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index e425d00328..d3c0e4fdcd 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -10947,6 +10947,80 @@ fn grouped_image_artwork_is_not_prepared() { assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "inner")); } +/// Artwork and groups refused for what they are (knockout groups, content with an operand too +/// many) are read once, not again on every pass: 10 logo forms and 30 such groups beside a text +/// group are 41 reads, within the 64-placement limit, where reading either again on the second +/// pass would pass it and prepare nothing. +#[test] +fn artwork_and_refused_groups_count_once_against_the_placement_limit() { + for knockout in [true, false] { + let refused_body = if knockout { "BT /F1 14 Tf 40 500 Td (kept) Tj ET" } else { "BT /F1 14 Tf 40 500 Td (kept) Tj ET 0 /Im Do" }; + let mut forms: Vec<(String, String)> = (1..=10).map(|i| (format!("Logo{i}"), "q 10 0 0 10 0 0 cm /Im Do Q".to_string())).collect(); + forms.extend((1..=30).map(|i| (format!("Refused{i}"), refused_body.to_string()))); + forms.push(("Body".into(), "BT /F1 14 Tf 40 300 Td (body) Tj ET".into())); + let content: String = forms.iter().map(|(name, _)| format!("/{name} Do\n")).collect(); + let mut doc = page_with_forms(&content, &forms); + // Forms are objects 7 onwards, in order: the refused groups follow the 10 logos. + for i in 0..30u32 { + let r = pdfcraft_cos::ObjRef::new(7 + 10 + i, 0); + let Object::Stream(mut s) = doc.get(r).as_ref().clone() else { panic!("fixture") }; + if knockout { + let group = build(&["<< /S /Transparency /K true >>"]).get(pdfcraft_cos::ObjRef::new(1, 0)).as_dict().unwrap().clone(); + s.dict.set(b"Group".to_vec(), Object::Dict(group)); + } + doc.set(r, Object::Stream(s)); + } + let result = text::forms::prepare(&mut doc, 0).unwrap_or_else(|e| panic!("knockout {knockout}: each group is read once ({e})")); + assert_eq!(result.expanded, 1); + assert_eq!(result.refused.len(), 30, "{:?}", result.refused); + assert!(text::text_blocks(&doc, 0).unwrap().iter().any(|b| b.text == "body")); + } +} + +/// A page-level `Do` with an operand too many is left as it is and settles nothing: the form +/// found by its first operand may not be the one the renderer draws (its last). Here `/A /B Do` +/// draws the knockout group B, and A, a text group, is still prepared where Parent draws it. +#[test] +fn a_page_do_with_an_operand_too_many_settles_no_form() { + let forms = [ + ("A".to_string(), "BT /F1 14 Tf 40 500 Td (alpha) Tj ET".to_string()), + ("B".to_string(), "BT /F1 14 Tf 40 400 Td (beta) Tj ET".to_string()), + ("Parent".to_string(), "/A Do".to_string()), + ]; + let mut doc = page_with_forms("/A /B Do\n/Parent Do", &forms); + let r = pdfcraft_cos::ObjRef::new(8, 0); + let Object::Stream(mut s) = doc.get(r).as_ref().clone() else { panic!("fixture") }; + let group = build(&["<< /S /Transparency /K true >>"]).get(pdfcraft_cos::ObjRef::new(1, 0)).as_dict().unwrap().clone(); + s.dict.set(b"Group".to_vec(), Object::Dict(group)); + doc.set(r, Object::Stream(s)); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 2, "Parent, then the A it draws: {:?}", result.refused); + assert!(result.refused.iter().any(|r| r.contains("an operand too many")), "{:?}", result.refused); +} + +/// A refusal that comes from where a form is drawn is not remembered for the form: a text group +/// refused where it is drawn translucent is still prepared where an expanded parent draws it +/// opaque, on the next pass. +#[test] +fn a_placement_refusal_does_not_settle_the_form_for_later_passes() { + let resources = "<< /Font << /F1 5 0 R >> /ExtGState << /Alpha << /ca 0.5 >> >> /XObject << /Shared 6 0 R /Parent 7 0 R >> >>"; + let form = |extra: &str, body: &str| stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 612 792] {extra} /Resources {resources}"), body); + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {resources} /Contents 4 0 R >>"), + stream("", "q /Alpha gs /Shared Do Q /Parent Do"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + form("/Group << /S /Transparency >>", "BT /F1 14 Tf 40 500 Td (shared) Tj ET"), + form("", "/Shared Do"), + ]; + let mut doc = build(&objs.iter().map(String::as_str).collect::>()); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 2, "Parent, then the Shared it draws opaque: {:?}", result.refused); + assert!(result.refused.iter().any(|r| r.starts_with("Shared: ") && r.contains("translucent")), "{:?}", result.refused); + assert_eq!(text::text_blocks(&doc, 0).unwrap().iter().filter(|b| b.text == "shared").count(), 1, "only the opaque one is the page's own text"); +} + /// The `Do` is matched to its resource by the name's bytes as written, which need not be UTF-8. #[test] fn grouped_text_under_a_non_utf8_resource_name_is_prepared() { diff --git a/crates/edit/src/text/forms.rs b/crates/edit/src/text/forms.rs index dff7440145..da46a16d91 100644 --- a/crates/edit/src/text/forms.rs +++ b/crates/edit/src/text/forms.rs @@ -32,6 +32,11 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result Result pending_path = true, @@ -59,23 +65,33 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result {} } let Some(image) = images.get(&at) else { continue }; - visits = visits.saturating_add(1); - if visits > 64 { - return Err(EditError::Invalid("too many grouped text placements on this page (limit 64); the page was left unchanged".into())); + if image.object.is_some_and(|o| settled.contains(&o)) { + continue; } if pending_path || in_text { result.refused.push(format!("{}: the group is drawn inside an unfinished path or text object", image.name)); continue; } + // The renderer draws a `Do`'s last operand; `image` was found by its first. With more + // than one they can be different forms, so this placement is left as it is. + if op.operands.len() != 1 { + result.refused.push(format!("{}: the group is drawn with an operand too many", image.name)); + continue; + } + visits = visits.saturating_add(1); + if visits > 64 { + return Err(EditError::Invalid("too many grouped text placements on this page (limit 64); the page was left unchanged".into())); + } let translucent = expand::translucent_at(&next, &merged, &ops, at); - // The resource name exactly as the `Do` writes it: a name need not be UTF-8, and - // `image.name` is only its display form. + // The resource name exactly as the `Do` writes it (its only operand): a name need not + // be UTF-8, and `image.name` is only its display form. let name = op.last_name().map_or_else(|| image.name.as_bytes().to_vec(), <[u8]>::to_vec); match expand::ungroup_ops(&next, &merged, &name, translucent, space.as_ref(), Matrix(image.matrix), &mut decode) { Ok(expand::Ungrouped { ops: inline, resources: mut res, patterns, shows_text }) => { // Vector and image artwork needs no preparation. Nested text groups are // visited on the next pass, with their inherited placement and clip in force. if !shows_text { + settled.extend(image.object); continue; } if pass == MAX_LEVELS { @@ -96,10 +112,17 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result { + result.refused.push(format!("{}: {}", image.name, refusal.error)); + if refusal.intrinsic { + refused_here.extend(image.object); + } } - Err(error) => result.refused.push(format!("{}: {error}", image.name)), } } + settled.extend(refused_here.difference(&expanded_here).copied()); if replacements.is_empty() { break; } diff --git a/crates/edit/src/text/forms/expand.rs b/crates/edit/src/text/forms/expand.rs index bf01e201d1..54495900cc 100644 --- a/crates/edit/src/text/forms/expand.rs +++ b/crates/edit/src/text/forms/expand.rs @@ -59,6 +59,16 @@ pub(crate) struct Ungrouped { pub(crate) shows_text: bool, } +/// Why [`ungroup_ops`] left a placement grouped. +pub(crate) struct Refusal { + pub(crate) error: EditError, + /// The form itself is the reason (malformed, tagged, a knockout group, and so on), so every + /// placement of it is refused the same way. Otherwise something about this placement or the + /// page so far is: its translucency or matrix, the resources it would merge into (which grow + /// as other groups are prepared), or the decode budget left. + pub(crate) intrinsic: bool, +} + /// Ungroup form `name` from the page's `resources`. `translucent` says whether the graphics state /// at the `Do` is translucent, blended or soft-masked, which a transparency group composites as a /// whole. `page_space` is the page group's blending colour space ([`page_blend_space`]). @@ -72,8 +82,12 @@ pub(crate) fn ungroup_ops( page_space: Option<&Object>, placement: Matrix, budget: &mut DecodeBudget, -) -> Result { - let refuse = |why: &str| EditError::Invalid(format!("this artwork can't be ungrouped: {why}")); +) -> Result { + let refusal = |why: &str, intrinsic: bool| Refusal { error: EditError::Invalid(format!("this artwork can't be ungrouped: {why}")), intrinsic }; + // Something about this placement or the page so far is the reason. + let refuse = |why: &str| refusal(why, false); + // The form itself is the reason: every placement of it is refused the same way. + let invalid = |why: &str| refusal(why, true); let xobjects = resources.get(b"XObject").map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).ok_or_else(|| refuse("it is missing"))?; let form = xobjects.get(name).ok_or_else(|| refuse("it is missing"))?; let Object::Stream(s) = &*doc.resolve(form) else { return Err(refuse("it is not a Form XObject")) }; @@ -88,32 +102,32 @@ pub(crate) fn ungroup_ops( let v: Vec = a.as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == n && v.iter().all(|x| x.is_finite())).then_some(v) }; - let bbox = numbers(b"BBox", 4).ok_or_else(|| refuse("its bounding box is missing or unreadable"))?; + let bbox = numbers(b"BBox", 4).ok_or_else(|| invalid("its bounding box is missing or unreadable"))?; let matrix = match s.dict.get(b"Matrix") { None => None, Some(_) => Some(Matrix( - <[f64; 6]>::try_from(numbers(b"Matrix", 6).ok_or_else(|| refuse("its matrix is unreadable"))?) - .map_err(|_| refuse("its matrix is unreadable"))?, + <[f64; 6]>::try_from(numbers(b"Matrix", 6).ok_or_else(|| invalid("its matrix is unreadable"))?) + .map_err(|_| invalid("its matrix is unreadable"))?, )), }; let own = s.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()); if s.dict.get(b"Resources").is_some() && own.is_none() { - return Err(refuse("its resources cannot be read")); + return Err(invalid("its resources cannot be read")); } // A form without its own resources uses the ones it is drawn with (PDF 1.1). let form_res = own.clone().unwrap_or_else(|| resources.clone()); // A tagged form's content is a structure element's: drawn inline it would lose that link. if s.dict.get(b"StructParent").is_some() || s.dict.get(b"StructParents").is_some() { - return Err(refuse("it is tagged content (Tags panel); edit its text in a copy without tags")); + return Err(invalid("it is tagged content (Tags panel); edit its text in a copy without tags")); } let group = s.dict.get(b"Group").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).filter(|g| is_name(doc, g, b"S", b"Transparency")); if s.dict.get(b"Group").is_some() && group.is_none() { - return Err(refuse("its group dictionary cannot be read")); + return Err(invalid("its group dictionary cannot be read")); } if let Some(group) = &group { if group.get(b"K").is_some_and(|k| matches!(*doc.resolve(k), Object::Bool(true))) { - return Err(refuse("it is a knockout transparency group")); + return Err(invalid("it is a knockout transparency group")); } if translucent { return Err(refuse("it is a transparency group drawn translucent, blended or under a soft mask")); @@ -123,7 +137,7 @@ pub(crate) fn ungroup_ops( if let Some(cs) = group.get(b"CS") && !same_blend_space(doc, cs, page_space) { - return Err(refuse("it is a transparency group that blends in a colour space of its own")); + return Err(invalid("it is a transparency group that blends in a colour space of its own")); } } @@ -131,9 +145,17 @@ pub(crate) fn ungroup_ops( let data = budget.decode_strict(s).map_err(|e| refuse(&format!("its content can't be read ({e})")))?; let parsed = parse(&data); if parsed.skipped > 0 { - return Err(refuse("its content has parts that can't be read")); + return Err(invalid("its content has parts that can't be read")); } let mut ops = parsed.ops; + // A name is renamed at a fixed operand position below, but the renderer takes operands from + // the end and ignores surplus ones before them: with a surplus operand the drawn name would + // keep its old spelling while its resource moved, and draw nothing or the wrong thing. The + // parser keeps such operands without counting them as skipped. Checked before the nested + // walk, which can only call it unverifiable. + if ops.iter().any(|op| takes_operands(&op.op).is_some_and(|n| op.operands.len() > n)) { + return Err(invalid("its content gives an operator more operands than it takes")); + } let shows_text = { // Check every form, including non-group forms, for recursive/unsupported content. // The form itself is being read: drawing itself again counts as blending. @@ -151,7 +173,7 @@ pub(crate) fn ungroup_ops( for cat in CATEGORIES { let Some(theirs) = form_res.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()) else { continue }; if theirs.len() > 4096 { - return Err(refuse("too many resources")); + return Err(invalid("too many resources")); } let mut mine = merged.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default(); if mine.len() > 4096 { @@ -203,20 +225,13 @@ pub(crate) fn ungroup_ops( for op in &mut ops { match op.op.as_slice() { b"BDC" | b"BMC" => marked = marked.saturating_add(1), - b"EMC" => marked = marked.checked_sub(1).ok_or_else(|| refuse("its content ends marked content it didn't begin"))?, - b"BT" if in_text => return Err(refuse("its content begins a text object inside another")), + b"EMC" => marked = marked.checked_sub(1).ok_or_else(|| invalid("its content ends marked content it didn't begin"))?, + b"BT" if in_text => return Err(invalid("its content begins a text object inside another")), b"BT" => in_text = true, - b"ET" if !in_text => return Err(refuse("its content ends a text object it didn't begin")), + b"ET" if !in_text => return Err(invalid("its content ends a text object it didn't begin")), b"ET" => in_text = false, _ => {} } - // A name is renamed at the operand position below, but the renderer takes operands from - // the end and ignores surplus ones before them: with a surplus operand the drawn name - // would keep its old spelling while its resource moved, and draw nothing or the wrong - // thing. The parser keeps such operands without counting them as skipped. - if takes_operands(&op.op).is_some_and(|n| op.operands.len() > n) { - return Err(refuse("its content gives an operator more operands than it takes")); - } let slot: Option<(&[u8], usize)> = match op.op.as_slice() { b"Tf" | b"Do" | b"gs" | b"sh" | b"cs" | b"CS" => { let cat: &[u8] = match op.op.as_slice() { @@ -232,13 +247,13 @@ pub(crate) fn ungroup_ops( b"BDC" | b"DP" => Some((&b"Properties"[..], 1)), b"q" => { if depth >= 4096 { - return Err(refuse("graphics state is too deeply nested")); + return Err(invalid("graphics state is too deeply nested")); } depth += 1; None } b"Q" => { - depth = depth.checked_sub(1).ok_or_else(|| refuse("its content restores more graphics states than it saves"))?; + depth = depth.checked_sub(1).ok_or_else(|| invalid("its content restores more graphics states than it saves"))?; None } _ => None, @@ -262,7 +277,7 @@ pub(crate) fn ungroup_ops( } if marked > 0 || in_text { - return Err(refuse("its content leaves marked content or a text object open")); + return Err(invalid("its content leaves marked content or a text object open")); } // Patterns keep their place relative to the shapes they fill: each one the form selects is @@ -351,7 +366,7 @@ pub(crate) fn ungroup_ops( if let Some(m) = matrix { out.push(Op::new("cm", m.0.iter().copied().map(num).collect())); } - let [a, b, c, d] = <[f64; 4]>::try_from(bbox).map_err(|_| refuse("invalid form bounds"))?; + let [a, b, c, d] = <[f64; 4]>::try_from(bbox).map_err(|_| invalid("invalid form bounds"))?; let (x0, y0, x1, y1) = (a.min(c), b.min(d), a.max(c), b.max(d)); out.push(Op::new("re", vec![num(x0), num(y0), num(x1 - x0), num(y1 - y0)])); out.push(Op::new("W", Vec::new())); diff --git a/crates/fonts/src/pdf/thai.rs b/crates/fonts/src/pdf/thai.rs index 8596d58734..0f950afc6d 100644 --- a/crates/fonts/src/pdf/thai.rs +++ b/crates/fonts/src/pdf/thai.rs @@ -120,7 +120,8 @@ pub fn recovered_thai_map(doc: &Document, font: &Dict) -> Option Option). +const PUA_BASES: [char; 27] = ['ฐ', 'ิ', 'ี', 'ึ', 'ื', '่', '้', '๊', '๋', '์', '่', '้', '๊', '๋', '์', 'ญ', 'ั', 'ํ', '็', '่', '้', '๊', '๋', '์', 'ุ', 'ู', 'ฺ']; + /// Whether a Thai character is a vowel or tone mark set above or below its base, with no advance /// of its own (U+0E31, U+0E34–U+0E3A, U+0E47–U+0E4E). Sara Am and the other vowels advance. fn is_thai_mark(ch: char) -> bool { @@ -236,17 +242,30 @@ pub(super) fn composite_glyph_maps( let kept: Vec<(usize, u32)> = (342u32..=368).enumerate().filter(|&(_, gid)| outlined(gid)).collect(); let in_order = kept.iter().all(|&(index, gid)| advances.advance_width(skrifa::GlyphId::new(gid)).is_some_and(|w| (w > 0.0) == matches!(index, 0 | 15))); + // Where the PDF says what an alternate's code is (an intact ToUnicode gives the character it + // stands for), it must be that alternate's base character: two alternates of the same kind + // in another order pass the advance check, but not this one. + let code_of = |gid: u32| match &cid_map { + None => u16::try_from(gid).ok(), + Some(_) => codes.get(&gid).copied(), + }; + let meanings_agree = kept.iter().all(|&(index, gid)| { + let stated = code_of(gid).and_then(|code| metrics.text_of(u32::from(code))).and_then(|t| only_char(&t)); + stated.filter(|c| matches!(u32::from(*c), 0x0e01..=0x0e5b)).is_none_or(|c| PUA_BASES.get(index) == Some(&c)) + }); if family.starts_with("THSarabunPSK") && unicode.get(&0x0e01) == Some(&214) && unicode.get(&0x0e2d) == Some(&258) && (alternate_meanings || unicode.contains_key(&0xf70a)) && in_order + && meanings_agree { for (index, gid) in kept { - if let Some(code) = match &cid_map { + let code = match &cid_map { None => u16::try_from(gid).ok(), Some(_) => codes.get(&gid).copied(), - } { + }; + if let Some(code) = code { unicode.insert(0xf700 + index as u32, gid); codes.entry(gid).or_insert(code); } @@ -555,13 +574,14 @@ mod tests { #[test] fn legacy_thai_alternates_need_their_glyph_order_confirmed() { let unicode = b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 2 beginbfchar <0160> <0e48> <0161> <0e49> endbfchar"; - let alternates = |program: &[u8]| { + let alternates_with = |program: &[u8], unicode: &[u8]| { let mut doc = Document::new_empty(); let font = thai_font(&mut doc, program, unicode); let metrics = Metrics::from_dict(&doc, &font); let (map, _) = composite_glyph_maps(&doc, &font, program, &metrics, &mut DecodeBudget::new(1 << 20, 0)).unwrap(); map.range(0xf700..=0xf71a).map(|(u, g)| (*u, *g)).collect::>() }; + let alternates = |program: &[u8]| alternates_with(program, unicode); let ordered = alternates(&thai_ordered_program()); assert!(ordered.contains(&(0xf70a, 352)) && ordered.contains(&(0xf710, 358)), "{ordered:?}"); let mut misplaced = thai_ordered_program(); @@ -572,6 +592,12 @@ mod tests { drop_glyph(&mut dropped, 353); let kept = alternates(&dropped); assert!(kept.contains(&(0xf70a, 352)) && !kept.iter().any(|(_, g)| *g == 353), "{kept:?}"); + // Two marks of the same kind (both zero-advance) in another order: the PDF's own map says + // glyph 343 is ี and 344 is ิ, the reverse of U+F701 and U+F702. None is used. + let stated = b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 4 beginbfchar <0157> <0e35> <0158> <0e34> <0160> <0e48> <0161> <0e49> endbfchar"; + assert!(alternates_with(&thai_ordered_program(), stated).is_empty(), "a stated meaning out of the PUA order withdraws them all"); + let agreeing = b"2 beginbfrange <00d6> <010f> <0e01> <0110> <012a> <0e3f> endbfrange 4 beginbfchar <0157> <0e34> <0158> <0e35> <0160> <0e48> <0161> <0e49> endbfchar"; + assert!(alternates_with(&thai_ordered_program(), agreeing).contains(&(0xf701, 343))); } #[test] diff --git a/docs/editing-existing-text.md b/docs/editing-existing-text.md index b7716cf4d9..6127f9a234 100644 --- a/docs/editing-existing-text.md +++ b/docs/editing-existing-text.md @@ -86,7 +86,8 @@ selects for Thai, not just a GPOS table); GSUB substitution is optional, as spec [the OpenType Thai model](https://learn.microsoft.com/en-us/typography/script-development/thai). Verified legacy Thai PUA fonts use positional substitutions instead of requiring GPOS; their alternates are used only when each kept one has a mark's zero advance (or, for the two -consonant forms, an advance). A THSarabunPSK program with its own Thai mark +consonant forms, an advance) and, where the PDF's own map names its character, the character +that alternate stands for. A THSarabunPSK program with its own Thai mark positioning, such as a complete face loaded with Load font, is shaped by that layout instead, still in its own codes. One without it takes the legacy path only when every Thai glyph it keeps is where the legacy glyph order puts it, and is otherwise shaped as an ordinary source font. A From 4b2a15101288029197e1c7dae1bfac0c6dc137a2 Mon Sep 17 00:00:00 2001 From: xullul Date: Sat, 7 Nov 2026 18:34:05 +0700 Subject: [PATCH 09/37] M7: count each grouped placement once across preparation passes Change: - Read and count only the placements a preparation pass newly inlines. The others were read on an earlier pass in the context they still have, so a group refused where it is drawn (translucent, for example) is no longer counted again on every pass: 33 such placements beside one text group used to reach the 64-placement limit on the second pass and prepare nothing. A form refused in one place is still read where an expanded parent draws it anew. If the page doesn't parse back to the operators written, every placement is read again, as before. - Read a group refused because its resources needed renaming again, without counting it, once names merged in for other groups make the renaming unnecessary (checked from its resource dictionaries alone, without decoding its content), and drop its earlier refusal when it is then prepared. Validation: edit 470 and the 2 automation text_prepare tests passed; formatting, strict Clippy for pdfcraft-edit, layers and assets passed. The wasm check was not run locally (the wasm32 target is not installed), nor were the engine suite and the workspace sweep. Each new regression failed with its fix disabled. --- crates/edit/src/tests.rs | 54 +++++++++++++++++++ crates/edit/src/text/forms.rs | 78 ++++++++++++++++++++++++---- crates/edit/src/text/forms/expand.rs | 28 +++++++++- 3 files changed, 147 insertions(+), 13 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index d3c0e4fdcd..5ae27517d3 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -11021,6 +11021,60 @@ fn a_placement_refusal_does_not_settle_the_form_for_later_passes() { assert_eq!(text::text_blocks(&doc, 0).unwrap().iter().filter(|b| b.text == "shared").count(), 1, "only the opaque one is the page's own text"); } +/// A placement refused where it is drawn is read once, not again on every pass, while the same +/// form drawn anew by an expanded parent is still read in its own place: 33 translucent +/// placements of a text group and a parent drawing it opaque are 35 reads, within the +/// 64-placement limit, where reading the 33 again on the second pass would pass it and prepare +/// nothing. +#[test] +fn a_group_refused_where_it_is_drawn_counts_once_against_the_placement_limit() { + let resources = "<< /Font << /F1 5 0 R >> /ExtGState << /Alpha << /ca 0.5 >> >> /XObject << /Shared 6 0 R /Parent 7 0 R >> >>"; + let form = |extra: &str, body: &str| stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 612 792] {extra} /Resources {resources}"), body); + let content = format!("{}/Parent Do", "q /Alpha gs /Shared Do Q\n".repeat(33)); + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {resources} /Contents 4 0 R >>"), + stream("", &content), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + form("/Group << /S /Transparency >>", "BT /F1 14 Tf 40 500 Td (shared) Tj ET"), + form("", "/Shared Do"), + ]; + let mut doc = build(&objs.iter().map(String::as_str).collect::>()); + let result = text::forms::prepare(&mut doc, 0).unwrap_or_else(|e| panic!("each placement is read once ({e})")); + assert_eq!(result.expanded, 2, "Parent, then the Shared it draws opaque: {:?}", result.refused); + assert!(result.refused.iter().any(|r| r.starts_with("Shared: ") && r.contains("translucent")), "{:?}", result.refused); + let drawn = pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops.iter().filter(|op| op.is("Do")).count(); + assert_eq!(drawn, 33, "the translucent placements stay grouped"); +} + +/// A group refused because its resources would need renaming (it draws a form that looks them +/// up by name) is prepared on a later pass once another group's merge has given the page those +/// very names. A lists the child as `/PCUg1`; B lists it as `/Other`, which merging B names +/// `/PCUg1` on the page. Its first refusal is then no longer reported. +#[test] +fn a_group_refused_for_renaming_is_prepared_once_the_page_has_its_names() { + let form = |resources: &str, body: &str| stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources {resources}"), body); + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> /XObject << /A 6 0 R /B 7 0 R >> >> /Contents 4 0 R >>" + .to_string(), + stream("", "/A Do\n/B Do"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + form("<< /Font << /F1 5 0 R >> /XObject << /PCUg1 8 0 R >> >>", "BT /F1 14 Tf 40 500 Td (alpha) Tj ET /PCUg1 Do"), + form("<< /Font << /F1 5 0 R >> /XObject << /Other 8 0 R >> >>", "BT /F1 14 Tf 40 400 Td (beta) Tj ET"), + // The child has no resources of its own: it looks names up in whatever draws it. + stream("/Type /XObject /Subtype /Form /BBox [0 0 10 10]", "0 0 10 10 re f"), + ]; + let mut doc = build(&objs.iter().map(String::as_str).collect::>()); + let result = text::forms::prepare(&mut doc, 0).unwrap(); + assert_eq!(result.expanded, 2, "B, then A once the page names the child as A does: {:?}", result.refused); + assert!(result.refused.is_empty(), "A was prepared after all: {:?}", result.refused); + let blocks = text::text_blocks(&doc, 0).unwrap(); + assert!(["alpha", "beta"].iter().all(|t| blocks.iter().any(|b| b.text == *t)), "{blocks:?}"); +} + /// The `Do` is matched to its resource by the name's bytes as written, which need not be UTF-8. #[test] fn grouped_text_under_a_non_utf8_resource_name_is_prepared() { diff --git a/crates/edit/src/text/forms.rs b/crates/edit/src/text/forms.rs index da46a16d91..fcc7a31e13 100644 --- a/crates/edit/src/text/forms.rs +++ b/crates/edit/src/text/forms.rs @@ -24,6 +24,17 @@ pub struct Preparation { /// grouped and is reported in [`Preparation::refused`]. const MAX_LEVELS: usize = 8; +/// What one pass of [`prepare`] inlined, as positions in the next pass's parse of the page. +struct Inlined { + /// How many operators that parse has; with any other count the positions are wrong. + ops: usize, + /// Where each inlined form's operators are: the placements in them are new. + ranges: Vec>, + /// Earlier placements refused only because their resources needed renaming + /// ([`expand::Refusal::renaming`]), each with the reason it was given. + waiting: Vec<(usize, String)>, +} + /// Prepare grouped source text atomically, within eight levels and 64 placements per page. /// Shared forms and other pages are untouched. Preparation is a distinct undoable engine edit. pub fn prepare(doc: &mut Document, page: usize) -> Result { @@ -33,10 +44,17 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result = None; // One pass opens one level of every placement; the last pass only reports what is deeper. for pass in 0..=MAX_LEVELS { let p = page_dict(&next, page)?; @@ -47,11 +65,19 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result (Some(i.ranges), i.waiting), + _ => (None, Vec::new()), + }; + let mut still_waiting = Vec::new(); let resources = p.dict.get(b"Resources").map(|r| next.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let space = expand::page_blend_space(&next, &p.dict); let images: std::collections::BTreeMap<_, _> = crate::page_images(&next, page)?.into_iter().filter(|i| i.is_form).map(|i| (i.op, i)).collect(); - let mut replacements = std::collections::HashMap::new(); + let mut replacements = std::collections::BTreeMap::new(); let mut merged = resources; let mut pending_path = false; let mut in_text = false; @@ -68,6 +94,25 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result::to_vec); + // An earlier placement is read again only when it was refused for a rename that the + // names merged since have made unnecessary. The last pass opens nothing, so there it + // keeps its reason. + let earlier = new.as_ref().is_some_and(|ranges| !ranges.iter().any(|r| r.contains(&at))); + if earlier { + let Some(w) = waiting.iter().position(|(i, _)| *i == at) else { continue }; + let (_, reason) = waiting.swap_remove(w); + if pass == MAX_LEVELS || !expand::merges_without_renaming(&next, &merged, &name) { + still_waiting.push((at, reason)); + continue; + } + // What reading it again finds replaces the reason it was given. + if let Some(i) = result.refused.iter().position(|r| *r == reason) { + result.refused.remove(i); + } + } if pending_path || in_text { result.refused.push(format!("{}: the group is drawn inside an unfinished path or text object", image.name)); continue; @@ -78,14 +123,13 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result 64 { - return Err(EditError::Invalid("too many grouped text placements on this page (limit 64); the page was left unchanged".into())); + if !earlier { + visits = visits.saturating_add(1); + if visits > 64 { + return Err(EditError::Invalid("too many grouped text placements on this page (limit 64); the page was left unchanged".into())); + } } let translucent = expand::translucent_at(&next, &merged, &ops, at); - // The resource name exactly as the `Do` writes it (its only operand): a name need not - // be UTF-8, and `image.name` is only its display form. - let name = op.last_name().map_or_else(|| image.name.as_bytes().to_vec(), <[u8]>::to_vec); match expand::ungroup_ops(&next, &merged, &name, translucent, space.as_ref(), Matrix(image.matrix), &mut decode) { Ok(expand::Ungrouped { ops: inline, resources: mut res, patterns, shows_text }) => { // Vector and image artwork needs no preparation. Nested text groups are @@ -115,10 +159,14 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result { - result.refused.push(format!("{}: {}", image.name, refusal.error)); + let reason = format!("{}: {}", image.name, refusal.error); + if refusal.renaming { + still_waiting.push((at, reason.clone())); + } if refusal.intrinsic { refused_here.extend(image.object); } + result.refused.push(reason); } } } @@ -126,6 +174,14 @@ pub fn prepare(doc: &mut Document, page: usize) -> Result (ops, false), None => (Vec::new(), true), diff --git a/crates/edit/src/text/forms/expand.rs b/crates/edit/src/text/forms/expand.rs index 54495900cc..d674e1f473 100644 --- a/crates/edit/src/text/forms/expand.rs +++ b/crates/edit/src/text/forms/expand.rs @@ -67,6 +67,10 @@ pub(crate) struct Refusal { /// page so far is: its translucency or matrix, the resources it would merge into (which grow /// as other groups are prepared), or the decode budget left. pub(crate) intrinsic: bool, + /// The form's resources need renaming where it is drawn, and it draws something that would + /// look them up by their old names. Names merged in for other forms can make the renaming + /// unnecessary, which [`merges_without_renaming`] checks without reading the content. + pub(crate) renaming: bool, } /// Ungroup form `name` from the page's `resources`. `translucent` says whether the graphics state @@ -83,7 +87,11 @@ pub(crate) fn ungroup_ops( placement: Matrix, budget: &mut DecodeBudget, ) -> Result { - let refusal = |why: &str, intrinsic: bool| Refusal { error: EditError::Invalid(format!("this artwork can't be ungrouped: {why}")), intrinsic }; + let refusal = |why: &str, intrinsic: bool| Refusal { + error: EditError::Invalid(format!("this artwork can't be ungrouped: {why}")), + intrinsic, + renaming: false, + }; // Something about this placement or the page so far is the reason. let refuse = |why: &str| refusal(why, false); // The form itself is the reason: every placement of it is refused the same way. @@ -215,7 +223,7 @@ pub(crate) fn ungroup_ops( // would find the page's, which differ wherever a name was renamed (every form resource that // isn't already the page's, the nested item's own entry included). if !renames.is_empty() && ops.iter().any(|op| inherits_resources(doc, &form_res, op)) { - return Err(refuse("it draws a nested form, Type 3 font or pattern that uses its resources")); + return Err(Refusal { renaming: true, ..refuse("it draws a nested form, Type 3 font or pattern that uses its resources") }); } let mut depth = 0usize; // Marked content and text objects must balance within the form's own stream: an extra EMC @@ -383,6 +391,22 @@ pub(crate) fn ungroup_ops( Ok(Ungrouped { ops: out, resources: merged, patterns, shows_text }) } +/// Whether form `name` in `resources` would merge into them without a rename: it has resources +/// of its own, and each of their entries is already in `resources` under the same name, as +/// [`ungroup_ops`] compares them. Only dictionaries are read, never the form's content. +pub(crate) fn merges_without_renaming(doc: &Document, resources: &Dict, name: &[u8]) -> bool { + let category = |d: &Dict, cat: &[u8]| d.get(cat).map(|x| doc.resolve(x)).and_then(|x| x.as_dict().cloned()); + let Some(form) = category(resources, b"XObject").and_then(|x| x.get(name).cloned()) else { return false }; + let Object::Stream(s) = &*doc.resolve(&form) else { return false }; + let Some(own) = s.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()) else { return false }; + CATEGORIES.into_iter().all(|cat| { + let Some(theirs) = category(&own, cat) else { return true }; + let mine = category(resources, cat).unwrap_or_default(); + // Past these sizes `ungroup_ops` refuses the form anyway. + theirs.len() <= 4096 && mine.len() <= 4096 && theirs.iter().all(|(key, value)| mine.get(key) == Some(value)) + }) +} + /// A copy of pattern `entry` whose `/Matrix` maps its space to the page's default space instead /// of the form's: its own matrix, then `placement`. `None` when it isn't a readable pattern, its /// matrix is unreadable, the result isn't finite or a tiling pattern is too large to copy. From 82116a5c7f37e4dac71858923240bf7ed783dba1 Mon Sep 17 00:00:00 2001 From: Tiago Kiel <186439951+TiagoCK@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:27:24 -0400 Subject: [PATCH 10/37] fix(edit): an edited OCR phrase is drawn at one size (refs #3313) Recognize text sizes each word from its own box and a guess at how much of the em its letters fill, so one phrase's words come out several points apart (9 to 15.6 pt on a LaTeX scan). Editing a phrase drew and saved every word at its own size. An invisible block's size is now the median of its words' sizes (TextBlock::phrase_size). The styled rewrite draws every run at it and scales each word's width calibration by its own size over the phrase's, so words keep the width the scan shows them at; the single-style rewrite (a new family or bold) uses it in place of the first word's size; and the inline editor lays each word out at it while typing. Co-Authored-By: Claude Opus 5.5 --- crates/edit/src/text/blocks.rs | 7 +++ crates/edit/src/text/rewrite.rs | 9 ++-- crates/edit/src/text_regions/grouping.rs | 16 +++++++ crates/edit/src/text_regions/styled.rs | 19 +++++++- crates/engine/tests/it/ocr_edit.rs | 45 +++++++++++++++++++ crates/ui-egui/src/tools/edit_text_ui.rs | 43 ++++++++++++++++++ .../ui-egui/src/tools/edit_text_ui/layout.rs | 10 +++-- docs/editing-existing-text.md | 4 +- parity/acrobat-features.toml | 4 +- 9 files changed, 145 insertions(+), 12 deletions(-) diff --git a/crates/edit/src/text/blocks.rs b/crates/edit/src/text/blocks.rs index 48fe956f4f..912fa873f4 100644 --- a/crates/edit/src/text/blocks.rs +++ b/crates/edit/src/text/blocks.rs @@ -37,6 +37,13 @@ pub struct TextBlock { } impl TextBlock { + /// The one size a rewrite and the editor set an OCR phrase in, in place of each word's own + /// (#3313): the block's size, which for invisible text is its words' median. `None` for + /// visible text, whose runs keep their sizes. + pub fn phrase_size(&self) -> Option { + (self.invisible && self.size.is_finite() && self.size > 0.0).then_some(self.size) + } + /// Preserve styles outside the changed ranges; inserted characters inherit the surrounding /// source run. The texts are aligned word by word, so words left untouched between two edits /// keep their own runs (#2198); past [`MAX_CHANGED_WORDS`] the draft counts as one change. diff --git a/crates/edit/src/text/rewrite.rs b/crates/edit/src/text/rewrite.rs index d27d23e0b5..b1111366f2 100644 --- a/crates/edit/src/text/rewrite.rs +++ b/crates/edit/src/text/rewrite.rs @@ -749,12 +749,13 @@ struct ParaStyle { k: f64, } -/// Spacing and scaling (text state), with the chosen values. -fn paragraph_style(o: &Origin, style: &BlockStyle) -> Result<(String, ParaStyle), EditError> { +/// Spacing and scaling (text state), with the chosen values. `phrase` is the size an OCR phrase +/// is drawn at when none is chosen, in place of its first word's own (#3313). +fn paragraph_style(o: &Origin, style: &BlockStyle, phrase: Option) -> Result<(String, ParaStyle), EditError> { let (font_name, old_size) = o.state.font.clone().ok_or_else(|| EditError::Invalid("the paragraph has no font".into()))?; let k = o.k.max(1e-6); // The size in text space: points ÷ the text-to-user scale. - let size = style.size.map_or(old_size, |pt| (pt / k).max(0.1)); + let size = style.size.or(phrase).map_or(old_size, |pt| (pt / k).max(0.1)); let mut state = o.state.clone(); if let Some(c) = style.char_spacing { state.char_spacing = c / k; @@ -842,7 +843,7 @@ pub(crate) fn rewrite_block_from( let o = &first.origin; let last = members.last().copied().unwrap_or(first); let emit = &last.origin; - let (font_name, ParaStyle { state: o_state, size, old_size, k }) = paragraph_style(o, style)?; + let (font_name, ParaStyle { state: o_state, size, old_size, k }) = paragraph_style(o, style, b.phrase_size())?; let source = source_font(doc, &fonts_res, &font_name, first, &flat, style); let reuse = source.reuse; // The chosen style also determines the real Japanese fallback outlines and advances. diff --git a/crates/edit/src/text_regions/grouping.rs b/crates/edit/src/text_regions/grouping.rs index 788499fafe..de7c47cd81 100644 --- a/crates/edit/src/text_regions/grouping.rs +++ b/crates/edit/src/text_regions/grouping.rs @@ -398,6 +398,11 @@ fn group_layout(lines: &[TextLine], mixed: bool) -> Vec { for block in &mut blocks { block.reflow_text = reflow_text(block, lines); } + for block in blocks.iter_mut().filter(|block| block.invisible && block.lines.len() > 1) { + if let Some(size) = median_size(block.lines.iter().filter_map(|i| lines.get(*i)).map(|line| line.size)) { + block.size = size; + } + } blocks } @@ -452,3 +457,14 @@ fn break_gap(block: &TextBlock, lines: &[TextLine], row_index: usize) -> Option< // measure leaves almost no room for the next word. (slack <= size * 0.75 && width >= size * 8.0 && (next[2] - next[0]) + size * 0.25 > slack).then_some(1) } + +/// The size an OCR phrase is set in. Recognize text sizes each word from its own box and a guess +/// at how much of the em its letters fill, so one phrase's words come out several points apart +/// (#3313); a rewrite draws the phrase at this one size. `None` when no size is usable. +fn median_size(sizes: impl Iterator) -> Option { + let mut sizes: Vec = sizes.filter(|size| size.is_finite() && *size > 0.0).collect(); + sizes.sort_by(f64::total_cmp); + let middle = sizes.len() / 2; + let upper = sizes.get(middle).copied()?; + if sizes.len().is_multiple_of(2) { sizes.get(middle.saturating_sub(1)).map(|lower| (lower + upper) / 2.0) } else { Some(upper) } +} diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 7a9204a8c0..cc32894a54 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -321,6 +321,19 @@ fn calibration_factors( calibration } +/// The calibration factors fit each word to its box at its own size (its `Tf` in its text space, +/// which a range's size doesn't change). An OCR phrase drawn at one size needs the inverse ratio +/// too, so every word still spans the box the scan shows it in. +fn fit_to_phrase(calibration: &mut HashMap, lines: &[TextLine], phrase: Option) { + let Some(phrase) = phrase else { return }; + for (index, factor) in calibration.iter_mut() { + let own = lines.get(*index).and_then(|source| Some(source.origin.state.font.as_ref()?.1 * source.origin.k)); + if let Some(own) = own.filter(|own| own.is_finite() && *own > 0.0) { + *factor = (*factor * own / phrase).clamp(0.5, 2.0); + } + } +} + fn validated_ranges<'a>(draft: &str, style: &'a BlockStyle) -> Result, EditError> { let mut ranges: Vec<_> = style.ranges.iter().collect(); ranges.sort_by_key(|range| range.range.start); @@ -891,8 +904,9 @@ pub(crate) fn rewrite_styled_block( .prepare()?; let chars = prepared.values; let substituted = prepared.substituted; + let phrase_size = block.phrase_size(); let size_of = |source: &TextLine, index: usize| { - overrides.get(&index).and_then(|r| r.size).or(style.size).unwrap_or(source.size).max(0.1) / source.origin.k.max(1e-6) + overrides.get(&index).and_then(|r| r.size).or(style.size).or(phrase_size).unwrap_or(source.size).max(0.1) / source.origin.k.max(1e-6) }; // A source run may carry TJ kerning/justification that is absent from its text state. // Keep its measured advance when rebuilding the run instead of making it wrap on entry. @@ -910,7 +924,8 @@ pub(crate) fn rewrite_styled_block( spaces.borrow_mut().insert(name.to_owned(), w); w }; - let calibration = calibration_factors(lines, &metrics, style, &overrides, space_of); + let mut calibration = calibration_factors(lines, &metrics, style, &overrides, space_of); + fit_to_phrase(&mut calibration, lines, phrase_size); let advance = |item: &(char, usize, String)| -> f64 { let source = lines.get(item.1).unwrap_or(first); let Some(m) = metrics.get(&item.2) else { return 0.0 }; diff --git a/crates/engine/tests/it/ocr_edit.rs b/crates/engine/tests/it/ocr_edit.rs index 2d358e327e..fa33a84858 100644 --- a/crates/engine/tests/it/ocr_edit.rs +++ b/crates/engine/tests/it/ocr_edit.rs @@ -160,6 +160,51 @@ fn a_rewritten_ocr_phrase_keeps_its_word_widths() { } } +/// Recognize text sizes each word from its box and a guess at how much of the em its letters fill, +/// so one phrase's words come out several points apart (#3313). Here the first word is the +/// largest and the middle one the smallest; the phrase is set in 11 pt. +fn uneven_phrase() -> (Session, pdfcraft_engine::DocId) { + scanned_with(vec![ + scanned_word("John", 20.0, 45.0, 70.0, 15.0), + scanned_word("Smith", 49.0, 78.0, 70.0, 9.0), + scanned_word("Example", 82.0, 124.0, 70.0, 11.0), + ]) +} + +/// An edited OCR phrase is redrawn in one size, the median of its words', and each word keeps +/// the width the scan shows it at (#3313). +#[test] +fn an_edited_ocr_phrase_is_drawn_at_one_size() { + let (mut s, id) = uneven_phrase(); + let block = s.get(id).unwrap().text_blocks(0).remove(0); + assert!((block.size - 11.0).abs() < 0.05, "the phrase's size is its words' median: {}", block.size); + + s.apply(id, Edit::EditTextBlock { page: 0, block: 0, text: "John Example".into(), style: Default::default() }).unwrap(); + let lines = s.get(id).unwrap().text_lines(0); + assert_eq!(lines.iter().map(|l| l.text.trim()).collect::>(), ["John", "Example"]); + for line in &lines { + assert!((line.size - 11.0).abs() < 0.05, "{:?} is drawn at {} pt, not the phrase's 11", line.text, line.size); + } + // Each word keeps its width; the rest of the phrase closes up where Smith was. + assert!((lines[0].rect[2] - lines[0].rect[0] - 25.0).abs() < 0.5, "John keeps its width: {:?}", lines[0].rect); + assert!((lines[1].rect[2] - lines[1].rect[0] - 42.0).abs() < 0.5, "Example keeps its width: {:?}", lines[1].rect); + assert!(lines[1].rect[0] >= lines[0].rect[2] - 0.5, "Example runs into John"); +} + +/// Choosing a font sends the phrase through the single-style rewrite, which takes its size from +/// the first run: that is the phrase's size too, not the first word's guess. +#[test] +fn a_new_font_for_an_ocr_phrase_keeps_its_one_size() { + let (mut s, id) = uneven_phrase(); + let style = pdfcraft_engine::BlockStyle { family: Some((pdfcraft_engine::FontFamily::Times, false, false)), ..Default::default() }; + s.apply(id, Edit::EditTextBlock { page: 0, block: 0, text: "John Smith Example".into(), style }).unwrap(); + let lines = s.get(id).unwrap().text_lines(0); + assert!(!lines.is_empty()); + for line in &lines { + assert!((line.size - 11.0).abs() < 0.05, "{:?} is drawn at {} pt, not the phrase's 11", line.text, line.size); + } +} + /// Visible text set with a horizontally stretched text matrix (3 × 1), as some generators write. fn stretched_visible() -> std::sync::Arc> { use pdfcraft_cos::{Dict, Document, Object, SaveOptions, Stream}; diff --git a/crates/ui-egui/src/tools/edit_text_ui.rs b/crates/ui-egui/src/tools/edit_text_ui.rs index f2833de356..0a13a78309 100644 --- a/crates/ui-egui/src/tools/edit_text_ui.rs +++ b/crates/ui-egui/src/tools/edit_text_ui.rs @@ -1345,6 +1345,49 @@ C", &picked assert_eq!(doc.text_lines(0).iter().map(|l| l.text.as_str()).collect::(), "Text preview"); } + /// Typing in a recognised phrase whose words OCR sized at 15, 9 and 11 pt lays every word out + /// at the phrase's one size, as the engine redraws it (#3313); each word used its own. + #[test] + fn typing_in_an_ocr_phrase_lays_every_word_out_at_one_size() { + use pdfcraft_engine::ocr::PlacedWord; + // A word box as OCR draws it around capitals: the cap height of a `size` pt word. + let word = |text: &str, x0: f64, x1: f64, size: f64| PlacedWord { + text: text.into(), + origin: [x0, 70.0], + across: [x1 - x0, 0.0], + up: [0.0, 0.718 * size], + confidence: None, + source: "ocrs".into(), + }; + let mut app = crate::PdfCraftApp::new(); + let png = pdfcraft_engine::export::encode_png(200, 100, &[235u8; 200 * 100 * 4]).unwrap(); + let bytes = app.session.create_from_images(&[("scan.png".into(), png)]).unwrap(); + app.open_bytes("scan.pdf", None, bytes.as_ref().clone()).unwrap(); + let id = app.views[0].id; + let words = vec![word("JOHN", 20.0, 45.0, 15.0), word("SMITH", 49.0, 78.0, 9.0), word("EXAMPLE", 82.0, 124.0, 11.0)]; + app.session.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); + let doc = app.session.get(id).unwrap(); + let block = doc.text_blocks(0).remove(0); + assert!(block.invisible && block.lines.len() == 3, "one phrase of three recognised words: {block:?}"); + let (_, mut ed) = preview_app(); + ed.text = format!("{}S", block.editor_text); + ed.original = block.editor_text.clone(); + ed.look = look_of(&block); + ed.look0 = look_of(&block); + ed.source = Some((block, doc.text_lines(0))); + + let ctx = egui::Context::default(); + let mut sizes = Vec::new(); + ctx.run_ui(egui::RawInput::default(), |ui| { + let job = editor_job(ui, &ed, &ed.text, 1.0, f32::INFINITY); + sizes = job.sections.iter().map(|section| section.format.font_id.size).collect(); + }) + .textures_delta + .clear(); + assert!(sizes.len() >= 3, "a section for each word: {sizes:?}"); + assert!(sizes.iter().all(|size| (size - 11.0).abs() < 0.05), "every word at the phrase's 11 pt: {sizes:?}"); + } + #[test] fn a_refused_paragraph_stays_open_with_its_text() { let (mut app, mut ed) = preview_app(); diff --git a/crates/ui-egui/src/tools/edit_text_ui/layout.rs b/crates/ui-egui/src/tools/edit_text_ui/layout.rs index 1946519203..ebcd923c6a 100644 --- a/crates/ui-egui/src/tools/edit_text_ui/layout.rs +++ b/crates/ui-egui/src/tools/edit_text_ui/layout.rs @@ -126,10 +126,14 @@ pub(super) fn editor_job(ui: &egui::Ui, ed: &LineEditor, text: &str, scale: f32, if centered { job.halign = egui::Align::Center; } + // An OCR phrase is set in one size, as the engine redraws it: each word's own is a guess + // from its box. Its words still keep their widths. + let phrase = block.phrase_size(); for (range, index) in block.edited_spans(text) { let Some(line) = lines.get(index) else { continue }; - let mut look = source_look(&line.base_font, line.size, line.color, line.bold, line.italic); - look.size = line.size; + let base = phrase.unwrap_or(line.size); + let mut look = source_look(&line.base_font, base, line.color, line.bold, line.italic); + look.size = base; if ed.look.family != ed.look0.family || ed.family_chosen { look.family = ed.look.family; } @@ -153,7 +157,7 @@ pub(super) fn editor_job(ui: &egui::Ui, ed: &LineEditor, text: &str, scale: f32, font.family = FontFamily::Name(name.clone().into()); } let native_width = ui.fonts_mut(|f| f.layout_no_wrap(line.text.clone(), font.clone(), Color32::BLACK).size().x); - let target_width = ((line.rect[2] - line.rect[0]) * look.size / line.size.max(0.01)) as f32 * scale; + let target_width = ((line.rect[2] - line.rect[0]) * look.size / base.max(0.01)) as f32 * scale; let spacing = if original_face { (target_width - native_width) / line.text.chars().count().saturating_sub(1).max(1) as f32 } else { 0.0 }; job.append( text.get(range).unwrap_or_default(), diff --git a/docs/editing-existing-text.md b/docs/editing-existing-text.md index 426058f7b7..d5776e0646 100644 --- a/docs/editing-existing-text.md +++ b/docs/editing-existing-text.md @@ -89,7 +89,9 @@ On a scanned page, recognised text (Scan & OCR) is an invisible layer over the p text groups it into phrases: words joined by a word space, with a table cell, a tab stop and each scanned line kept apart. Applying an edit draws the new text visibly in the scan's ink colour (or the one chosen) and first covers the old words with the paper colour sampled around them. Each -word keeps its position and width, and a longer phrase grows rightwards rather than wrapping. The +word keeps its position and width, and a longer phrase grows rightwards rather than wrapping. +OCR sizes each word from its own box, so a phrase's words can come out several points apart: the +editor and the rewrite set the whole phrase in one size, the median of its words' sizes. The cover is one flat colour, and the text uses the layer's Helvetica, not the scanned typeface. This applies to any invisible text (rendering mode 3), not just a PdfCraft OCR layer: editing invisible text, such as another OCR tool's layer, makes the rewritten paragraph visible over a diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 7582b3f6bc..8d4960599a 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6755,8 +6755,8 @@ milestone = "M10" status = "partial" commands = ["edit.edit_text", "ocr.correct"] tools = ["text_edit", "ocr_recognize"] -evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once", "crates/engine/src/ocr.rs::editable_writer_refuses_empty_and_writes_visible_words", "crates/engine/src/ocr.rs::editable_output_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::editable_writer_maps_rotated_pages", "crates/automation/tests/it/automation.rs::ocr_recognize_editable_output_writes_visible_text"] -notes = "On demand, per edit: in Edit text & images, recognised text (the invisible OCR layer over a scan) is grouped into phrases (words across a word space; a wider gap such as a table cell, and each scanned line, stay separate) and opens for editing on a click. Applying draws the new text visibly in the scan's sampled ink colour (or the chosen one) and first covers the old words with the paper colour sampled around them, so the scanned word no longer shows; when an image or form is drawn after the OCR layer over where the edit lands (its old box, moved, rewrapped or grown to more lines), or an inline image or shading after it (a layer written under its scan, #3973), the cover and new text are drawn on top of the page instead, so they show; the phrase keeps each word's position and width and grows rightwards instead of wrapping. Turning Edit text & images on for a scanned page (an image covering at least half of a page that has no text of its own, once per page per window; comments and text boxes over it do not count as text, a page with real text and a background picture is left alone; one undoable \"Recognize text\" step) now recognizes its words first, as Acrobat's \"Converting scanned page to Editable Text and Images\" does (English only until more OCR languages exist, gap 7). Still missing: separating the photographs on the page from the lettering, matching the scanned typeface (the layer's Helvetica is used), covering on a textured or patterned background (one flat colour is used), a scan painted by a pattern fill or a filled rectangle over a layer written under it, and, for text drawn on top, the clip, optional content and marked content (tags) that enclosed the OCR layer." +evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_edited_ocr_phrase_is_drawn_at_one_size", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once", "crates/engine/src/ocr.rs::editable_writer_refuses_empty_and_writes_visible_words", "crates/engine/src/ocr.rs::editable_output_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::editable_writer_maps_rotated_pages", "crates/automation/tests/it/automation.rs::ocr_recognize_editable_output_writes_visible_text"] +notes = "On demand, per edit: in Edit text & images, recognised text (the invisible OCR layer over a scan) is grouped into phrases (words across a word space; a wider gap such as a table cell, and each scanned line, stay separate) and opens for editing on a click. Applying draws the new text visibly in the scan's sampled ink colour (or the chosen one) and first covers the old words with the paper colour sampled around them, so the scanned word no longer shows; when an image or form is drawn after the OCR layer over where the edit lands (its old box, moved, rewrapped or grown to more lines), or an inline image or shading after it (a layer written under its scan, #3973), the cover and new text are drawn on top of the page instead, so they show; the phrase keeps each word's position and width and grows rightwards instead of wrapping, and is set in one size, the median of its words' (OCR sizes each word from its own box, several points apart), both while typing and when applied (#3313). Turning Edit text & images on for a scanned page (an image covering at least half of a page that has no text of its own, once per page per window; comments and text boxes over it do not count as text, a page with real text and a background picture is left alone; one undoable \"Recognize text\" step) now recognizes its words first, as Acrobat's \"Converting scanned page to Editable Text and Images\" does (English only until more OCR languages exist, gap 7). Still missing: separating the photographs on the page from the lettering, matching the scanned typeface (the layer's Helvetica is used), covering on a textured or patterned background (one flat colour is used), a scan painted by a pattern fill or a filled rectangle over a layer written under it, and, for text drawn on top, the clip, optional content and marked content (tags) that enclosed the OCR layer." [[feature]] id = "ocr.ocr-downsample" From 7cb8ca3c01bb21e70dfb9137184be1a6cbdc43ad Mon Sep 17 00:00:00 2001 From: Tiago Kiel <186439951+TiagoCK@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:53:26 -0400 Subject: [PATCH 11/37] fix(edit): OCR phrase size counts words and keeps widths past 2x (refs #3313) Review follow-ups: - The phrase's median counts each word once: neighbouring words OCR sized alike share a run, so 15, 9 and 9 pt gave 12 pt, not 9. - The styled rewrite's width calibration is not clamped again after the phrase ratio, so a word sized at more than twice the phrase keeps the width the scan shows it at. - The single-style rewrite (a new family or bold) scales the first word's Tz by its own size over the phrase's, so the line is as wide whatever size OCR guessed for that word. Co-Authored-By: Claude Opus 5.5 --- crates/edit/src/text/rewrite.rs | 5 +++ crates/edit/src/text_regions/grouping.rs | 8 +++- crates/edit/src/text_regions/styled.rs | 8 +++- crates/engine/tests/it/ocr_edit.rs | 50 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/edit/src/text/rewrite.rs b/crates/edit/src/text/rewrite.rs index b1111366f2..0a2941076b 100644 --- a/crates/edit/src/text/rewrite.rs +++ b/crates/edit/src/text/rewrite.rs @@ -763,6 +763,11 @@ fn paragraph_style(o: &Origin, style: &BlockStyle, phrase: Option) -> Resul if let Some(sc) = style.scale { state.scale = (sc / 100.0).clamp(0.1, 10.0); } + // At the phrase's size, the first word's text matrix still carries its own width: scale it + // back, so the line is as wide as the scan whatever size OCR guessed for that word. + if phrase.is_some() && style.size.is_none() && style.scale.is_none() { + state.scale = (state.scale * old_size / size).clamp(0.1, 10.0); + } Ok((font_name, ParaStyle { state, size, old_size, k })) } diff --git a/crates/edit/src/text_regions/grouping.rs b/crates/edit/src/text_regions/grouping.rs index de7c47cd81..69b3a16850 100644 --- a/crates/edit/src/text_regions/grouping.rs +++ b/crates/edit/src/text_regions/grouping.rs @@ -399,7 +399,13 @@ fn group_layout(lines: &[TextLine], mixed: bool) -> Vec { block.reflow_text = reflow_text(block, lines); } for block in blocks.iter_mut().filter(|block| block.invisible && block.lines.len() > 1) { - if let Some(size) = median_size(block.lines.iter().filter_map(|i| lines.get(*i)).map(|line| line.size)) { + // Each word counts once: neighbouring words OCR sized alike share a run. + let words = block + .lines + .iter() + .filter_map(|i| lines.get(*i)) + .flat_map(|line| std::iter::repeat_n(line.size, line.text.split_whitespace().count().max(1))); + if let Some(size) = median_size(words) { block.size = size; } } diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index cc32894a54..444cfdd22a 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -323,13 +323,17 @@ fn calibration_factors( /// The calibration factors fit each word to its box at its own size (its `Tf` in its text space, /// which a range's size doesn't change). An OCR phrase drawn at one size needs the inverse ratio -/// too, so every word still spans the box the scan shows it in. +/// too, so every word still spans the box the scan shows it in. The ratio is not clamped again: +/// a word OCR sized at three times the phrase's needs three times the calibration. fn fit_to_phrase(calibration: &mut HashMap, lines: &[TextLine], phrase: Option) { let Some(phrase) = phrase else { return }; for (index, factor) in calibration.iter_mut() { let own = lines.get(*index).and_then(|source| Some(source.origin.state.font.as_ref()?.1 * source.origin.k)); if let Some(own) = own.filter(|own| own.is_finite() && *own > 0.0) { - *factor = (*factor * own / phrase).clamp(0.5, 2.0); + let fitted = *factor * own / phrase; + if fitted.is_finite() { + *factor = fitted; + } } } } diff --git a/crates/engine/tests/it/ocr_edit.rs b/crates/engine/tests/it/ocr_edit.rs index fa33a84858..c0dd805ddd 100644 --- a/crates/engine/tests/it/ocr_edit.rs +++ b/crates/engine/tests/it/ocr_edit.rs @@ -205,6 +205,56 @@ fn a_new_font_for_an_ocr_phrase_keeps_its_one_size() { } } +/// Two neighbouring words OCR sized alike join into one run; the median still counts each word, +/// so 15, 9 and 9 pt make a 9 pt phrase, not the 12 pt of its two runs. +#[test] +fn words_joined_into_one_run_each_count_toward_the_phrase_size() { + let (s, id) = scanned_with(vec![ + scanned_word("John", 20.0, 45.0, 70.0, 15.0), + scanned_word("SMITH", 49.0, 78.0, 70.0, 9.0), + scanned_word("JONES", 82.0, 112.0, 70.0, 9.0), + ]); + let doc = s.get(id).unwrap(); + assert_eq!(doc.text_lines(0).len(), 2, "the two 9 pt words share a run"); + let size = doc.text_blocks(0)[0].size; + assert!((size - 9.0).abs() < 0.05, "the phrase is set in its words' median, 9 pt: {size}"); +} + +/// A word OCR sized at more than twice the phrase's size still keeps the width it has on the scan. +#[test] +fn a_word_far_larger_than_its_phrase_keeps_its_width() { + let (mut s, id) = scanned_with(vec![ + scanned_word("John", 20.0, 45.0, 70.0, 25.0), + scanned_word("Smith", 49.0, 78.0, 70.0, 10.0), + scanned_word("Example", 82.0, 124.0, 70.0, 10.5), + ]); + s.apply(id, Edit::EditTextBlock { page: 0, block: 0, text: "John Smith Exam".into(), style: Default::default() }).unwrap(); + let lines = s.get(id).unwrap().text_lines(0); + assert_eq!(lines[0].text.trim(), "John"); + assert!((lines[0].rect[2] - lines[0].rect[0] - 25.0).abs() < 0.5, "John keeps its width: {:?}", lines[0].rect); +} + +/// With a new font the phrase is one run in its first word's text state. How wide it is must not +/// depend on how far OCR misjudged that word's size: the same boxes give the same line. +#[test] +fn a_new_font_for_an_ocr_phrase_is_as_wide_whatever_its_first_word_was_sized() { + let width = |first: f64| { + let (mut s, id) = scanned_with(vec![ + scanned_word("John", 20.0, 45.0, 70.0, first), + scanned_word("Smith", 49.0, 78.0, 70.0, 11.0), + scanned_word("Example", 82.0, 124.0, 70.0, 11.0), + ]); + let style = pdfcraft_engine::BlockStyle { family: Some((pdfcraft_engine::FontFamily::Helvetica, false, false)), ..Default::default() }; + // Short enough not to wrap in the phrase's box either way. + s.apply(id, Edit::EditTextBlock { page: 0, block: 0, text: "John Smith".into(), style }).unwrap(); + let lines = s.get(id).unwrap().text_lines(0); + assert_eq!(lines.len(), 1, "one line: {lines:?}"); + lines.iter().map(|l| l.rect[2]).fold(f64::MIN, f64::max) - lines.iter().map(|l| l.rect[0]).fold(f64::MAX, f64::min) + }; + let (guessed, right) = (width(15.0), width(11.0)); + assert!((guessed - right).abs() < 0.5, "a 15 pt guess for John makes the line {guessed} wide, an 11 pt one {right}"); +} + /// Visible text set with a horizontally stretched text matrix (3 × 1), as some generators write. fn stretched_visible() -> std::sync::Arc> { use pdfcraft_cos::{Dict, Document, Object, SaveOptions, Stream}; From 9121f65564d8bedb9cbffc15c9ea817442abf59c Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 12/37] M10: OCR models load from bytes; recognised lines travel as JSON (ocr.ocr-web) The web build has no model files: it downloads them. `Ocr::from_bytes` loads the pair only when each file's SHA-256 matches ATTRIBUTION.toml (rten's ONNX parser can spin forever on bytes that aren't a model), and `OcrsRecognizer::from_ocr` wraps the result as an engine that stays available without files. The web build's workers send what they read back as JSON: `lines_to_json` / `lines_from_json` carry each word's box and, when it has one, its confidence; the reader credits the words to the engine the worker ran, and refuses anything else (a missing or non-finite number, too many words) with an error. --- Cargo.lock | 12 ++++ crates/ocr/Cargo.toml | 2 + crates/ocr/src/lib.rs | 139 ++++++++++++++++++++++++++++++++++++++++- crates/ocr/src/ocrs.rs | 42 +++++++++++-- 4 files changed, 190 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6710dddeea..9cfe8c9c00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4540,10 +4540,22 @@ dependencies = [ "pdfcraft-fonts", "png 0.18.1", "rten", + "serde_json", + "sha2 0.11.1", "tempfile", "thiserror 2.0.21", ] +[[package]] +name = "pdfcraft-ocr-worker" +version = "0.18.1" +dependencies = [ + "js-sys", + "pdfcraft-ocr", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "pdfcraft-optimize" version = "0.18.1" diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 1b04dbb522..60e66d46c9 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -14,6 +14,8 @@ workspace = true pdfcraft-fonts = { workspace = true } ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } rten = { version = "0.26.0", default-features = false, features = ["onnx_format", "rten_format"] } +serde_json.workspace = true +sha2 = "0.11" thiserror.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index ff5b920dca..aac95f18c6 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -29,7 +29,9 @@ pub mod tesseract; pub use confidence::{ConfidenceBand, band_for, is_suspect, low_confidence}; pub use merge::{MergeStrategy, StrategyOutcome, mean_confidence, recognize_with_strategy, rover_merge}; -pub use ocrs::{ALPHABET, DETECTION_MODEL, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL}; +pub use ocrs::{ + ALPHABET, DETECTION_MODEL, DETECTION_MODEL_SHA256, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256, +}; pub use pdfcraft_fonts::TextPlan; pub use pdfcraft_fonts::helvetica_width; @@ -464,6 +466,58 @@ pub fn text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec< out } +/// The recognised lines as JSON, `[[["word", left, top, right, bottom], …], …]`, with the +/// word's confidence as a sixth item when it has one: how the web build's OCR workers send what +/// they found back to the page. +pub fn lines_to_json(lines: &[Line]) -> String { + let word = |w: &Word| { + let mut items = vec![serde_json::json!(w.text), w.rect[0].into(), w.rect[1].into(), w.rect[2].into(), w.rect[3].into()]; + items.extend(w.confidence.map(serde_json::Value::from)); + serde_json::Value::Array(items) + }; + serde_json::Value::Array(lines.iter().map(|l| serde_json::Value::Array(l.words.iter().map(word).collect())).collect()).to_string() +} + +/// At most this many words come back for one page (a dense page has a few thousand). +const MAX_JSON_WORDS: usize = 100_000; + +/// [`lines_to_json`] read back, the words credited to `source` (the engine the worker ran). +/// Anything else (a word without its box, a box or confidence that isn't a finite number, too +/// many words) is an error rather than a guess. +pub fn lines_from_json(json: &str, source: &str) -> Result, OcrError> { + let bad = |why: &str| OcrError::Recognize(format!("unreadable result: {why}")); + let value: serde_json::Value = serde_json::from_str(json).map_err(|e| bad(&e.to_string()))?; + let lines = value.as_array().ok_or_else(|| bad("not a list of lines"))?; + let number = |v: &serde_json::Value| v.as_f64().map(|v| v as f32).filter(|v| v.is_finite()); + let mut count = 0usize; + let mut out = Vec::with_capacity(lines.len().min(MAX_JSON_WORDS)); + for line in lines { + let mut words = Vec::new(); + for word in line.as_array().ok_or_else(|| bad("a line is not a list of words"))? { + count += 1; + if count > MAX_JSON_WORDS { + return Err(bad("too many words")); + } + let (text, rect, confidence) = match word.as_array().map(Vec::as_slice) { + Some([serde_json::Value::String(text), l, t, r, b]) => (text, [l, t, r, b], None), + Some([serde_json::Value::String(text), l, t, r, b, c]) => (text, [l, t, r, b], Some(c)), + _ => return Err(bad("a word is not [text, left, top, right, bottom, confidence?]")), + }; + let mut r = [0f32; 4]; + for (slot, v) in r.iter_mut().zip(rect) { + *slot = number(v).ok_or_else(|| bad("a word's box is not finite numbers"))?; + } + let mut w = Word::new(text.clone(), r, source); + if let Some(c) = confidence { + w = w.with_confidence(number(c).ok_or_else(|| bad("a word's confidence is not a finite number"))?); + } + words.push(w); + } + out.push(Line { words }); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -854,6 +908,89 @@ mod tests { assert_eq!(ok.gray(-1, -1), Some(255)); } + /// The web build loads the models from downloaded bytes; they read the same text. + #[test] + fn reads_rendered_text_with_models_from_bytes() { + let Some(models) = Models::find().filter(|m| m.detection.ends_with(DETECTION_MODEL)) else { + eprintln!("skipped: ONNX OCR models not installed"); + return; + }; + let ocr = Ocr::from_bytes(std::fs::read(&models.detection).unwrap(), std::fs::read(&models.recognition).unwrap()).unwrap(); + let rec = OcrsRecognizer::from_ocr(ocr); + assert!(rec.available(), "a recogniser loaded from bytes needs no model files"); + let (w, h, px) = test_image::hello(); + let lines = rec.recognize(&OcrImage::new(w, h, px).unwrap(), &RecognizeOptions::default()).unwrap(); + let text: Vec = lines.iter().map(Line::text).collect(); + assert!(text.iter().any(|t| t.contains("HELL") && t.contains("WOR")), "{text:?}"); + } + + /// The web build's workers send lines back as JSON; they arrive unchanged, credited to the + /// engine the worker ran. + #[test] + fn lines_survive_json() { + let lines = vec![ + Line { + words: vec![ + Word::new("Total \"€250\"", [1.5, 2.0, 30.25, 12.0], "ocrs"), + Word::new("due", [32.0, 2.0, 50.0, 12.0], "ocrs").with_confidence(87.5), + ], + }, + Line::default(), + Line { words: vec![Word::new("日本語", [0.0, 20.0, 9.0, 30.0], "ocrs")] }, + ]; + assert_eq!(lines_from_json(&lines_to_json(&lines), "ocrs").unwrap(), lines); + assert_eq!(lines_from_json(&lines_to_json(&[]), "ocrs").unwrap(), Vec::::new()); + let other = lines_from_json(&lines_to_json(&lines), "tesseract").unwrap(); + assert!(other.iter().flat_map(|l| &l.words).all(|w| w.source == "tesseract")); + } + + /// A result that isn't what a worker sends is an error, not a crash or a guess. + #[test] + fn malformed_json_lines_are_an_error() { + for json in [ + "", + "{}", + "[1]", + "[[1]]", + "[[[\"a\", 1, 2, 3]]]", + "[[[\"a\", 1, 2, 3, 4, 5, 6]]]", + "[[[1, 1, 2, 3, 4]]]", + "[[[\"a\", \"1\", 2, 3, 4]]]", + "[[[\"a\", 1e999, 2, 3, 4]]]", + "[[[\"a\", null, 2, 3, 4]]]", + "[[[\"a\", 1, 2, 3, 4, null]]]", + "[[[\"a\", 1, 2, 3, 4, \"90\"]]]", + ] { + assert!(matches!(lines_from_json(json, "ocrs"), Err(OcrError::Recognize(_))), "{json}"); + } + let many = format!("[[{}]]", vec!["[\"a\",0,0,1,1]"; MAX_JSON_WORDS + 1].join(",")); + assert!(lines_from_json(&many, "ocrs").is_err()); + let deep = format!("{}{}", "[".repeat(10_000), "]".repeat(10_000)); + assert!(lines_from_json(&deep, "ocrs").is_err()); + } + + /// A download that isn't a model (an HTML error page, a truncated file) is an error, not a + /// crash or a hang. + #[test] + fn models_from_bad_bytes_are_an_error() { + for (detection, recognition) in [(Vec::new(), Vec::new()), (b"".to_vec(), b"RTEN".to_vec()), (vec![0xff; 64], vec![0; 64])] { + assert!(matches!(Ocr::from_bytes(detection, recognition), Err(OcrError::Load(..)))); + } + } + + /// The hashes [`Ocr::from_bytes`] checks are the ones `cargo xtask models` verifies. + #[test] + fn model_hashes_match_attribution() { + let manifest = include_str!("../../../ATTRIBUTION.toml"); + for (file, sha256) in [(DETECTION_MODEL, DETECTION_MODEL_SHA256), (RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256)] { + let entry = manifest + .split("[[fetched]]") + .find(|e| e.contains(&format!("file = \"{file}\""))) + .unwrap_or_else(|| panic!("{file} is not in ATTRIBUTION.toml")); + assert!(entry.lines().any(|l| l == format!("sha256 = \"{sha256}\"")), "{file}: SHA-256 differs from ATTRIBUTION.toml"); + } + } + mod test_image { /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. pub fn hello() -> (u32, u32, Vec) { diff --git a/crates/ocr/src/ocrs.rs b/crates/ocr/src/ocrs.rs index 7f8feb47c4..94f2abfa18 100644 --- a/crates/ocr/src/ocrs.rs +++ b/crates/ocr/src/ocrs.rs @@ -32,6 +32,15 @@ pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; /// The model files, as named in ATTRIBUTION.toml. pub const DETECTION_MODEL: &str = "text-detection.onnx"; pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; +/// Their SHA-256s, as in ATTRIBUTION.toml. [`Ocr::from_bytes`] loads nothing else: rten's ONNX +/// parser can spin forever on bytes that aren't a model (64 × 0xff does). +pub const DETECTION_MODEL_SHA256: &str = "a917b23dbd9524b465df7e922641b3ff2981623df4ded5a0234004ef2fee7cfe"; +pub const RECOGNITION_MODEL_SHA256: &str = "86c145c2edb96c8caed5b1ebb8f44d706408922211451c309c625157dd6061c5"; + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect() +} pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -84,9 +93,25 @@ pub struct Ocr { impl Ocr { pub fn load(models: &Models) -> Result { let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); + Self::with_models(load(&models.detection)?, load(&models.recognition)?) + } + + /// Load the models from the contents of their files, where there is no file system (the web + /// build downloads them). Bytes that aren't the files ATTRIBUTION.toml names are an error. + pub fn from_bytes(detection: Vec, recognition: Vec) -> Result { + let load = |name: &str, sha256: &str, data: Vec| { + if sha256_hex(&data) != sha256 { + return Err(OcrError::Load(name.into(), "not the expected model file (SHA-256 differs)".into())); + } + rten::Model::load(data).map_err(|e| OcrError::Load(name.into(), e.to_string())) + }; + Self::with_models(load(DETECTION_MODEL, DETECTION_MODEL_SHA256, detection)?, load(RECOGNITION_MODEL, RECOGNITION_MODEL_SHA256, recognition)?) + } + + fn with_models(detection: rten::Model, recognition: rten::Model) -> Result { let params = ocrs::OcrEngineParams { - detection_model: Some(load(&models.detection)?), - recognition_model: Some(load(&models.recognition)?), + detection_model: Some(detection), + recognition_model: Some(recognition), alphabet: Some(ALPHABET.into()), ..Default::default() }; @@ -139,12 +164,21 @@ impl Ocr { /// The ocrs engine as a [`Recognizer`] (the default engine). pub struct OcrsRecognizer { engine: Ocr, + /// Loaded from bytes ([`OcrsRecognizer::from_ocr`]), not from model files: it stays + /// available without them. + in_memory: bool, } impl OcrsRecognizer { /// Load the models from `models`. pub fn load(models: &Models) -> Result { - Ok(OcrsRecognizer { engine: Ocr::load(models)? }) + Ok(OcrsRecognizer { engine: Ocr::load(models)?, in_memory: false }) + } + + /// A recogniser loaded some other way than from model files: the web build downloads the + /// models and loads them with [`Ocr::from_bytes`]. + pub fn from_ocr(engine: Ocr) -> OcrsRecognizer { + OcrsRecognizer { engine, in_memory: true } } /// Load the models found by [`Models::find`]. @@ -159,7 +193,7 @@ impl Recognizer for OcrsRecognizer { } fn available(&self) -> bool { - Models::find().is_some() + self.in_memory || Models::find().is_some() } fn languages(&self) -> Vec<(String, String)> { From 0bfb50a853fa75aebe6814d2ca2de350c3b4a55e Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 13/37] M10: read OCR jobs a page at a time; install a downloaded engine (ocr.ocr-web) `OcrJob::steps` splits `run` where the engine reads: `OcrSteps::next_page` renders the page, crops it to the region and preprocesses it, an engine reads `OcrShot::image`, and `OcrShot::place` maps the words back through the preprocessing and the crop. `run` is the same loop with `OcrShot::read`, so desktop results don't change. With only one engine, every strategy is that engine's reading, which is what the web build's workers return. `FileJob` splits `recognize_file` the same way (open, read, finish), keeping the skip-files probe on opening. `ocr::install` puts a recogniser loaded from downloaded bytes in the engine cache, so `recognizers`, `available`, `engine_unavailable_reason` and `engines_status` see it. --- crates/engine/src/ocr.rs | 539 ++++++++++++++++++++++++++----------- crates/engine/src/tests.rs | 29 ++ 2 files changed, 409 insertions(+), 159 deletions(-) diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index c4854d7e9f..2977406bdf 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -15,7 +15,8 @@ use pdfcraft_render::text::TextFilter; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; pub use pdfcraft_ocr::{ - ConfidenceBand, LANGUAGES, MergeStrategy, Models, Ocr, OcrError, PlacedWord, Recognizer, band_for, is_suspect, low_confidence, + ConfidenceBand, DETECTION_MODEL, LANGUAGES, Line, MergeStrategy, Models, Ocr, OcrError, OcrImage, PlacedWord, RECOGNITION_MODEL, Recognizer, + StrategyOutcome, Word, band_for, is_suspect, lines_from_json, low_confidence, }; use crate::{DocId, Edit, EditError, Session}; @@ -257,6 +258,23 @@ struct EngineCache { ocrs: Option>, } +/// The engines built so far (see [`recognizers`] and [`install`]). +static ENGINES: Mutex> = Mutex::new(None); + +/// Share an ocrs recogniser loaded some other way than from the model files [`Models::find`] +/// looks for: the web build downloads the models and loads them with [`Ocr::from_bytes`]. From +/// then on [`recognizers`] hands it out and the engine counts as available. +pub fn install(ocr: Ocr) -> Arc { + let ocrs: Arc = Arc::new(pdfcraft_ocr::OcrsRecognizer::from_ocr(ocr)); + ENGINES.lock().unwrap_or_else(|e| e.into_inner()).get_or_insert_with(|| EngineCache { ocrs: None }).ocrs = Some(ocrs.clone()); + ocrs +} + +/// Whether an ocrs recogniser has been loaded (from the model files, or by [`install`]). +fn ocrs_loaded() -> bool { + ENGINES.lock().unwrap_or_else(|e| e.into_inner()).as_ref().is_some_and(|c| c.ocrs.is_some()) +} + /// How long a negative engine probe (no program there) is remembered. A short TTL, not /// forever: a tesseract installed while the app runs is found by the next job, without a /// restart. An engine that was found is remembered for the session. @@ -408,10 +426,11 @@ fn models_installed_cached() -> bool { found } -/// The web build has no disk search worth caching. +/// The web build has no model files: the models count as installed once [`install`] has +/// loaded the ones it downloaded. #[cfg(target_arch = "wasm32")] fn models_installed_cached() -> bool { - Models::find().is_some() + ocrs_loaded() } /// The program's `--version` line, remembered per program path for the session (`ocr_status` @@ -473,7 +492,7 @@ pub fn engines_status() -> Vec { /// takes a location, because an agent must not choose which program runs; the tools report the /// default settings.) pub fn engines_status_for(settings: &OcrSettings) -> Vec { - let ocrs = Models::find().is_some(); + let ocrs = ocrs_loaded() || Models::find().is_some(); let mut out = vec![EngineStatus { id: "ocrs", available: ocrs, why_not: (!ocrs).then(models_missing), program: None, version: None }]; #[cfg(not(target_arch = "wasm32"))] { @@ -524,7 +543,6 @@ pub fn recognizers(settings: &OcrSettings) -> Result { let configured = configured_tesseract(settings); let tesseract = if matches!(settings.engine, EngineChoice::Tesseract | EngineChoice::Auto) { tesseract_engine(configured.as_deref()) } else { None }; - static ENGINES: Mutex> = Mutex::new(None); let mut slot = ENGINES.lock().unwrap_or_else(|e| e.into_inner()); let cache = slot.get_or_insert_with(|| EngineCache { ocrs: None }); if cache.ocrs.is_none() { @@ -567,7 +585,7 @@ fn models_missing() -> String { /// never waits. pub fn available() -> bool { let configured = std::env::var_os("PDFCRAFT_TESSERACT").map(|v| v.to_string_lossy().into_owned()); - Models::find().is_some() || tesseract_engine(configured.as_deref()).is_some() + ocrs_loaded() || Models::find().is_some() || tesseract_engine(configured.as_deref()).is_some() } /// Why `engine` cannot run right now, named for the UI's tooltips (the Engine dropdown @@ -703,138 +721,226 @@ impl OcrJob { /// Read the pages. `progress(done, total)` is called before each page; returning `false` /// stops (the pages read so far are returned). pub fn run(self, recognizers: &Recognizers, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { - let config = - RenderConfig { password: self.password.as_deref().map(Arc::from), hide_comments: self.settings.native_text_only, ..Default::default() }; - let mut r = PageRenderer::new(self.bytes.clone(), config); - let mut out = Vec::new(); let total = self.pages.len(); - let options = pdfcraft_ocr::RecognizeOptions { language: self.settings.language.clone() }; - for (done, &page) in self.pages.iter().enumerate() { - if !progress(done, total) { - break; + let mut steps = self.steps(); + let mut out = Vec::new(); + while progress(steps.done(), total) { + match steps.next_page() { + None => break, + Some(Err(skipped)) => out.push(skipped), + Some(Ok(shot)) => out.push(shot.read(recognizers)), } - let Some(info) = self.infos.get(page) else { continue }; - let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; - // The boxes of all text on the page, in view space: an earlier OCR layer's, and words - // only /ActualText names, so neither is read again. - let existing: Vec<[f32; 4]> = r - .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) - .text - .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) - .unwrap_or_default(); - if self.settings.skip_text_pages && !self.settings.force_ocr && !existing.is_empty() && has_own_text(&mut r, page) { - out.push(skip("the page already contains text")); - continue; - } - // A scan is read at its own resolution (enlarging it only blurs the letters), at - // most the chosen one. - let scale = self.render_scale(page); - let shot = r.render(RenderRequest { page, scale, ..Default::default() }); - if let Some(e) = shot.error { - out.push(skip(&e)); - continue; - } - let full = match pdfcraft_ocr::OcrImage::new(shot.width, shot.height, shot.rgba.to_vec()) { - Ok(img) => img, - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - // The optional region: crop the rendered page, clamped; an empty region skips the - // page with a named reason (nothing was asked of it). - let (image, offset) = match self.settings.region { - None => (full, [0.0f32; 2]), - Some(region) => match clamp_region(region, shot.width, shot.height) { - Some([x0, y0, x1, y1]) => match crop(&full, x0, y0, x1, y1) { - Some(c) => (c, [x0 as f32, y0 as f32]), - None => { - out.push(skip("the selected region is empty")); - continue; - } - }, - None => { - out.push(skip("the selected region is outside the page")); - continue; - } - }, - }; - // Preprocessing sits between the page raster and recognition; its word boxes come - // back in processed coordinates and are mapped to the ORIGINAL raster's before - // placement. The render dpi is the real one (pixels per point × 72), never a - // fixed value. - let pre = pdfcraft_ocr::preprocess::preprocess( - image, - scale * 72.0, - &pdfcraft_ocr::preprocess::PreprocessOptions { - auto_rotate: self.settings.auto_rotate, - deskew: self.settings.deskew, - denoise: self.settings.denoise, - binarize: self.settings.binarize, - }, - ); - let (image, inverse) = match pre { - Ok(p) => (p.image, p.inverse_transform), - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - let outcome = match pdfcraft_ocr::recognize_with_strategy( - recognizers.primary.as_ref(), - recognizers.secondary.as_deref(), - self.settings.strategy, - &image, - &options, - ) { - Ok(o) => o, - Err(e) => { - out.push(skip(&e.to_string())); - continue; - } - }; - let scale = shot.width as f32 / info.width.max(1e-3); - let to_user = |x: f32, y: f32| { - // processed pixels → original raster pixels (the exact inverse) → past the - // region crop → page view space. - let (ox, oy) = inverse.apply(f64::from(x), f64::from(y)); - let [u, v] = info.view_to_user((ox as f32 + offset[0]) / scale, (oy as f32 + offset[1]) / scale); - [u as f64, v as f64] - }; - // Validate recognition geometry and compare against existing text in view space, - // undoing preprocessing and region offsets before testing overlap. - let (fresh, already): (Vec<_>, Vec<_>) = outcome.lines.iter().flat_map(|l| &l.words).filter(|w| box_writable(w)).partition(|w| { - let [l, t, r, b] = w.rect; - let bounds = [(l, t), (r, t), (r, b), (l, b)].into_iter().fold( - [f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], - |mut bounds, (x, y)| { - let (x, y) = inverse.apply(f64::from(x), f64::from(y)); - let x = (x as f32 + offset[0]) / scale; - let y = (y as f32 + offset[1]) / scale; - bounds[0] = bounds[0].min(x); - bounds[1] = bounds[1].min(y); - bounds[2] = bounds[2].max(x); - bounds[3] = bounds[3].max(y); - bounds - }, - ); - covered(bounds, &existing) < ALREADY_ON_PAGE - }); - let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect::>(); - let mut notes = outcome.notes; - // Only visible editable output needs display glyphs. Searchable output preserves - // every Unicode character with empty Type3 programs and ToUnicode maps. - if self.settings.output_mode == OutputMode::EditableText && !words.is_empty() { - let missing = pdfcraft_fonts::TextPlan::for_words(words.iter().map(|w| w.text.as_str())).missing(); - if missing > 0 { - notes.push(format!("{missing} characters had no font that could show them and appear as ? in the output")); - } - } - out.push(OcrPage { page, words, already_on_page: already.len(), skipped: None, notes }); } progress(total, total); out } + + /// Read the pages one at a time, preparing each page's image and recognising it separately: + /// the web build prepares pages on the page and recognises them in Web Workers. + pub fn steps(self) -> OcrSteps { + let config = + RenderConfig { password: self.password.as_deref().map(Arc::from), hide_comments: self.settings.native_text_only, ..Default::default() }; + let renderer = PageRenderer::new(self.bytes.clone(), config); + OcrSteps { job: self, renderer, next: 0 } + } +} + +/// [`OcrJob`] a page at a time: [`OcrSteps::next_page`] renders the next page, crops it to the +/// region and preprocesses it; an engine reads [`OcrShot::image`], and [`OcrShot::place`] puts +/// the words found on the page ([`OcrShot::read`] does both with the job's engines). +pub struct OcrSteps { + job: OcrJob, + renderer: PageRenderer, + next: usize, +} + +impl OcrSteps { + /// The number of pages to read. + pub fn total(&self) -> usize { + self.job.pages.len() + } + + /// The number of pages prepared (or skipped) so far. + pub fn done(&self) -> usize { + self.next + } + + /// The settings the job reads with. + pub fn settings(&self) -> &OcrSettings { + &self.job.settings + } + + /// Prepare the next page for recognition. `Err` is a page that isn't read (it already has + /// text, or couldn't be rendered or prepared), with the reason; `None` once every page is + /// done. + pub fn next_page(&mut self) -> Option> { + loop { + let page = *self.job.pages.get(self.next)?; + self.next += 1; + let Some(info) = self.job.infos.get(page).cloned() else { continue }; + return Some(self.prepare(page, info)); + } + } + + fn prepare(&mut self, page: usize, info: PageInfo) -> Result { + let settings = &self.job.settings; + let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; + let r = &mut self.renderer; + // The boxes of all text on the page, in view space: an earlier OCR layer's, and words + // only /ActualText names, so neither is read again. + let existing: Vec<[f32; 4]> = r + .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) + .text + .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) + .unwrap_or_default(); + if settings.skip_text_pages && !settings.force_ocr && !existing.is_empty() && has_own_text(r, page) { + return Err(skip("the page already contains text")); + } + // A scan is read at its own resolution (enlarging it only blurs the letters), at + // most the chosen one. + let scale = self.job.render_scale(page); + let shot = r.render(RenderRequest { page, scale, ..Default::default() }); + if let Some(e) = shot.error { + return Err(skip(&e)); + } + let full = pdfcraft_ocr::OcrImage::new(shot.width, shot.height, shot.rgba.to_vec()).map_err(|e| skip(&e.to_string()))?; + // The optional region: crop the rendered page, clamped; an empty region skips the + // page with a named reason (nothing was asked of it). + let (image, offset) = match settings.region { + None => (full, [0.0f32; 2]), + Some(region) => match clamp_region(region, shot.width, shot.height) { + Some([x0, y0, x1, y1]) => match crop(&full, x0, y0, x1, y1) { + Some(c) => (c, [x0 as f32, y0 as f32]), + None => return Err(skip("the selected region is empty")), + }, + None => return Err(skip("the selected region is outside the page")), + }, + }; + // Preprocessing sits between the page raster and recognition; its word boxes come + // back in processed coordinates and are mapped to the ORIGINAL raster's before + // placement. The render dpi is the real one (pixels per point × 72), never a + // fixed value. + let pre = pdfcraft_ocr::preprocess::preprocess( + image, + scale * 72.0, + &pdfcraft_ocr::preprocess::PreprocessOptions { + auto_rotate: settings.auto_rotate, + deskew: settings.deskew, + denoise: settings.denoise, + binarize: settings.binarize, + }, + ) + .map_err(|e| skip(&e.to_string()))?; + Ok(OcrShot { + page, + image: Some(pre.image), + inverse: pre.inverse_transform, + offset, + pixels_per_point: shot.width as f32 / info.width.max(1e-3), + info, + existing, + strategy: settings.strategy, + options: pdfcraft_ocr::RecognizeOptions { language: settings.language.clone() }, + output_mode: settings.output_mode, + }) + } +} + +/// A page prepared for recognition: the image an engine reads, and what placing the words +/// found in it needs. +pub struct OcrShot { + pub page: usize, + /// The page's pixels as the engines read them: rendered, cropped to the region and + /// preprocessed (`None` once [`OcrShot::take_image`] has handed them on). + image: Option, + /// Processed pixels → the rendered raster's (the exact inverse of preprocessing). + inverse: pdfcraft_ocr::preprocess::InverseTransform, + /// Where the region crop starts in the rendered raster. + offset: [f32; 2], + /// The rendered raster's pixels per point of page view space. + pixels_per_point: f32, + info: PageInfo, + /// The boxes of the page's own text, in view space. + existing: Vec<[f32; 4]>, + strategy: MergeStrategy, + options: pdfcraft_ocr::RecognizeOptions, + output_mode: OutputMode, +} + +impl OcrShot { + /// What the engines read (`None` once taken). + pub fn image(&self) -> Option<&OcrImage> { + self.image.as_ref() + } + + /// Hand the image on (the web build sends it to a worker), keeping what placing the words + /// needs: the page's pixels need not stay in memory while the worker reads them. + pub fn take_image(&mut self) -> Option { + self.image.take() + } + + /// Recognise the image with `recognizers` (per the job's strategy) and place the words. + pub fn read(&self, recognizers: &Recognizers) -> OcrPage { + let Some(image) = &self.image else { return self.place(Err("the page's image was handed to another reader".into())) }; + let outcome = pdfcraft_ocr::recognize_with_strategy( + recognizers.primary.as_ref(), + recognizers.secondary.as_deref(), + self.strategy, + image, + &self.options, + ); + self.place(outcome.map_err(|e| e.to_string())) + } + + /// Put the lines recognised in [`Self::image`] on the page (or skip the page with the + /// error). A single engine's reading, which is all the web build's workers run, is + /// `StrategyOutcome { lines, ..Default::default() }`: every strategy reads that way when + /// there is no second engine. + pub fn place(&self, outcome: Result) -> OcrPage { + let page = self.page; + let outcome = match outcome { + Ok(o) => o, + Err(e) => return OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(e), notes: Vec::new() }, + }; + let (info, inverse, offset, scale) = (&self.info, self.inverse, self.offset, self.pixels_per_point); + let to_user = |x: f32, y: f32| { + // processed pixels → original raster pixels (the exact inverse) → past the + // region crop → page view space. + let (ox, oy) = inverse.apply(f64::from(x), f64::from(y)); + let [u, v] = info.view_to_user((ox as f32 + offset[0]) / scale, (oy as f32 + offset[1]) / scale); + [u as f64, v as f64] + }; + // Validate recognition geometry and compare against existing text in view space, + // undoing preprocessing and region offsets before testing overlap. + let (fresh, already): (Vec<_>, Vec<_>) = outcome.lines.iter().flat_map(|l| &l.words).filter(|w| box_writable(w)).partition(|w| { + let [l, t, r, b] = w.rect; + let bounds = [(l, t), (r, t), (r, b), (l, b)].into_iter().fold( + [f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], + |mut bounds, (x, y)| { + let (x, y) = inverse.apply(f64::from(x), f64::from(y)); + let x = (x as f32 + offset[0]) / scale; + let y = (y as f32 + offset[1]) / scale; + bounds[0] = bounds[0].min(x); + bounds[1] = bounds[1].min(y); + bounds[2] = bounds[2].max(x); + bounds[3] = bounds[3].max(y); + bounds + }, + ); + covered(bounds, &self.existing) < ALREADY_ON_PAGE + }); + let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect::>(); + let mut notes = outcome.notes; + // Only visible editable output needs display glyphs. Searchable output preserves + // every Unicode character with empty Type3 programs and ToUnicode maps. + if self.output_mode == OutputMode::EditableText && !words.is_empty() { + let missing = pdfcraft_fonts::TextPlan::for_words(words.iter().map(|w| w.text.as_str())).missing(); + if missing > 0 { + notes.push(format!("{missing} characters had no font that could show them and appear as ? in the output")); + } + } + OcrPage { page, words, already_on_page: already.len(), skipped: None, notes } + } } /// Whether a word's pixel box may reach the content stream: width and height (right − left, @@ -1095,36 +1201,70 @@ pub fn recognize_file( recognizers: &Recognizers, progress: impl FnMut(usize, usize) -> bool, ) -> Result { - let mut s = Session::new(); - let id = s.open(name, None, bytes.clone(), password).map_err(|e| e.to_string())?; - if let Some(why) = s.get(id).and_then(|d| d.read_only_reason.clone()) { - return Err(why); + match FileJob::open(name, bytes, password, settings)? { + Opened::Skipped(result) => Ok(result), + Opened::Read(mut file) => { + let job = file.job.take().ok_or("the document could not be read")?; + let pages = job.run(recognizers, progress); + file.finish(pages) + } } - let job = s.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; - // The skip-files probe runs before any engine work: a file with text needs no engine at - // all, and the original bytes come back unchanged. - if settings.skip_text_files && !settings.force_ocr && job.has_text() { - let pages = job - .pages - .iter() - .map(|&page| OcrPage { - already_on_page: 0, - page, - words: Vec::new(), - skipped: Some("the file already contains text".into()), - notes: Vec::new(), - }) - .collect(); - return Ok(FileResult { bytes, pages, file_skipped: Some("the file already contains text".into()) }); +} + +/// [`recognize_file`] in two halves, for callers that read the pages themselves (the web build +/// reads them in Web Workers): [`FileJob::open`], read `job`, then [`FileJob::finish`]. +pub struct FileJob { + session: Session, + id: DocId, + bytes: Arc>, + /// The pages to read (taken by the caller). + pub job: Option, +} + +/// What [`FileJob::open`] found: a file to read, or one the skip-files option passes over +/// (its result is final, and no engine is needed). +pub enum Opened { + Read(Box), + Skipped(FileResult), +} + +impl FileJob { + pub fn open(name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings) -> Result { + let mut session = Session::new(); + let id = session.open(name, None, bytes.clone(), password).map_err(|e| e.to_string())?; + if let Some(why) = session.get(id).and_then(|d| d.read_only_reason.clone()) { + return Err(why); + } + let job = session.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; + // The skip-files probe runs before any engine work: a file with text needs no engine at + // all, and the original bytes come back unchanged. + if settings.skip_text_files && !settings.force_ocr && job.has_text() { + let pages = job + .pages + .iter() + .map(|&page| OcrPage { + already_on_page: 0, + page, + words: Vec::new(), + skipped: Some("the file already contains text".into()), + notes: Vec::new(), + }) + .collect(); + return Ok(Opened::Skipped(FileResult { bytes, pages, file_skipped: Some("the file already contains text".into()) })); + } + Ok(Opened::Read(Box::new(FileJob { session, id, bytes, job: Some(job) }))) } - let pages = job.run(recognizers, progress); - if s.apply_ocr(id, &pages).map_err(|e| e.to_string())? == 0 { - return Ok(FileResult { bytes, pages, file_skipped: None }); + + /// Add the words found and save: the searchable file, or the original when nothing was found. + pub fn finish(mut self, pages: Vec) -> Result { + if self.session.apply_ocr(self.id, &pages).map_err(|e| e.to_string())? == 0 { + return Ok(FileResult { bytes: self.bytes, pages, file_skipped: None }); + } + // Checked like a save before the caller writes it (#2166). + let bytes = self.session.save_bytes(self.id).map_err(|e| e.to_string())?; + self.session.check_save(self.id, &bytes).map_err(|e| e.to_string())?; + Ok(FileResult { bytes, pages, file_skipped: None }) } - // Checked like a save before the caller writes it (#2166). - let bytes = s.save_bytes(id).map_err(|e| e.to_string())?; - s.check_save(id, &bytes).map_err(|e| e.to_string())?; - Ok(FileResult { bytes, pages, file_skipped: None }) } #[cfg(test)] @@ -1639,6 +1779,87 @@ mod tests { assert_eq!(again.pages[0].skipped.as_deref(), Some("the page already contains text")); } + /// The web build prepares pages with [`OcrJob::steps`] and has a worker read each image, + /// its lines coming back as JSON: the pages it ends with are the ones [`OcrJob::run`] gives, + /// with a region crop and preprocessing between the render and the engine. + #[test] + fn reading_page_by_page_matches_a_run() { + let mut s = Session::new(); + let bytes = s.create_from_text("t", "One page\nanother page").unwrap(); + let id = s.open("t.pdf", None, bytes, None).unwrap(); + let settings = OcrSettings { force_ocr: true, deskew: true, binarize: true, region: Some([40.0, 30.0, 900.0, 700.0]), ..Default::default() }; + let (r, fake) = fake_recognizers(); + let run = s.ocr_job(id, &[], settings.clone()).unwrap().run(&r, |_, _| true); + assert!(run.iter().all(|p| p.skipped.is_none() && p.words.len() == 1), "{run:?}"); + let mut steps = s.ocr_job(id, &[], settings).unwrap().steps(); + assert_eq!((steps.done(), steps.total()), (0, run.len())); + let mut stepped = Vec::new(); + while let Some(next) = steps.next_page() { + let shot = next.unwrap(); + let json = pdfcraft_ocr::lines_to_json(&fake.recognize(shot.image().unwrap(), &pdfcraft_ocr::RecognizeOptions::default()).unwrap()); + let lines = lines_from_json(&json, "fake").map_err(|e| e.to_string()); + stepped.push(shot.place(lines.map(|lines| StrategyOutcome { lines, ..Default::default() }))); + } + assert_eq!(steps.done(), run.len()); + assert_eq!(stepped, run); + // An engine's failure skips the page with the reason, as in a run. + let mut steps = s.ocr_job(id, &[0], OcrSettings { force_ocr: true, ..Default::default() }).unwrap().steps(); + let mut shot = steps.next_page().unwrap().unwrap(); + let page = shot.place(Err("worker gone".into())); + assert_eq!(page.skipped.as_deref(), Some("worker gone")); + // Once the image is handed on, it can't be read here as well. + assert!(shot.take_image().is_some() && shot.image().is_none()); + assert!(shot.read(&r).skipped.is_some()); + assert!(steps.next_page().is_none()); + } + + /// A page the job doesn't read comes out of [`OcrSteps::next_page`] as skipped, with its + /// reason, before any engine is involved. + #[test] + fn page_by_page_skips_text_pages_without_an_engine() { + let mut s = Session::new(); + let bytes = s.create_from_text("t", "A page with text").unwrap(); + let id = s.open("t.pdf", None, bytes, None).unwrap(); + let mut steps = s.ocr_job(id, &[], OcrSettings::default()).unwrap().steps(); + let Some(Err(page)) = steps.next_page() else { panic!("the text page should be skipped") }; + assert_eq!(page.skipped.as_deref(), Some("the page already contains text")); + } + + /// The web build has no model files: a recogniser loaded from downloaded bytes is installed + /// and from then on is the engine every job reads with. + #[test] + fn an_installed_recognizer_is_available_and_handed_out() { + let Some(models) = Models::find().filter(|m| m.detection.ends_with(DETECTION_MODEL)) else { + eprintln!("skipped: ONNX OCR models not installed"); + return; + }; + let ocr = Ocr::from_bytes(std::fs::read(&models.detection).unwrap(), std::fs::read(&models.recognition).unwrap()).unwrap(); + let installed = install(ocr); + assert!(installed.available()); + assert!(available()); + let r = recognizers(&OcrSettings::default()).unwrap(); + assert!(Arc::ptr_eq(&r.primary, &installed), "the installed recogniser is the one handed out"); + assert!(engines_status().iter().any(|e| e.id == "ocrs" && e.available)); + } + + /// The batch's halves: a file to read opens with its job, and [`FileJob::finish`] saves + /// the searchable copy; a file the skip-files option passes over is final on opening. + #[test] + fn a_file_job_reads_in_two_halves() { + let s = Session::new(); + let bytes = s.create_from_text("t", "Already typed here").unwrap(); + let (r, fake) = fake_recognizers(); + let skip = OcrSettings { skip_text_files: true, ..Default::default() }; + let Ok(Opened::Skipped(out)) = FileJob::open("t.pdf", bytes.clone(), None, skip) else { panic!("the file should be skipped") }; + assert_eq!(out.file_skipped.as_deref(), Some("the file already contains text")); + let force = OcrSettings { force_ocr: true, ..Default::default() }; + let Ok(Opened::Read(mut file)) = FileJob::open("t.pdf", bytes.clone(), None, force.clone()) else { panic!("the file should be read") }; + let pages = file.job.take().unwrap().run(&r, |_, _| true); + let out = file.finish(pages).unwrap(); + assert_eq!((fake.calls(), out.words(), out.file_skipped.clone()), (1, 1, None)); + assert_eq!(out, recognize_file("t.pdf", bytes, None, force, &r, |_, _| true).unwrap(), "the same as reading it in one call"); + } + /// Two inputs of one name (from different folders, or differing only in case) would /// write the same output: both are found, each once. #[test] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index d230d098de..41912df9c5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5681,6 +5681,35 @@ fn backgrounds_and_watermarks_from_files() { assert!(s.apply(id, Edit::AddBackground { pages: vec![0], settings: Background::default(), replace: true, file: Some(junk) }).is_err()); } +/// The web build reads a page at a time (prepared on the page, recognised in a worker, the +/// lines sent back as JSON); with the real ocrs engine that finds the same words as reading in +/// one go (needs the models: `cargo xtask models`; skipped without them). +#[test] +fn reading_a_page_at_a_time_finds_the_same_words() { + let settings = ocr::OcrSettings::default(); + let Ok(engines) = ocr::recognizers(&settings) else { + eprintln!("skipped: OCR models not installed"); + return; + }; + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "Invoice 42: pay by Friday").unwrap(); + let typed = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(typed).unwrap()).png(0, 150.0).unwrap(); + let scan = s.create_from_images(&[("scan.png".into(), png.clone()), ("scan2.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, scan, None).unwrap(); + let at_once = s.ocr_job(id, &[], settings.clone()).unwrap().run(&engines, |_, _| true); + let mut steps = s.ocr_job(id, &[], settings).unwrap().steps(); + let mut one_by_one = Vec::new(); + while let Some(next) = steps.next_page() { + let shot = next.unwrap(); + let lines = engines.primary.recognize(shot.image().unwrap(), &pdfcraft_ocr::RecognizeOptions::default()).unwrap(); + let sent = ocr::lines_from_json(&pdfcraft_ocr::lines_to_json(&lines), engines.primary.id()).map_err(|e| e.to_string()); + one_by_one.push(shot.place(sent.map(|lines| ocr::StrategyOutcome { lines, ..Default::default() }))); + } + assert_eq!(one_by_one, at_once); + assert!(at_once.len() == 2 && at_once.iter().all(|p| p.skipped.is_none() && !p.words.is_empty()), "{at_once:?}"); +} + /// Scan & OCR ▸ Recognize text on a page that is only a picture of text (needs the models: /// `cargo xtask models`; skipped without them). #[test] From a10906e99101cd25eebdac69c05ff9ae5344ef64 Mon Sep 17 00:00:00 2001 From: bmh129 <11478682+bmh129@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:48:12 -0600 Subject: [PATCH 14/37] M10: Recognize Text in the browser, read in Web Workers (ocr.ocr-web) The web build downloads the OCR models the first time Recognize Text (or Edit on a scanned page) needs them, installs the engine, and reads pages in up to two Web Workers (apps/pdfcraft-ocr-worker) built with WebAssembly SIMD, so the page keeps responding. The page prepares each page and sends its image to an idle worker, which runs ocrs and sends the lines back. Where no worker can start, the page reads with the installed engine. Answers go back to the run that sent the page (ocr_route), so a single run and a batch can overlap. Recognize Text in Multiple Files on the web keeps main's skipped-files bucket, low-confidence notes and cancel between files. Files chosen before the models are in wait for them. A scan Edit finds while no engine can read it waits for the models instead of counting as no scan. Trunk builds the worker beside the app; `cargo xtask web-worker-link` hashes its files and names it in index.html. The web zip ships the models in models/. --- .cargo/config.toml | 5 + apps/pdfcraft-ocr-worker/Cargo.toml | 18 + apps/pdfcraft-ocr-worker/src/main.rs | 101 +++++ apps/pdfcraft-web/Trunk.toml | 7 + apps/pdfcraft-web/index.html | 2 + crates/ui-egui/Cargo.toml | 2 +- crates/ui-egui/src/app.rs | 4 + crates/ui-egui/src/canvas.rs | 5 + crates/ui-egui/src/lib.rs | 11 + crates/ui-egui/src/workflows.rs | 4 + crates/ui-egui/src/workflows/ocr_route.rs | 139 ++++++ crates/ui-egui/src/workflows/ocr_ui.rs | 270 ++++++++++- crates/ui-egui/src/workflows/ocr_web.rs | 521 ++++++++++++++++++++++ packaging/web/package.sh | 9 +- xtask/src/layers.rs | 3 +- xtask/src/main.rs | 6 + xtask/src/web.rs | 145 ++++++ 17 files changed, 1224 insertions(+), 28 deletions(-) create mode 100644 apps/pdfcraft-ocr-worker/Cargo.toml create mode 100644 apps/pdfcraft-ocr-worker/src/main.rs create mode 100644 crates/ui-egui/src/workflows/ocr_route.rs create mode 100644 crates/ui-egui/src/workflows/ocr_web.rs create mode 100644 xtask/src/web.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 5592118f6c..a1caaeaa38 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,7 @@ [alias] xtask = "run -p xtask --" + +# WebAssembly SIMD (every current browser has it; there's no runtime detection on wasm). The OCR +# engine's kernels (rten-simd) and other vectorised code only use it when compiled in. +[target.wasm32-unknown-unknown] +rustflags = ["-C", "target-feature=+simd128"] diff --git a/apps/pdfcraft-ocr-worker/Cargo.toml b/apps/pdfcraft-ocr-worker/Cargo.toml new file mode 100644 index 0000000000..7360c13440 --- /dev/null +++ b/apps/pdfcraft-ocr-worker/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pdfcraft-ocr-worker" +description = "PdfCraft in the browser: the Web Worker that recognises text (wasm32; trunk builds it with pdfcraft-web)" +version.workspace = true +edition.workspace = true +license.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +pdfcraft-ocr = { workspace = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +wasm-bindgen = "0.2" +js-sys = "0.3" +web-sys = { version = "0.3", features = ["DedicatedWorkerGlobalScope", "MessageEvent"] } diff --git a/apps/pdfcraft-ocr-worker/src/main.rs b/apps/pdfcraft-ocr-worker/src/main.rs new file mode 100644 index 0000000000..a6a41e24eb --- /dev/null +++ b/apps/pdfcraft-ocr-worker/src/main.rs @@ -0,0 +1,101 @@ +//! PdfCraft in the browser: the Web Worker that recognises text, so reading a scanned page doesn't +//! stop the page responding and several pages are read at once. Trunk builds it alongside +//! `pdfcraft-web` (see apps/pdfcraft-web/index.html); the page starts and feeds the workers +//! (crates/ui-egui/src/workflows/ocr_ui.rs). +//! +//! Messages are plain objects with a `type`: +//! +//! - `ready` (to the page): the worker is listening; send the models now. +//! - `models` (from the page): `{ detection, recognition }`, each a `Uint8Array` holding a model +//! file. Answered with `loaded`, or `failed` with an `error`. +//! - `page` (from the page): `{ id, width, height, pixels }`, the RGBA pixels of a rendered page +//! as a `Uint8Array`. Answered with `lines` `{ id, lines }`, the recognised lines as +//! `pdfcraft_ocr::lines_to_json` writes them, or `failed` `{ id, error }`. + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +#[cfg(target_arch = "wasm32")] +fn main() { + use wasm_bindgen::{JsCast, closure::Closure}; + + let Ok(scope) = js_sys::global().dyn_into::() else { + return; + }; + let reply_to = scope.clone(); + let mut ocr = None; + let on_message = Closure::::new(move |e: web_sys::MessageEvent| { + let _ = reply_to.post_message(&worker::handle(&mut ocr, &e.data())); + }); + scope.set_onmessage(Some(on_message.as_ref().unchecked_ref())); + // The handler lives as long as the worker. + on_message.forget(); + let _ = scope.post_message(&worker::message(&[("type", "ready".into())])); +} + +#[cfg(not(target_arch = "wasm32"))] +fn main() { + eprintln!("pdfcraft-ocr-worker runs in a browser: trunk builds it with pdfcraft-web"); +} + +#[cfg(target_arch = "wasm32")] +mod worker { + use pdfcraft_ocr::Ocr; + use wasm_bindgen::{JsCast, JsValue}; + + /// The models are about 12 MB together. + const MAX_MODEL_BYTES: u32 = 64 << 20; + /// The renderer keeps pages far smaller; anything larger isn't a page. + const MAX_PIXEL_BYTES: u64 = 1 << 30; + + pub fn message(fields: &[(&str, JsValue)]) -> JsValue { + let o = js_sys::Object::new(); + for (k, v) in fields { + let _ = js_sys::Reflect::set(&o, &(*k).into(), v); + } + o.into() + } + + /// Answer one message from the page. + pub fn handle(ocr: &mut Option, data: &JsValue) -> JsValue { + let get = |k: &str| js_sys::Reflect::get(data, &k.into()).unwrap_or(JsValue::UNDEFINED); + let id = get("id"); + let reply = match get("type").as_string().as_deref() { + Some("models") => load(&get("detection"), &get("recognition")).map(|o| { + *ocr = Some(o); + message(&[("type", "loaded".into())]) + }), + Some("page") => match ocr.as_ref() { + None => Err("the text recognition models aren't loaded".into()), + Some(o) => recognize(o, &get("width"), &get("height"), &get("pixels")) + .map(|lines| message(&[("type", "lines".into()), ("id", id.clone()), ("lines", lines.into())])), + }, + _ => Err("unknown message".into()), + }; + reply.unwrap_or_else(|error| message(&[("type", "failed".into()), ("id", id), ("error", error.into())])) + } + + fn bytes(v: &JsValue, max: u64) -> Result, String> { + let a = v.dyn_ref::().ok_or("expected a Uint8Array")?; + if u64::from(a.length()) > max { + return Err(format!("{} bytes is too large", a.length())); + } + Ok(a.to_vec()) + } + + fn load(detection: &JsValue, recognition: &JsValue) -> Result { + let max = u64::from(MAX_MODEL_BYTES); + Ocr::from_bytes(bytes(detection, max)?, bytes(recognition, max)?).map_err(|e| e.to_string()) + } + + fn recognize(ocr: &Ocr, width: &JsValue, height: &JsValue, pixels: &JsValue) -> Result { + let side = |v: &JsValue| v.as_f64().filter(|v| v.fract() == 0.0 && (1.0..=f64::from(u32::MAX)).contains(v)).map(|v| v as u32); + let (Some(w), Some(h)) = (side(width), side(height)) else { + return Err("the page size is not a positive whole number".into()); + }; + let px = bytes(pixels, MAX_PIXEL_BYTES)?; + if px.len() as u64 != u64::from(w) * u64::from(h) * 4 { + return Err("the pixels don't match the page size".into()); + } + ocr.recognize(&px, w, h).map(|lines| pdfcraft_ocr::lines_to_json(&lines)).map_err(|e| e.to_string()) + } +} diff --git a/apps/pdfcraft-web/Trunk.toml b/apps/pdfcraft-web/Trunk.toml index 563c2267c1..f03ccc6ea1 100644 --- a/apps/pdfcraft-web/Trunk.toml +++ b/apps/pdfcraft-web/Trunk.toml @@ -9,3 +9,10 @@ filehash = true [serve] addresses = ["127.0.0.1"] port = 8766 + +# Trunk leaves the OCR worker's files unhashed and unnamed in index.html: hash them and name the +# worker for the page (xtask/src/web.rs). +[[hooks]] +stage = "post_build" +command = "cargo" +command_arguments = ["xtask", "web-worker-link"] diff --git a/apps/pdfcraft-web/index.html b/apps/pdfcraft-web/index.html index 46b3a25e21..f93bb0812d 100644 --- a/apps/pdfcraft-web/index.html +++ b/apps/pdfcraft-web/index.html @@ -5,6 +5,8 @@ PdfCraft + +