From 7cea6dfeadf31b512fd791b2ca0b78c3d694aaad Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:19:08 +0100 Subject: [PATCH 01/46] M7: a font's metrics are past their allowance only when it refused a read `Reads::decode` and the Type 3 glyph-box read took any failed read, with nothing left of the allowance after it, for one the allowance refused. The Thai edit refuses on what the allowance itself counts (`DecodeBudget::refused`), so the two could disagree: once the edit's allowance was spent to the byte, an empty stream behind a filter nothing decodes, which is unread with any allowance, made a font's metrics read past it with no refusal counted. The edit then skipped that font of the face and went through without it, which its refusal is there to prevent. Both now take a read as refused only when `DecodeBudget::refused` grew during it, so `Metrics::past_allowance` is set exactly when the allowance counted a refusal, as its doc now says. Tests: - `an_empty_stream_reads_with_nothing_left` adds such a stream: it isn't past the allowance, and the flag agrees with the count in each case. - `a_type3_description_unread_on_its_own_is_not_past_the_allowance`: the same for a glyph description after one that spends the allowance to the byte; a description with data after it is still refused. - `a_font_of_the_face_unread_on_its_own_once_the_allowance_is_spent_still_refuses_the_edit`: the edit above is refused, where it went through. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/tests.rs | 32 ++++++++++++++++++++ crates/fonts/src/pdf.rs | 17 ++++++++--- crates/fonts/src/pdf/tests.rs | 57 +++++++++++++++++++++++++++++++---- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index f2033a64de..1f2a6c6a9a 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -14337,6 +14337,38 @@ fn a_thai_edit_that_spends_its_allowance_to_the_byte_goes_through() { assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); } +/// Only a read the allowance refuses leaves a font of the face unread for want of it. With the +/// allowance spent to the byte as above, another font of the face, whose ToUnicode is an empty +/// stream behind a filter nothing decodes (unread with any allowance), still has its program read, +/// which the allowance refuses: the edit is refused, rather than going through without that font. +#[test] +fn a_font_of_the_face_unread_on_its_own_once_the_allowance_is_spent_still_refuses_the_edit() { + let mut program = legacy_thai_program(); + program.resize((16 << 20) - 192, 0); + let mut unicode = LEGACY_THAI_UNICODE.to_vec(); + unicode.resize(256, b' '); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &program, &unicode); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + // Not a subset, so its metrics read no program, only the ToUnicode. + let mut other = legacy_thai_font(&mut doc, "THSarabunPSK", &legacy_thai_program(), LEGACY_THAI_UNICODE); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + other.set(b"ToUnicode".to_vec(), Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))))); + let other = doc.add(Object::Dict(other)); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |page| { + let Some(Object::Dict(mut resources)) = page.get(b"Resources").cloned() else { panic!("fixture") }; + let Some(Object::Dict(mut fonts)) = resources.get(b"Font").cloned() else { panic!("fixture") }; + fonts.set(b"F2".to_vec(), Object::Ref(other)); + resources.set(b"Font".to_vec(), Object::Dict(fonts)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + }) + .unwrap(); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let error = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).unwrap_err().to_string(); + assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); +} + /// A THSarabunPSK-named subset that has OpenType Thai mark positioning is a modern face: it is /// shaped with its own layout like any source font, not by the legacy shaper (which would ask /// for Windows PUA alternates such a face doesn't need). diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index f68761e02e..00ace3bf44 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -70,7 +70,9 @@ pub struct Metrics { /// A stream these metrics come from (a CMap, the program, `/CIDToGIDMap`, a Type 3 glyph /// description) was left unread because the decode allowance [`Metrics::from_dict_within`] /// was given was spent: what it would have decided (code lengths, CIDs, widths, meanings, a - /// Type 3 font's height) is a guess. + /// Type 3 font's height) is a guess. It is set exactly when the allowance counted a refusal + /// during the read ([`DecodeBudget::refused`]), which callers that share it may refuse on; a + /// stream that fails on its own data doesn't set it, with or without anything left. pub past_allowance: bool, } @@ -420,12 +422,16 @@ impl<'a> Reads<'a> { /// stage past `max` is unread. When what is left is more than what the stages before that one /// made and `max`, it is charged those two and isn't refused; otherwise, with more left than /// the stages' total (behind one filter, `max`), it is charged that total and isn't refused; - /// with no more left than that total, it spends all of it and counts as refused. + /// with no more left than that total, it spends all of it and counts as refused. Only what + /// the allowance counts as refused ([`DecodeBudget::refused`]) is: a stream that fails on its + /// own data fails the same with more left, even when it spends the last of it or nothing was + /// left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. fn decode(&mut self, stream: &Stream, max: usize) -> Option> { + let refused = self.budget.refused(); match self.budget.decode_stages_within(self.doc, stream, max) { Ok(data) => Some(data), Err(_) => { - self.refused |= self.budget.remaining() == 0; + self.refused |= self.budget.refused() > refused; None } } @@ -491,10 +497,13 @@ fn type3_glyph_box(doc: &Document, font: &Dict, reads: &mut Reads<'_>) -> Option break; } let before = reads.budget.remaining(); + let refused = reads.budget.refused(); let read = reads.budget.decode_with_damage(doc, s, MAX_TYPE3_GLYPH_BYTES.min(left)).map(|(data, _)| data); left = left.saturating_sub(before.saturating_sub(reads.budget.remaining())); let Ok(data) = read else { - if reads.budget.remaining() == 0 { + // As in `Reads::decode`, only a refusal the allowance counts: a description that fails + // on its own data, spending the last of it, leaves the rest to be refused above. + if reads.budget.refused() > refused { // The rest go unread too, for want of the reading's allowance. reads.refused = true; complete = false; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 975b71289e..7a925c0b78 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -309,17 +309,27 @@ fn a_fonts_streams_are_paid_for_from_its_allowance() { } /// With nothing left, an empty stream still reads, as the allowance reads one, and is no reason to -/// call the font read past it; a stream with data is left unread, and is. +/// call the font read past it; a stream with data is left unread, and is. An empty stream behind a +/// filter nothing decodes is unread with any allowance, so it isn't either: the font is read past +/// its allowance exactly when the allowance counts a refusal, which is what callers refuse on. #[test] fn an_empty_stream_reads_with_nothing_left() { - let past = |data: &[u8]| { + let past = |filter: Option<&str>, data: &[u8]| { let mut doc = Document::new_empty(); - let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), data.to_vec())))); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); let f = font(&mut doc, vec![("Subtype", Object::name("TrueType")), ("ToUnicode", unicode)]); - Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(0, 0)).past_allowance + let mut budget = DecodeBudget::new(0, 0); + let past = Metrics::from_dict_within(&doc, &f, &mut budget).past_allowance; + assert_eq!(past, budget.refused() > 0, "{filter:?}"); + past }; - assert!(!past(b"")); - assert!(past(b"1 beginbfchar <41> <0058> endbfchar")); + assert!(!past(None, b"")); + assert!(!past(Some("NoSuchFilter"), b"")); + assert!(past(None, b"1 beginbfchar <41> <0058> endbfchar")); } /// A stream behind one filter past its own limit, with more left than its limit, is left unread as @@ -969,6 +979,41 @@ fn type3_glyph_boxes_are_paid_for_from_the_allowance() { assert!(m.past_allowance, "a byte short, the second description is left unread"); } +/// A glyph description that fails on its own once the allowance is spent to the byte, an empty one +/// behind a filter nothing decodes, fails the same with more left: it doesn't make the metrics read +/// past the allowance, and the descriptions after it are read, or refused by the allowance. +#[test] +fn a_type3_description_unread_on_its_own_is_not_past_the_allowance() { + let mut doc = Document::new_empty(); + let procs = ["1 0 0 -20 1 300 d1", "", "1 0 0 0 1 400 d1"]; + let mut f = type3(&mut doc, [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0, 0, 0, 0], &procs); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let unread = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new())))); + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.set(b"g1".to_vec(), unread); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + with_differences(&mut f, &["g0", "g1", "g2"].map(String::from)); + let first = procs[0].len(); + let whole = Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(usize::MAX, 0)); + assert!(!whole.past_allowance); + assert_eq!((whole.ascent, whole.descent), (400.0, -20.0)); + // The first description spends it to the byte; the third, which has data, is refused. + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!(budget.refused(), 1); + assert!(m.past_allowance); + // Without the third, nothing is refused, and the metrics aren't past the allowance. + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.remove(b"g2"); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!((budget.refused(), budget.remaining()), (0, 0)); + assert!(!m.past_allowance); + assert_eq!((m.ascent, m.descent), (300.0, -20.0)); +} + /// A Type 3 font measured from its glyphs' `d1` boxes keeps them over a descriptor's `/Ascent` /// and `/Descent`, which are optional for Type 3 (ISO 32000-2 Table 120). #[test] From 973ddc17e9f5c17726e86d5ecfae7e9a21b7a540 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:46:04 +0100 Subject: [PATCH 02/46] M7: a test pins the editor's program list against a stream unread on its own The editor's program list (`embedded_sfnt_within`) reads a legacy Thai font's metrics after its program, and passes over the program when they are past the allowance. With the program spending the allowance to the byte, an empty ToUnicode behind a filter nothing decodes no longer passes it over; a ToUnicode with data is still refused, and the program isn't listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf/tests.rs | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 7a925c0b78..7c1d5ca0c0 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -434,6 +434,45 @@ fn each_stage_of_a_metrics_stream_is_held_to_its_limit() { assert_eq!(read(MAX_CID_TO_GID / 4), (false, false)); } +/// The editor's program list reads a legacy Thai font's metrics after its program. With the program +/// spending the allowance to the byte, a ToUnicode that is an empty stream behind a filter nothing +/// decodes, unread with any allowance, doesn't put the metrics past the allowance: the program is +/// listed, as with plenty. A ToUnicode with data is refused by the allowance, and the program isn't. +#[test] +fn an_editors_program_is_listed_past_a_stream_unread_on_its_own() { + let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let listed = |filter: Option<&str>, data: &[u8]| { + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); + let program = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), bytes.to_vec())))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("TrueType")), + ("BaseFont", Object::name("THSarabunPSK")), + ("FirstChar", Object::Int(65)), + ("LastChar", Object::Int(65)), + ("Widths", Object::Array(vec![Object::Int(500)])), + ("Encoding", Object::name("WinAnsiEncoding")), + ("FontDescriptor", Object::Dict(descriptor)), + ("ToUnicode", unicode), + ], + ); + let mut budget = DecodeBudget::new(bytes.len(), 0); + let listed = embedded_sfnt_within(&doc, &f, &mut budget).is_some(); + assert_eq!(budget.remaining(), 0, "the program spends the allowance to the byte"); + (listed, budget.refused()) + }; + assert_eq!(listed(Some("NoSuchFilter"), b""), (true, 0)); + assert_eq!(listed(None, b"1 beginbfchar <41> <0058> endbfchar"), (false, 1)); +} + /// The editor's program is held to its limit as the metrics reads' streams are: a 12 MiB program /// whose Flate data is about 8 MiB reads behind ASCIIHex and Flate as behind Flate alone, though /// the two stages make about 20 MiB together. A program of the limit reads behind either, and one a From 5de168955bd285ac3b5370cb1fbbfa6009e45247 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:59:28 +0100 Subject: [PATCH 03/46] M7: the docs say which failed reads the allowance doesn't count `Metrics::past_allowance`, `Reads::decode` and the Type 3 glyph-box read said a stream that fails on its own data isn't counted as refused even when it spends the last of the allowance. A stream past its own limit fails with any allowance, yet with no more left than its stages' total it is counted, as `Reads::decode` itself says just above. They now name the case that isn't: a read that fails without the allowance refusing it, such as an empty stream behind a filter nothing decodes. The program-list test's doc no longer says more than it checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf.rs | 12 +++++++----- crates/fonts/src/pdf/tests.rs | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 00ace3bf44..37bbc52819 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -72,7 +72,8 @@ pub struct Metrics { /// was given was spent: what it would have decided (code lengths, CIDs, widths, meanings, a /// Type 3 font's height) is a guess. It is set exactly when the allowance counted a refusal /// during the read ([`DecodeBudget::refused`]), which callers that share it may refuse on; a - /// stream that fails on its own data doesn't set it, with or without anything left. + /// read that fails without one, such as an empty stream behind a filter nothing decodes, + /// doesn't set it, with or without anything left. pub past_allowance: bool, } @@ -423,9 +424,9 @@ impl<'a> Reads<'a> { /// made and `max`, it is charged those two and isn't refused; otherwise, with more left than /// the stages' total (behind one filter, `max`), it is charged that total and isn't refused; /// with no more left than that total, it spends all of it and counts as refused. Only what - /// the allowance counts as refused ([`DecodeBudget::refused`]) is: a stream that fails on its - /// own data fails the same with more left, even when it spends the last of it or nothing was - /// left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. + /// the allowance counts as refused ([`DecodeBudget::refused`]) is noted: a read that fails + /// without one, such as an empty stream behind a filter nothing decodes, fails the same with + /// more left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. fn decode(&mut self, stream: &Stream, max: usize) -> Option> { let refused = self.budget.refused(); match self.budget.decode_stages_within(self.doc, stream, max) { @@ -502,7 +503,8 @@ fn type3_glyph_box(doc: &Document, font: &Dict, reads: &mut Reads<'_>) -> Option left = left.saturating_sub(before.saturating_sub(reads.budget.remaining())); let Ok(data) = read else { // As in `Reads::decode`, only a refusal the allowance counts: a description that fails - // on its own data, spending the last of it, leaves the rest to be refused above. + // without one (an empty one nothing decodes, once the allowance is spent) leaves the + // rest to be read, or refused above. if reads.budget.refused() > refused { // The rest go unread too, for want of the reading's allowance. reads.refused = true; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 7c1d5ca0c0..4c5f90044f 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -437,7 +437,7 @@ fn each_stage_of_a_metrics_stream_is_held_to_its_limit() { /// The editor's program list reads a legacy Thai font's metrics after its program. With the program /// spending the allowance to the byte, a ToUnicode that is an empty stream behind a filter nothing /// decodes, unread with any allowance, doesn't put the metrics past the allowance: the program is -/// listed, as with plenty. A ToUnicode with data is refused by the allowance, and the program isn't. +/// listed. A ToUnicode with data is refused by the allowance, and the program isn't. #[test] fn an_editors_program_is_listed_past_a_stream_unread_on_its_own() { let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); From e5d7184558e60d56cf2fd38691d083fb5244010d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 07:36:25 +0100 Subject: [PATCH 04/46] M11: Optimize stores a picture's one-bit masks as monochrome images The image pass visits the images pages draw by name. A picture's stencil `/Mask` and its `/SMask` are images of their own that pages don't name, so the monochrome step never reached them (the colour and grey settings resample an eight-bit soft mask with its picture, but store no mask as Group 4 or JBIG2). In a mixed-raster scan, a colour layer drawn through a one-bit mask that holds the text, most of the black-and-white data stayed as it was written (Flate, say). After the pictures, each picture's stencil mask and one-bit soft mask, once however many pictures name it, now goes through the monochrome step at its size (`decide_mask`): stored again as the compression asks (Group 4, JBIG2) when that is smaller, its bits as they were, so it masks as it did. It isn't resampled (its resolution isn't measured) and shares no JBIG2 dictionary (only images named directly do, which redaction can store again). A stencil pages also draw by name stays one of the pictures; a soft mask they draw is still stored as a mask. A soft mask of more bits stays, to be resampled with its picture, which wants it at eight. The report counts masks as images; Retain leaves them as they are. The pictures and the masks go through one loop (`work_through`), so masks are worked out on threads and from the run's allowance as pictures are. Tests: `a_pictures_one_bit_masks_are_stored_again`, `a_pictures_mask_stored_as_jbig2_shares_no_dictionary` and, through `doc_optimize`, `doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4`. `a_mask_past_the_runs_allowance_stays_and_is_counted` holds a mask past the run's allowance to staying, counted as refused, and the progress to counting masks with pictures. `a_picture_with_a_stencil_mask_is_downsampled_under_its_mask` now checks the mask's size and bits rather than its bytes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/automation.rs | 55 +++++ crates/optimize/README.md | 19 +- crates/optimize/src/images.rs | 251 ++++++++++++++++++++--- crates/optimize/src/lib.rs | 9 +- crates/optimize/tests/optimize.rs | 104 +++++++++- parity/acrobat-features.toml | 4 +- 7 files changed, 401 insertions(+), 43 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4342e162f8..38bd692b61 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1726,7 +1726,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; a picture's stencil /Mask and one-bit /SMask are stored again the same way, at their size, not resampled and sharing no dictionary, counted as images; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index e0ceba48c0..6be92da1c3 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -8790,6 +8790,61 @@ fn doc_optimize_stores_black_and_white_images_as_ccitt_group_4() { } } +/// A mixed-raster page: a flat colour layer drawn through a one-bit soft mask, [`scanned_page`]'s +/// scan, which the page no longer draws by name. `doc_optimize` stores the mask as Group 4, at its +/// size, still the picture's soft mask, and counts it as an image; with `mono: retain` it stays. +#[test] +fn doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4() { + use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; + let dir = workdir("optimize-mono-mask"); + let mut doc = Document::open(std::sync::Arc::new(scanned_page())).unwrap(); + let find = |doc: &Document, kind: &[u8], key: &[u8]| { + doc.object_numbers().into_iter().map(|num| ObjRef { num, generation: doc.generation(num) }).find(|r| match &*doc.get(*r) { + Object::Stream(s) => s.dict.name(key) == Some(kind), + Object::Dict(d) => d.name(key) == Some(kind), + _ => false, + }) + }; + let scan = find(&doc, b"Image", b"Subtype").unwrap(); + let page = find(&doc, b"Page", b"Type").unwrap(); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("XObject")); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(250)); + d.set(b"Height".to_vec(), Object::Int(100)); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"SMask".to_vec(), Object::Ref(scan)); + let picture = doc.add(Object::Stream(Stream::flate(d, &[0, 102, 204].repeat(250 * 100)))); + doc.update_dict(page, |p| { + let mut x = Dict::new(); + x.set(b"Scan".to_vec(), Object::Ref(picture)); + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + p.set(b"Resources".to_vec(), Object::Dict(res)); + }) + .unwrap(); + std::fs::write(dir.join("masked.pdf"), write_full(&doc, &SaveOptions::default()).unwrap()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "masked.pdf" }))["doc"].as_u64().unwrap(); + let kept = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "kept.pdf", "mono": { "compression": "retain" } })); + assert_eq!(kept["images_recompressed"].as_u64(), Some(0), "{kept}"); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "small.pdf" })); + assert_eq!((r["images"].as_u64(), r["images_recompressed"].as_u64()), (Some(2), Some(1)), "the picture and its mask: {r}"); + // In the copy written, the picture's soft mask is Group 4, at its size. + let small = Document::open(std::sync::Arc::new(std::fs::read(dir.join("small.pdf")).unwrap())).unwrap(); + let picture = small + .object_numbers() + .into_iter() + .map(|num| small.get(ObjRef { num, generation: small.generation(num) })) + .find(|o| matches!(&**o, Object::Stream(s) if s.dict.contains(b"SMask"))) + .unwrap(); + let Object::Stream(picture) = &*picture else { unreachable!() }; + let Object::Stream(mask) = &*small.get(picture.dict.reference(b"SMask").unwrap()) else { panic!("the soft mask is a stream") }; + assert_eq!(mask.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..])); + assert_eq!((mask.dict.int(b"Width"), mask.dict.int(b"Height")), (Some(1000), Some(400))); +} + /// A scan drawn at 600 ppi is averaged down to 300 ppi by default. `mono: { downsample: false }` /// keeps its size, `ppi` and `above_ppi` choose others, and values of the wrong type are refused. #[test] diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 3207b69cab..a6dfd9caed 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -32,7 +32,8 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going damaged partway, which may draw an image larger past the damage), no image is resampled. Colour and grayscale images above a threshold are resampled with a bicubic (Catmull-Rom) filter to a target resolution and stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A - stencil `/Mask` has its own resolution, so the picture changes under it and it stays as it is. + stencil `/Mask` has its own resolution, so the picture changes under it and it keeps its size and + its bits (stored again as a picture's mask, below). An image is replaced only if the result is smaller. Images over 80 million pixels are left alone by the image steps, here and below, whatever their settings (memory); Clean Up may still Flate an unfiltered or LZW one. Left alone by these colour and grey @@ -108,10 +109,18 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going - Apart from the downsampling it is lossless: the samples keep their meaning, so the colour space, decode array and `/Mask` stay, but for packed RGB, which becomes grey with one decode pair. Group 4 wins on scans, whose ragged edges defeat Flate; on clean computer-drawn 1-bit - art Flate can be smaller. Not yet: inline images, an image's own `/SMask` or `/Mask` - image (only images drawn on a page are visited), Group 4 for rows wider than 32,767 - samples, and JBIG2 past 65,535 samples on a side, where the decoder stops (its 2^28 in all - is past the 80 million pixels above). + art Flate can be smaller. + - **A picture's masks:** after the pictures, each picture's stencil `/Mask` and one-bit + `/SMask` (a picture's own images, which the pages don't draw by name), once however many + pictures name it, is stored again as the compression asks when that is smaller, at its size: + its bits stay as they are, so it masks as it did. A stencil the pages also draw by name is one + of the images above; a soft mask they draw is still stored as a mask. It isn't resampled, as its resolution isn't measured (the walk measures the + picture's), and it shares no JBIG2 dictionary, as only images named directly do. A soft mask of + more bits stays as it is, to be resampled with its picture, which wants it at eight. The report + counts masks as images. + - Not yet: inline images, a picture's mask resampled, or packed to one bit from more, Group 4 + for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side, where the decoder + stops (its 2^28 in all is past the 80 million pixels above). - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller, a dictionary shared by up to 16 images when that is smaller still. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 7bfa44dafd..24fba3ef7a 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -1364,9 +1364,9 @@ pub(crate) fn run( .collect(); let mut refs: Vec<(ObjRef, f64)> = ppi.into_iter().filter(|(r, _)| !masks.contains(r)).collect(); refs.sort_by_key(|(r, _)| (r.num, r.generation)); - let total = refs.len(); - let mut jbig2 = false; - let mut group: Vec = Vec::new(); + // Then the pictures' masks of one bit a sample, each once ([`decide_mask`]). + let mask_refs = picture_masks(doc, &refs); + let total = refs.len() + mask_refs.len(); // By number: a repaired object answers to two generations. let mut smask_uses: HashMap = HashMap::new(); for (r, _) in &refs { @@ -1374,23 +1374,96 @@ pub(crate) fn run( *smask_uses.entry(m.num).or_default() += 1; } } + let holds = settings.mono.compression == MonoCompression::Jbig2; + let threads = threads_for_items(doc, &refs, settings); + let mut out = Out { doc, report, budget, jbig2: false, group: Vec::new() }; + let pictures = Items { items: &refs, threads, first: 0, total }; + work_through( + &mut out, + &pictures, + &|doc, rest| batch_len(doc, rest, &smask_uses, holds), + &|doc, r, ppi, budget| decide(doc, r, ppi, defaults.contains(&r.num), settings, budget), + progress, + )?; + share(out.doc, &mut out.group, out.report); + let threads = threads_for_items(out.doc, &mask_refs, settings); + let masks = Items { items: &mask_refs, threads, first: refs.len(), total }; + // Each mask is an object of its own (by number), which only its own decision changes. + work_through( + &mut out, + &masks, + &|_, rest| rest.len().min(PICTURES_AT_ONCE), + &|doc, r, _, budget| decide_mask(doc, r, settings, budget), + progress, + )?; + if !progress(Stage::Images { done: total, total }) { + return Err(OptimizeError::Cancelled); + } + if out.jbig2 { + at_least_pdf_1_4(out.doc)?; + } + Ok(()) +} + +/// What the image pass changes and counts as it goes: the document, the report, the run's +/// allowance, whether it stored an image as JBIG2, and the JBIG2 images held to share a dictionary +/// ([`share`]). +struct Out<'a> { + doc: &'a mut Document, + report: &'a mut Report, + budget: &'a mut ImageBudget, + jbig2: bool, + group: Vec, +} + +/// Images the pass works through, each with its effective resolution, on up to `threads` threads; +/// for progress, the first of them is number `first` of the pass's `total`. +struct Items<'a> { + items: &'a [(ObjRef, f64)], + threads: usize, + first: usize, + total: usize, +} + +/// How many of `items` to work on at once: one below [`MIN_FOR_THREADS`], else as many as the +/// workers, and as [`PICTURE_MEMORY`] holds of the largest. +fn threads_for_items(doc: &Document, items: &[(ObjRef, f64)], settings: &Settings) -> usize { // Too few to be worth the threads, which keeps a small file's progress in step too. - let threads = if total < MIN_FOR_THREADS { - 1 - } else { - let largest = refs.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); - threads_for(crate::pool::workers(settings.workers), largest) - }; + if items.len() < MIN_FOR_THREADS { + return 1; + } + let largest = items.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); + threads_for(crate::pool::workers(settings.workers), largest) +} + +/// How [`work_through`] works out what's to become of an image, at its effective resolution, within +/// an allowance; threads share it. +type Decide<'a> = dyn Fn(&Document, ObjRef, f64, &mut ImageBudget) -> Decision + Sync + 'a; + +/// How many of the images left, from the first, [`work_through`] may work out beside each other. +type Batch<'a> = dyn Fn(&Document, &[(ObjRef, f64)]) -> usize + 'a; + +/// Works out what's to become of each of `items` by `decide`, and does it, in their order: a batch +/// of them beside each other where `batch` says they can be, one at a time otherwise. +fn work_through( + out: &mut Out<'_>, + items: &Items<'_>, + batch: &Batch<'_>, + decide: &Decide<'_>, + progress: &mut dyn FnMut(Stage) -> bool, +) -> Result<(), OptimizeError> { + let (threads, total) = (items.threads, items.total); let mut done = 0; - while let Some(rest) = refs.get(done..).filter(|rest| !rest.is_empty()) { - let run = if threads > 1 { batch_len(doc, rest, &smask_uses, settings.mono.compression == MonoCompression::Jbig2) } else { 0 }; + while let Some(rest) = items.items.get(done..).filter(|rest| !rest.is_empty()) { + let at = items.first.saturating_add(done); + let run = if threads > 1 { batch(out.doc, rest) } else { 0 }; let Some(batch) = rest.get(..run).filter(|batch| batch.len() > 1) else { - if !progress(Stage::Images { done, total }) { + if !progress(Stage::Images { done: at, total }) { return Err(OptimizeError::Cancelled); } if let Some((r, ppi)) = rest.first() { - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget); - apply(doc, decision, report, &mut jbig2, &mut group); + let decision = decide(out.doc, *r, *ppi, out.budget); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += 1; continue; @@ -1399,10 +1472,10 @@ pub(crate) fn run( // order below: a picture the order can't pay for is done again as if there were no threads, // so the same pictures are refused with any number of them. Once more than what's left has // been used, no more are started. - let allowed = budget.work_left(); + let allowed = out.budget.work_left(); let used = AtomicU64::new(0); let tried = { - let (doc, defaults) = (&*doc, &defaults); + let doc = &*out.doc; crate::pool::map( batch, threads, @@ -1411,36 +1484,74 @@ pub(crate) fn run( return None; } let mut own = ImageBudget::with_work(allowed); - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, &mut own); + let decision = decide(doc, *r, *ppi, &mut own); let cost = allowed.saturating_sub(own.work_left()); used.fetch_add(cost, Ordering::Relaxed); (own.refused() == 0).then_some((decision, cost)) }, - &mut |finished| progress(Stage::Images { done: done.saturating_add(finished), total }), + &mut |finished| progress(Stage::Images { done: at.saturating_add(finished), total }), )? }; for ((r, ppi), tried) in batch.iter().zip(tried) { let decision = match tried.flatten() { - Some((decision, cost)) if cost <= budget.work_left() => { - budget.charge(cost); + Some((decision, cost)) if cost <= out.budget.work_left() => { + out.budget.charge(cost); decision } - _ => decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget), + _ => decide(out.doc, *r, *ppi, out.budget), }; - apply(doc, decision, report, &mut jbig2, &mut group); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += batch.len(); } - share(doc, &mut group, report); - if !progress(Stage::Images { done: total, total }) { - return Err(OptimizeError::Cancelled); - } - if jbig2 { - at_least_pdf_1_4(doc)?; - } Ok(()) } +/// The masks of `pictures` the pass stores again after them, each once (by number): their soft +/// masks and their stencil masks, unless one is drawn as a picture of its own, and so is one of +/// them. A colour-key `/Mask` is an array, not an image. Their resolution isn't measured. +fn picture_masks(doc: &Document, pictures: &[(ObjRef, f64)]) -> Vec<(ObjRef, f64)> { + let drawn: HashSet = pictures.iter().map(|(r, _)| r.num).collect(); + let mut seen = HashSet::new(); + let mut masks = Vec::new(); + for (r, _) in pictures { + let object = doc.get(*r); + let Object::Stream(s) = &*object else { continue }; + for key in [&b"SMask"[..], b"Mask"] { + if let Some(m) = s.dict.reference(key) + && !drawn.contains(&m.num) + && seen.insert(m.num) + { + masks.push((m, f64::NAN)); + } + } + } + masks.sort_by_key(|(r, _)| (r.num, r.generation)); + masks +} + +/// Work out what's to become of mask `r` of a picture, changing nothing yet: stored again as the +/// monochrome settings ask ([`mono`]), at its size, when it has one bit a sample (a stencil mask, +/// or a soft mask of one bit) and that is smaller. The bits stay as they are, so it masks as it did. +/// It isn't resampled: its resolution isn't measured, and its edges are the picture's. A soft mask +/// of more bits is left to be resampled with its picture ([`process`]), which wants it at eight. It +/// shares no JBIG2 dictionary: those are for images named directly, which redaction can store again. +fn decide_mask(doc: &Document, r: ObjRef, settings: &Settings, budget: &mut ImageBudget) -> Decision { + let obj = doc.get(r); + let Object::Stream(s) = &*obj else { return Decision::NotAnImage }; + let stencil = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + if !stencil && s.dict.int(b"BitsPerComponent") != Some(1) { + return Decision::NotAnImage; + } + let refused = budget.refused(); + // Its resolution unmeasured, it isn't resampled. As where a default colour space is in force: + // at one bit a sample that changes nothing, and nothing is packed. + match mono(doc, s, f64::NAN, true, &settings.mono, budget) { + Some((new, _, _)) => Decision::Replace { image: r, before: s.raw.len(), new, resampled: false, smask: None, bits: None }, + None => Decision::Stays { refused: budget.refused() > refused }, + } +} + /// A picture's soft mask, if it names one. fn smask_of(doc: &Document, r: ObjRef) -> Option { match &*doc.get(r) { @@ -1511,7 +1622,7 @@ fn threads_for(workers: usize, largest: u64) -> usize { /// What's to become of one picture. enum Decision { - /// Not an image stream after all. + /// Not an image stream after all, or a picture's mask the pass doesn't store again. NotAnImage, /// An image that stays as it is, `refused` if the allowance couldn't pay for it. Stays { refused: bool }, @@ -1994,6 +2105,86 @@ mod dimension_tests { } } + /// A picture's mask past the run's allowance stays as it is, counted as refused, as an image + /// is; one the allowance paid for and that stays (its noise is no smaller as Group 4) isn't. The + /// progress counts the masks with the pictures. + #[test] + fn a_mask_past_the_runs_allowance_stays_and_is_counted() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let noise: Vec = (0..32 * 256) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + let page_of = |pictures: usize, masked: bool| { + let mut doc = Document::new_empty(); + let mut xobjects = Dict::new(); + let mut drawn = String::from("q 10 0 0 10 0 0 cm "); + let mut masks = Vec::new(); + for i in 0..pictures { + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(16)); + dict.set(b"Height".to_vec(), Object::Int(16)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceCMYK")); + if masked { + let mut mask = Dict::new(); + mask.set(b"Subtype".to_vec(), Object::name("Image")); + mask.set(b"ImageMask".to_vec(), Object::Bool(true)); + mask.set(b"Width".to_vec(), Object::Int(256)); + mask.set(b"Height".to_vec(), Object::Int(256)); + let mask = doc.add(Object::Stream(Stream::flate(mask, &noise))); + dict.set(b"Mask".to_vec(), Object::Ref(mask)); + masks.push(mask); + } + let picture = doc.add(Object::Stream(Stream::flate(dict, &[0; 16 * 16 * 4]))); + xobjects.set(format!("I{i}").into_bytes(), Object::Ref(picture)); + drawn.push_str(&format!("/I{i} Do ")); + } + drawn.push('Q'); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), drawn.as_bytes()))); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + (doc, page, masks) + }; + let keep = crate::ImageSettings { downsample: false, compression: crate::Compression::Retain, ..crate::ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let cost = |pictures: usize, masked: bool| { + let (mut doc, page, _) = page_of(pictures, masked); + let mut budget = ImageBudget::with_work(u64::MAX); + let mut report = Report::default(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + assert_eq!(report.images_refused, 0); + u64::MAX - budget.work_left() + }; + let mask = cost(1, true) - cost(1, false); + assert!(mask > 0); + let (mut doc, page, masks) = page_of(3, true); + let before: Vec<_> = masks.iter().map(|m| doc.get(*m)).collect(); + let mut budget = ImageBudget::with_work(cost(3, false) + 2 * mask); + let mut report = Report::default(); + let mut last = None; + let mut progress = |stage: Stage| { + if let Stage::Images { done, total } = stage { + last = Some((done, total)); + } + true + }; + run(&mut doc, &[page], &settings, &mut report, &mut progress, &mut budget).unwrap(); + assert_eq!((report.images, report.images_refused), (6, 1), "{report:?}"); + assert_eq!(last, Some((6, 6)), "three pictures and their three masks"); + assert!(masks.iter().zip(&before).all(|(m, b)| doc.get(*m) == *b), "every mask stays as it was"); + } + /// An image the allowance ran out on between being packed and being resampled is rewritten, /// as the packed one, so it isn't one that stayed as it was. #[test] diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 97296844a2..0f80698de4 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -15,7 +15,8 @@ //! that use only palette entries 0 and 1, packed to one bit) above a threshold are averaged down to a //! target resolution when grey, image masks or indexed, halftones and dithers aside (see the README), and //! are stored as CCITT Group 4 or lossless JBIG2, or 1-bit -//! Flate once resampled or packed, when that is smaller. +//! Flate once resampled or packed, when that is smaller. So are pictures' stencil and one-bit soft +//! masks, at their size. //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings, form submission, import and reset actions. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted @@ -154,7 +155,8 @@ pub enum MonoCompression { /// How black-and-white images (1-bit, CCITT fax among them; and eight-bit grey and RGB of only black /// and white, and indexed images that use only palette entries 0 and 1, packed to one bit), and -/// image masks, are resampled and stored. +/// image masks, are resampled and stored. Pictures' stencil and one-bit soft masks are stored the +/// same way, at their size. #[derive(Clone, Copy, Debug, PartialEq)] pub struct MonoSettings { /// Average images whose effective resolution is above `above_ppi` down to `target_ppi`. @@ -201,7 +203,7 @@ pub struct Settings { pub merge_fragments: bool, pub color: ImageSettings, pub gray: ImageSettings, - /// Black-and-white images, and image masks. + /// Black-and-white images, image masks, and pictures' one-bit masks. pub mono: MonoSettings, pub discard_thumbnails: bool, pub discard_alternate_images: bool, @@ -309,6 +311,7 @@ pub struct Report { /// Images merged from fragments, and how many fragments went into them. pub merged_images: usize, pub fragments_merged: usize, + /// Images the image step read: those the pages draw, and their pictures' one-bit masks. pub images: usize, pub images_resampled: usize, pub images_recompressed: usize, diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7ab25262f8..7c3e76f9b9 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -5363,7 +5363,8 @@ fn stencil(doc: &mut Document, w: u32, h: u32) -> ObjRef { } /// A picture with a stencil mask is made smaller like any other: the mask has a resolution of its -/// own, so it stays exactly as it was and the hole it cuts stays where it was. +/// own, so it keeps its size and its bits (stored again as Group 4, as a picture's mask is) and the +/// hole it cuts stays where it was. #[test] fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let mut doc = Document::new_empty(); @@ -5381,7 +5382,9 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let after = stream(&doc, img); assert_eq!(after.dict.int(b"Width"), Some(300), "150 ppi"); assert_eq!(after.dict.reference(b"Mask"), Some(mask), "the mask stays on it"); - assert_eq!(stream(&doc, mask), mask_before, "and doesn't change"); + let mask_after = stream(&doc, mask); + assert_eq!((mask_after.dict.int(b"Width"), mask_after.dict.int(b"Height")), (Some(1200), Some(1200)), "the mask isn't resampled"); + assert_eq!(decoded_image(&doc, &mask_after).unwrap(), decoded_image(&doc, &mask_before).unwrap(), "and keeps its bits"); let rendered = render(&doc); assert_eq!(rendered.len(), before.len()); let changed = before.iter().zip(&rendered).filter(|(a, b)| a.abs_diff(**b) > 96).count(); @@ -5396,6 +5399,103 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { } } +/// A picture's masks of one bit a sample, which the page doesn't draw by name, are stored again as +/// the monochrome settings ask, losslessly: a stencil `/Mask` (with a decode array that inverts it) +/// and a one-bit `/SMask` become Group 4, keep their size and bits, and stay the pictures' masks; +/// the page looks the same to the pixel. A mask two pictures share is stored once, and a stencil +/// the page also draws by name is stored as that image. An eight-bit soft mask of only black and +/// white stays as it is, to be resampled with its picture. With `mono: Retain` nothing changes. +#[test] +fn a_pictures_one_bit_masks_are_stored_again() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let decode = Object::Array(vec![1.into(), 0.into()]); + let stencil_mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true)), ("Decode", decode.clone())]); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let wide: Vec = text.iter().flat_map(|b| (0..8).rev().map(move |i| if b >> i & 1 == 1 { 255 } else { 0 })).collect(); + let eight_bit = mono_image(&mut doc, w, h, &wide, None, &[("BitsPerComponent", 8.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let drawn = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let a = masked(&mut doc, "Mask", stencil_mask); + let b = masked(&mut doc, "Mask", stencil_mask); + let c = masked(&mut doc, "SMask", one_bit); + let d = masked(&mut doc, "SMask", eight_bit); + let e = masked(&mut doc, "Mask", drawn); + page( + &mut doc, + &[("A", a), ("B", b), ("C", c), ("D", d), ("E", e), ("M", drawn)], + "q 300 0 0 120 20 660 cm /A Do Q q 300 0 0 120 320 660 cm /B Do Q q 300 0 0 120 20 520 cm /C Do Q \ + q 300 0 0 120 320 520 cm /D Do Q q 300 0 0 120 20 380 cm /E Do Q q 0 0.4 0.8 rg 300 0 0 120 320 380 cm /M Do Q", + ); + // Pictures in colour and grey are left as they are, so only the masks can change. + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let before = render(&doc); + let masks = [stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&doc, r)); + + let mut kept = doc.clone(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..settings.clone() }).unwrap(); + assert_eq!(r.images_recompressed, 0, "Retain keeps every mask as it is"); + assert_eq!([stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&kept, r)), masks); + + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_recompressed, 3, "the shared stencil once, the soft mask, the stencil drawn by name once: {r:?}"); + for (mask, old) in [(stencil_mask, &masks[0]), (one_bit, &masks[1]), (drawn, &masks[3])] { + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{mask:?}"); + assert_eq!((new.dict.int(b"Width"), new.dict.int(b"Height")), (Some(w as i64), Some(h as i64))); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, old).unwrap(), "{mask:?} keeps its bits"); + } + assert_eq!(stream(&doc, stencil_mask).dict.get(b"Decode"), Some(&decode), "the decode array stays"); + assert_eq!(stream(&doc, eight_bit), masks[2], "an eight-bit soft mask stays"); + for (picture, key, mask) in + [(a, "Mask", stencil_mask), (b, "Mask", stencil_mask), (c, "SMask", one_bit), (d, "SMask", eight_bit), (e, "Mask", drawn)] + { + assert_eq!(stream(&doc, picture).dict.reference(key.as_bytes()), Some(mask), "{picture:?} keeps its mask"); + } + assert!(render(&doc) == before, "the page looks the same"); +} + +/// Stored as JBIG2, a picture's mask is coded on its own: it shares no symbol dictionary with the +/// images the page draws by name, which do share one. +#[test] +fn a_pictures_mask_stored_as_jbig2_shares_no_dictionary() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let picture = image(&mut doc, 200, 80, 3, false, None); + let mut s = stream(&doc, picture); + s.dict.set(b"Mask".to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + let named: Vec = (0..3) + .map(|_| mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))])) + .collect(); + page( + &mut doc, + &[("P", picture), ("G0", named[0]), ("G1", named[1]), ("G2", named[2])], + "q 300 0 0 120 20 660 cm /P Do Q q 300 0 0 120 320 660 cm /G0 Do Q q 300 0 0 120 20 520 cm /G1 Do Q q 300 0 0 120 320 520 cm /G2 Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let jbig2 = MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }; + let before = render(&doc); + let old = stream(&doc, mask); + optimize(&mut doc, &Settings { color: keep, gray: keep, mono: jbig2, ..Settings::default() }).unwrap(); + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..])); + assert!(!new.dict.contains(b"DecodeParms"), "no /JBIG2Globals"); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, &old).unwrap()); + assert!(named.iter().any(|r| stream(&doc, *r).dict.get(b"DecodeParms").is_some()), "the images drawn by name share one"); + assert!(render(&doc) == before, "the page looks the same"); +} + /// A colour-key mask names sample values: resampling or JPEG would make holes where there were none /// and fill those there were, so the picture stays as it is, however the array is written. A mask /// that can't be read as a stencil is no reason to change the picture either. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ae3261876d..046c080c62 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7549,8 +7549,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] -evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary"] -notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." +evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] +notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." [[feature]] id = "optimize.only-if-smaller" From 506502386e009e0721a14a9149852326e91ece49 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 13:59:52 +0100 Subject: [PATCH 05/46] M8: the Type 3 cleanup charges the page only when there is a Type 3 font CodeRabbit found this on #6805, which landed #6341 on main. `prune_type3_from_resources` reads what the content draws through its resources (`resource_ops`) whenever the resources list any font, and only then looks for a Type 3 font, the only kind it prunes. Since #6341 that read is charged to what the content walk left of the page's allowance. So a page with only TrueType fonts and a large form without resources of its own could be refused, by Apply Redactions or Remove Hidden Information, for a read that prunes nothing: a 100 MiB form crossing the mark is read by the walk, the page's check and this read, 300 MiB against the page's 256. Without a Type 3 font, the read is now charged to an allowance of a page of its own; with one, to the page's, as before. The read is still made with any font listed, as CodeRabbit's suggested early return would not: it refuses bytes the parser can't read in the streams and forms it reads, and an early return would drop that refusal on every page without a Type 3 font. The walk reads past such bytes, so where nothing else reads the same content, text a stray `)` hides in a `TJ` array under the mark would stay while Apply Redactions and Remove Hidden Information reported success (found in review). The refusal covers only what the cleanups read, so such bytes still go by elsewhere, here as on main. The read and the pruning tell a Type 3 font the same way (`is_type3`). Tests: - `the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance` now also redacts the 100 MiB page with no Type 3 font, which goes on; it and the two tests like it (a rewritten form's cleanups, Remove Hidden Information) list an unused Type 3 font where their font cleanup runs, so that it still charges the page's allowance (`list_type3`). - `text_beside_bytes_the_parser_cannot_read_refuses_the_redaction`: four such `TJ` arrays in a page's second stream, one in a form without resources, and hidden text beside one, on pages listing only a TrueType font, are refused by Apply Redactions and Remove Hidden Information. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests.rs | 96 ++++++++++++++++++++++++++++++++++---- crates/redact/src/type3.rs | 35 +++++++++----- 2 files changed, 109 insertions(+), 22 deletions(-) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..465294dc9a 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3571,14 +3571,43 @@ fn what_content_draws_through_its_resources_is_read_within_its_allowance() { } } +/// An unused Type 3 font, listed as /T3 beside /F1 in the `/Font` of `holder`'s resources (a page's +/// or a form's), so that the font cleanup, which prunes only Type 3 fonts, charges its read of what +/// they draw to the page's allowance (without one it has an allowance of its own). +fn list_type3(doc: &mut Document, holder: ObjRef) { + let font = Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Font")), + (b"Subtype".to_vec(), Object::name("Type3")), + (b"FontBBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1000.into(), 1000.into()])), + (b"FontMatrix".to_vec(), Object::Array(vec![Object::Real(0.001), 0.into(), 0.into(), Object::Real(0.001), 0.into(), 0.into()])), + (b"CharProcs".to_vec(), Object::Dict(Dict::new())), + ])); + let font = Object::Ref(doc.add(font)); + let list = |d: &mut Dict| { + let Some(Object::Dict(res)) = d.get_mut(b"Resources") else { panic!("{holder:?} has resources") }; + let Some(Object::Dict(fonts)) = res.get_mut(b"Font") else { panic!("{holder:?} lists fonts") }; + fonts.set(b"T3".to_vec(), font); + }; + match &*doc.get(holder) { + Object::Stream(s) => { + let mut s = s.clone(); + list(&mut s.dict); + doc.set(holder, Object::Stream(s)); + } + _ => doc.update_dict(holder, list).unwrap(), + } +} + /// The cleanups after the content walk spend what the walk left of the page's allowance, one /// allowance between them. A form without resources of its own that crosses the mark is read by -/// the walk, then again by the check of what the page still draws and by the font cleanup: at -/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at -/// 50 MiB it goes on. +/// the walk, then again by the check of what the page still draws and by the font cleanup, which +/// charges the page's allowance when the page lists a Type 3 font: at 100 MiB, three reads are more +/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. With no Type 3 font +/// the font cleanup's read has an allowance of its own, so the page's pays for two reads of +/// 100 MiB, and the redaction goes on. #[test] fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", @@ -3588,16 +3617,61 @@ fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), ], ); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } +/// Text under the mark in a content stream that also holds bytes the parser can't read, inside a +/// `TJ` array, is read past by the walk and would stay. The font cleanup's read of what the content +/// draws refuses such bytes, and makes that read with any font listed, a Type 3 font or not (with +/// none, on an allowance of its own): Apply Redactions refuses the page, whether the stream is the +/// page's or a form's without resources of its own, and so does Remove Hidden Information. +#[test] +fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { + use crate::sanitize::{Hidden, remove_hidden}; + let page_of = |second: &[u8]| { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET"), + stream("", second), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]) + }; + for tail in [ + &b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET"[..], + b"BT /F1 10 Tf 20 260 Td [(TAIL) } (SECRET)] TJ ET", + b"BT /F1 10 Tf 20 260 Td [(TAILSECRET) )] TJ ET", + b"BT /F1 10 Tf 20 260 Td [) (TAILSECRET)] TJ ET", + ] { + let mut doc = page_of(tail); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "{}", String::from_utf8_lossy(tail)); + } + let mut doc = one_page( + b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /FmBare Do", + "/XObject << /FmBare 7 0 R >>", + vec![stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET")], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "a form's text"); + let mut doc = page_of(b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET"); + doc.set(ObjRef::new(4, 0), Object::Stream(Stream::from_raw(Dict::new(), b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET".to_vec()))); + assert!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), "hidden text"); +} + /// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten /// form has resources of its own, so the page's cleanups don't follow it, and draws through them a /// form without any that crosses the mark: the walk reads that form, then the check of what the -/// rewritten form still draws and its font cleanup read it again. At 100 MiB, three reads are more -/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. +/// rewritten form still draws and its font cleanup (it lists a Type 3 font) read it again. At +/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at 50 MiB +/// it goes on. #[test] fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { for (mib, refused) in [(100, true), (50, false)] { @@ -3613,6 +3687,7 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); + list_type3(&mut doc, ObjRef::new(7, 0)); mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); } @@ -3620,9 +3695,9 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { /// Removing hidden text spends what its content walk left of the page's allowance too. The page /// draws a form holding hidden text, which is rewritten, and a form without resources of its own: -/// the walk reads the second, then the check of what the page still draws and the font cleanup -/// read it again. At 100 MiB, three reads are more than the page's 256 MiB, and the removal is -/// refused; at 50 MiB it goes on. +/// the walk reads the second, then the check of what the page still draws and the font cleanup (the +/// page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the page's +/// 256 MiB, and the removal is refused; at 50 MiB it goes on. #[test] fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3636,6 +3711,7 @@ fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); + list_type3(&mut doc, ObjRef::new(3, 0)); assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); } } diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 4c3f2211e8..759f66000b 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -539,6 +539,14 @@ fn gs_font_object(doc: &Document, state_obj: &Object) -> Option { font_entry.into_iter().next() } +/// Whether font `f` (a reference, or a font dictionary) is a Type 3 font, the only kind the font +/// cleanup prunes: its `/Subtype` is read directly, as the cleanup's other readers read it, so one +/// given by reference isn't recognised by any of them. Which allowance the cleanup's read is charged +/// to, and what it prunes, both go by this. +fn is_type3(doc: &Document, f: &Object) -> bool { + doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3")) +} + /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when /// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's @@ -556,12 +564,18 @@ pub(crate) fn prune_type3_from_resources( } // The operators of forms that draw through these resources count too: dropping a font a child - // form still shows would leave its text without one. - let mut ops = crate::interp::resource_ops(doc, budget, res, content)?; + // form still shows would leave its text without one. Without a Type 3 font nothing is pruned, + // but the read is still made: it refuses bytes the parser can't read in what the content draws, + // which for a text redaction is the only check that does (the walk reads past them, so text + // they hide would stay under the mark). It is then charged to an allowance of a page of its own, + // not to what the walk left of the page's, since it decides nothing about the fonts. + let type3 = fonts.iter().any(|(_, f)| is_type3(doc, f)); + let mut own = pdfcraft_cos::DecodeBudget::page(); + let mut ops = crate::interp::resource_ops(doc, if type3 { &mut *budget } else { &mut own }, res, content)?; // So do those of a page's annotation appearances, and of soft mask groups, without resources of // their own. They are read only when there is a Type 3 font to keep, so other files are not // refused over them. - if fonts.iter().any(|(_, f)| doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3"))) { + if type3 { if let Some(page) = page { // One total for all of them, as for the content: each appearance under the cap could // still add up to far more than it. Their decoded bytes, and their forms', are charged to @@ -605,15 +619,12 @@ pub(crate) fn prune_type3_from_resources( let mut changed = false; let font_names: Vec> = fonts.iter().map(|(n, _)| n.to_vec()).collect(); for name in font_names { - if let Some(font_obj) = fonts.get(&name) { - let resolved = doc.resolve(font_obj); - if let Some(d) = resolved.as_dict() - && d.name(b"Subtype") == Some(b"Type3") - && !drawn_font_names.contains(&name) - { - fonts.remove(&name); - changed = true; - } + if let Some(font_obj) = fonts.get(&name) + && is_type3(doc, font_obj) + && !drawn_font_names.contains(&name) + { + fonts.remove(&name); + changed = true; } } From ac1eba10927457983b992a768845ca872a822cbc Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 13:59:52 +0100 Subject: [PATCH 06/46] M8: the form and hidden-text callers are tested without a Type 3 font too Review of the allowance change. Only Apply's page caller had a row with no Type 3 font listed. `the_cleanups_of_a_rewritten_form_spend_the_pages_allowance` and `removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance` now have one too: at 100 MiB the redaction and the removal go on. Doc comments that said every cleanup of a page spends the page's allowance (`prune_type3_from_resources`, `resource_ops`, `prune`) now say the font cleanup has one of its own when no Type 3 font is listed. The doc of `text_beside_bytes_the_parser_cannot_read_refuses_the_redaction` says the cleanups run only when the walk rewrites some of the page, and read what it rewrote as rewritten. The comment in `prune_type3_from_resources` no longer calls its read the only check for text: `prune_form_resources` (for a rewritten form), `prune` (when an image or form is replaced) and `prune_properties` make the same read when they run, and `unlink_type3` reads every page when the document still lists a Type 3 font. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp/prune.rs | 8 +++--- crates/redact/src/tests.rs | 44 ++++++++++++++++++++----------- crates/redact/src/type3.rs | 11 +++++--- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 39089601ec..43cc886826 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -9,8 +9,9 @@ use super::{ /// by a form without /Resources of its own, which draws through `xo` too. Such forms are read /// within `budget`, what is left of the page's allowance ([`resource_ops`]). Fails, as the cleanup /// of the fonts and properties these forms draw with does, when that can't be told: such a form -/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget`, the walk -/// passes 2,000,000 operators, or the parser can't read all of such a form or of `content`. +/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget` (for the +/// fonts' cleanup, an allowance of its own when no Type 3 font is listed), the walk passes +/// 2,000,000 operators, or the parser can't read all of such a form or of `content`. /// Corrupt data in such a form is read, tolerantly, up to the damage, which may be nothing. pub(crate) fn prune(doc: &Document, budget: &mut DecodeBudget, xo: &mut Dict, replaced: &[Vec], content: &[Vec]) -> Result<(), CosError> { if replaced.is_empty() { @@ -473,7 +474,8 @@ pub(super) fn clip_out(rects: &[[f64; 4]], ctm: &Matrix, around: [f64; 4]) -> Op /// The operators of `content`, and of every form without /Resources of its own that it draws /// through `res`, nested ones too: what draws with these resources. Each such form is read once, /// with its `/DecodeParms` given by reference, its decoded bytes charged to `budget`, so the -/// cleanups of one page that walk the same forms spend one allowance between them. An error +/// cleanups of one page that walk the same forms spend one allowance between them (all but the +/// fonts' cleanup when no Type 3 font is listed, which has one of its own). An error /// when that can't all be read: the parser can't read all of `content` or of such a form, the walk /// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. Operators are /// read one at a time, so the walk stops at one past the limit rather than holding them all. diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 465294dc9a..99e808139b 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3625,11 +3625,16 @@ fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance } } -/// Text under the mark in a content stream that also holds bytes the parser can't read, inside a -/// `TJ` array, is read past by the walk and would stay. The font cleanup's read of what the content -/// draws refuses such bytes, and makes that read with any font listed, a Type 3 font or not (with -/// none, on an allowance of its own): Apply Redactions refuses the page, whether the stream is the -/// page's or a form's without resources of its own, and so does Remove Hidden Information. +/// Text under the mark in a content stream that holds bytes the parser can't read, inside a `TJ` +/// array, is read past by the walk and would stay. The font cleanup's read of what the content +/// draws refuses such bytes in the streams and forms the walk leaves as they were, and makes that +/// read with any font listed, a Type 3 font or not (with none, on an allowance of its own). Here +/// the walk rewrites the text in the page's first stream and leaves the broken `TJ` as it was: +/// Apply Redactions refuses the page, whether that `TJ` is in the page's second stream or in a +/// form without resources of its own, and so does Remove Hidden Information. The cleanups run only +/// when the walk rewrites some of the page, and read what it rewrote as rewritten, so such bytes in +/// a stream it rewrites still go by, and so do those on a page it rewrites nothing of, unless the +/// document still lists a Type 3 font (`unlink_type3` then reads every page). #[test] fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3669,12 +3674,13 @@ fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { /// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten /// form has resources of its own, so the page's cleanups don't follow it, and draws through them a /// form without any that crosses the mark: the walk reads that form, then the check of what the -/// rewritten form still draws and its font cleanup (it lists a Type 3 font) read it again. At +/// rewritten form still draws and its font cleanup (when it lists a Type 3 font) read it again. At /// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at 50 MiB -/// it goes on. +/// it goes on. With no Type 3 font the font cleanup's read has an allowance of its own, so the +/// page's pays for two reads of 100 MiB, and the redaction goes on. #[test] fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmText Do", @@ -3687,21 +3693,25 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); - list_type3(&mut doc, ObjRef::new(7, 0)); + if type3 { + list_type3(&mut doc, ObjRef::new(7, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } /// Removing hidden text spends what its content walk left of the page's allowance too. The page /// draws a form holding hidden text, which is rewritten, and a form without resources of its own: -/// the walk reads the second, then the check of what the page still draws and the font cleanup (the -/// page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the page's -/// 256 MiB, and the removal is refused; at 50 MiB it goes on. +/// the walk reads the second, then the check of what the page still draws and the font cleanup +/// (when the page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the +/// page's 256 MiB, and the removal is refused; at 50 MiB it goes on. With no Type 3 font the font +/// cleanup's read has an allowance of its own, so the page's pays for two reads of 100 MiB, and the +/// removal goes on. #[test] fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { use crate::sanitize::{Hidden, remove_hidden}; - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmHidden Do /FmBare Do", @@ -3711,8 +3721,10 @@ fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); - list_type3(&mut doc, ObjRef::new(3, 0)); - assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 759f66000b..8f8579a7f4 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -550,7 +550,8 @@ fn is_type3(doc: &Document, f: &Object) -> bool { /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when /// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's -/// allowance. +/// allowance, when the resources list a Type 3 font, and otherwise to an allowance of a page of its +/// own (below). pub(crate) fn prune_type3_from_resources( doc: &Document, budget: &mut pdfcraft_cos::DecodeBudget, @@ -566,9 +567,11 @@ pub(crate) fn prune_type3_from_resources( // The operators of forms that draw through these resources count too: dropping a font a child // form still shows would leave its text without one. Without a Type 3 font nothing is pruned, // but the read is still made: it refuses bytes the parser can't read in what the content draws, - // which for a text redaction is the only check that does (the walk reads past them, so text - // they hide would stay under the mark). It is then charged to an allowance of a page of its own, - // not to what the walk left of the page's, since it decides nothing about the fonts. + // and on a page with nothing replaced and no properties listed no other check does, unless the + // document still lists a Type 3 font (`unlink_type3` then reads every page). The walk reads + // past them, so text they hide would stay under the mark. The read is then charged to an + // allowance of a page of its own, not to what the walk left of the page's, since it decides + // nothing about the fonts. let type3 = fonts.iter().any(|(_, f)| is_type3(doc, f)); let mut own = pdfcraft_cos::DecodeBudget::page(); let mut ops = crate::interp::resource_ops(doc, if type3 { &mut *budget } else { &mut own }, res, content)?; From f2d64ca1367412dbf998de27fa9777d419089f9b Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 15:53:40 +0100 Subject: [PATCH 07/46] M8: redaction refuses a page whose content it can't fully parse The redaction walk reads past bytes the content parser can't read: a stray `)` or `}` in a `TJ` array, say. Text they hide is neither found nor removed, while other readers draw it: Poppler's `pdftotext` extracts `[(TAIL) ) (SECRET)] TJ` as TAILSECRET. The only refusal was a side effect of the reads after the walk, which run only when the walk rewrote something, read what it rewrote as rewritten, and in the font cleanup's case only when the resources list a font. So Apply Redactions reported success and left the text where the walk rewrites nothing (the only text under the mark in such a `TJ`, or one in a form with resources of its own), in a stream it rewrites (it wrote the array out as `(TAIL) (SECRET) TJ`, without the stray byte or the brackets), and on a page whose font is set only through an ExtGState. Remove Hidden Information never counted hidden text in such a `TJ`: it found nothing to remove, or left it beside what it removed. The walk now records that it met such bytes (`Output::unparsed`, from `Parsed::skipped`, nested forms included), and Apply Redactions refuses the page with `RedactError::Unreadable`, as it does an inline image whose end is a guess. Remove Hidden Information counts such a page in its scan, so the removal runs and refuses it, as it does content that doesn't decode; it refuses an inline image whose end is a guess the same way, which it took on trust before. Apply walks only the forms that reach a mark, so bytes in a form away from the marks don't make the walk refuse the page; in a form without resources of its own, the check after the walk still refuses them when it runs, as before. Tests (`crates/redact/src/tests/unparsed.rs`): text under a mark in four broken `TJ` arrays, alone and beside other text; the same in a stream the walk rewrites; in a form under the mark with resources of its own; on a page without font resources; hidden text in such a `TJ`, alone and beside other hidden text, through the scan, Remove Hidden Information and Sanitize Document; hidden text beside an inline image the renderer can't end; and a form away from the marks, which is still redacted around. Two tests in `tests.rs` now expect the walk's refusal; the check after the walk keeps a case of its own, a form without resources away from the mark. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/README.md | 5 +- crates/redact/src/interp.rs | 7 ++ crates/redact/src/lib.rs | 6 +- crates/redact/src/sanitize.rs | 11 +++ crates/redact/src/tests.rs | 48 +++++++----- crates/redact/src/tests/unparsed.rs | 114 ++++++++++++++++++++++++++++ 6 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 crates/redact/src/tests/unparsed.rs diff --git a/crates/redact/README.md b/crates/redact/README.md index e47df98f17..d8f4b46e94 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -130,7 +130,10 @@ Text is placed with the font and size set by `Tf` or by an ExtGState's `/Font` t form XObject starts from the text state of the content that draws it. Content that can't be decoded makes the operation fail rather than leave it unredacted, and so does an inline image the renderer's tokenizer reads no image for (`RedactError::Unreadable`): where the page's operators -resume after it would be a guess. A Redact annotation with neither a usable `/QuadPoints` array nor a usable +resume after it would be a guess. So do bytes the content parser can't read, in the page's content +or in a form under an area (a stray `)` or `}` in a `TJ` array, say): the interpreter reads past +them, so text they hide, which other readers may draw, would be neither found nor removed. Remove +hidden information refuses a page for both too, in whatever content it walks. A Redact annotation with neither a usable `/QuadPoints` array nor a usable `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..ae525989a1 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -143,6 +143,11 @@ pub(crate) struct Output { /// ([`pdfcraft_content::Parsed::ambiguous_images`]): what this scope placed after it may not /// be what the page shows, so a redaction must not trust it. pub ambiguous: bool, + /// The content holds bytes the parser can't read ([`pdfcraft_content::Parsed::skipped`]), in + /// this scope or a nested form: the walk reads past them, so text they hide (a stray `)` in a + /// `TJ` array, say), which other readers may still draw, is neither found nor removed, and a + /// rewrite would drop the bytes but keep the strings. A redaction must not trust it. + pub unparsed: bool, pub tag_changes: Vec<(ObjRef, ObjRef, HashSet, HashSet)>, pub error: Option, } @@ -407,6 +412,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], let joined = Pieces::join(streams); let mut parsed = joined.parse(); walk.out.ambiguous = parsed.ambiguous_images > 0; + walk.out.unparsed = parsed.skipped > 0; // A rewritten piece loses its comments, so keep where they began and ended: a comment that // ran on into the next piece still hides that piece's first line. let comment_edges = joined.comment_edges(&parsed); @@ -1150,6 +1156,7 @@ fn xobject( out.residue += inner.residue; out.vertical |= inner.vertical; out.ambiguous |= inner.ambiguous; + out.unparsed |= inner.unparsed; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); // A Type 3 glyph without resources shown inside the form may paint with the page's diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 7c1a65f55a..5938f20a68 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -791,8 +791,10 @@ fn redact_page_content( } // An inline image the renderer reads no image for (no `EI` it accepts, or parameters it // can't read): where the page's operators resume after it is a guess, so neither removal - // nor the verify pass can be trusted on this page. - if out.ambiguous { + // nor the verify pass can be trusted on this page. Likewise bytes the parser can't read in + // what the walk read, the page's content or a form under a mark: text they hide is neither + // found nor removed, and other readers may draw it. + if out.ambiguous || out.unparsed { return Err(RedactError::Unreadable(pi + 1)); } doomed_xobjects.extend_from_slice(&out.doomed); diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index 0d4ee2e18a..3da224e8ac 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -659,6 +659,17 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re if let Some(error) = out.error { return Err(error.into()); } + // Bytes the parser can't read, or an inline image whose end is a guess: the walk reads past + // them, so hidden text or a hidden layer's content they hold, which other readers may draw, + // is neither found nor removed. As for content that doesn't decode, count the page so the + // removal runs and refuses it. + if out.unparsed || out.ambiguous { + if write { + return Err(RedactError::Unreadable(pi + 1)); + } + total += 1; + continue; + } let blocks = scope.layer_blocks; let glyphs = scope.report.glyphs; let retained_for_page = std::mem::take(&mut scope.retained_off_layers); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..4e132f20c6 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3297,31 +3297,42 @@ fn image_and_a_tail(catalog: &str, image: &str, image_dict: &str, tail: &[u8]) - ]) } -/// The refusal of the check of what content still draws through its resources (`resource_ops`). -fn unread_content_refusal(result: &Result) -> bool { - matches!(result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) - if detail == "redaction could not fully read the content that uses these resources") -} - -/// What the page draws can't be told when a content stream the redaction leaves as it was holds -/// bytes the parser can't read: the redaction of an image is refused, as it was already on a page -/// with fonts or properties to clean up. +/// What the page draws can't be told when content the redaction leaves as it was holds bytes the +/// parser can't read: the redaction of an image is refused. In a content stream of the page, the +/// walk reads past them and refuses the page itself (`Unreadable`). In a form without resources of +/// its own that lies away from the mark, which the walk doesn't enter, the check of what the page +/// still draws through its resources refuses it. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_the_redaction_of_an_image() { for (tail, refused) in [(&b"q Q )"[..], true), (b"q Q", false)] { let mut doc = image_and_a_tail("", "Im1", "", tail); mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - let result = apply(&mut doc, None); - if refused { - assert!(unread_content_refusal(&result), "{result:?}"); - } else { - assert_eq!(result.map(|r| r.images_removed).ok(), Some(1)); - } + let result = apply(&mut doc, None).map(|r| r.images_removed); + assert_eq!(result, if refused { Err(RedactError::Unreadable(1)) } else { Ok(1) }); } + let mut doc = image_and_a_tail("", "Im1", "", b"/Fm Do"); + let bbox = Object::Array(vec![0.into(), 0.into(), 300.into(), 100.into()]); + let form = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject")), (b"Subtype".to_vec(), Object::name("Form")), (b"BBox".to_vec(), bbox)]); + let form = doc.add(Object::Stream(Stream::from_raw(form, b"q Q )".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |d| { + if let Some(Object::Dict(res)) = d.get_mut(b"Resources") + && let Some(Object::Dict(xo)) = res.get_mut(b"XObject") + { + xo.set(b"Fm".to_vec(), Object::Ref(form)); + } + }) + .unwrap(); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail == "redaction could not fully inspect inherited form resources"), + "{result:?}" + ); } -/// Remove Hidden Information makes the same check: removing an off layer's image is refused on a -/// page whose drawing can't be told. +/// Remove Hidden Information refuses the same page: removing an off layer's image is refused on a +/// page whose drawing can't be told. Its walk reads every content stream, and refuses the page +/// itself. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3329,7 +3340,7 @@ fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() let mut doc = image_and_a_tail("/OCProperties << /OCGs [7 0 R] /D << /OFF [7 0 R] >> >>", "ImOC", "/OC 7 0 R", tail); let result = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); if refused { - assert!(unread_content_refusal(&result), "{result:?}"); + assert_eq!(result, Err(RedactError::Unreadable(1))); } else { assert!(result.unwrap().iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)); } @@ -6152,6 +6163,7 @@ mod metadata_matching; mod property_retirement; mod surviving_references; mod text_state; +mod unparsed; /// `[(FIRSTARR)] [(LASTARR)] TJ` is malformed (a surplus array operand), but the operator still /// draws only the one immediately before it (LASTARR, 7 glyphs, 35 pt wide from x = 10) -- not diff --git a/crates/redact/src/tests/unparsed.rs b/crates/redact/src/tests/unparsed.rs new file mode 100644 index 0000000000..4fcc9fcb75 --- /dev/null +++ b/crates/redact/src/tests/unparsed.rs @@ -0,0 +1,114 @@ +//! Bytes the content parser can't read are read past by the redaction walk, so text they hide (a +//! stray `)` or `}` in a `TJ` array) is neither found nor removed, while other readers still draw +//! it. A page whose walk meets such bytes is refused, by Apply Redactions and by Remove Hidden +//! Information. + +use super::*; +use crate::sanitize::{Hidden, remove_hidden, sanitize, scan}; + +/// `TJ` arrays each holding a byte the parser can't read beside the text. +const BROKEN: [&[u8]; 4] = [b"[(TAIL) ) (SECRET)] TJ", b"[(TAIL) } (SECRET)] TJ", b"[(TAILSECRET) )] TJ", b"[) (TAILSECRET)] TJ"]; + +/// Over the top third of the 300 × 300 page. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +/// `tj` shown in /F1 at (20, 260), under [`MARK`]. +fn shown(tj: &[u8]) -> Vec { + [b"BT /F1 10 Tf 20 260 Td ".as_slice(), tj, b" ET"].concat() +} + +/// The only text under the mark sits in such a `TJ`: the walk finds nothing there and rewrites +/// nothing, so it went on, and the text stayed in the file. +#[test] +fn text_under_a_mark_beside_bytes_the_parser_cannot_read_is_refused() { + for tj in BROKEN { + let mut doc = one_page(&shown(tj), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } + // With other text on the page, outside the mark. + let mut doc = one_page(&[b"BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET ".as_slice(), &shown(BROKEN[0])].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "beside other text"); +} + +/// In a stream the walk rewrites, it removed the text it could read and wrote the broken `TJ` out +/// as `(TAIL) (SECRET) TJ`: the stray byte and the brackets went, the strings stayed. +#[test] +fn bytes_the_parser_cannot_read_in_a_stream_the_walk_rewrites_are_refused() { + for tj in BROKEN { + let mut doc = one_page(&[b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET ".as_slice(), &shown(tj)].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } +} + +/// A form under the mark with resources of its own, which the walk enters but doesn't rewrite: +/// no later read of the page follows it. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_under_a_mark_are_refused() { + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", shown(BROKEN[0])); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, so +/// the page is redacted. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_away_from_the_marks_are_left() { + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 100] /Resources << /Font << /F1 5 0 R >> >>", + [b"BT /F1 10 Tf 20 20 Td ".as_slice(), BROKEN[0], b" ET"].concat(), + ); + let mut doc = one_page(b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("(SECRET)")); +} + +/// The page sets its font only through an ExtGState `/Font` and lists no `/Font` resources, so the +/// cleanups after the walk, which look at the listed fonts, read nothing. +#[test] +fn bytes_the_parser_cannot_read_on_a_page_without_font_resources_are_refused() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /ExtGState << /GS1 << /Font [6 0 R 10] >> >> >> >>" + .to_vec(), + stream("", b"BT /GS1 gs 20 230 Td (SECRET) Tj ET"), + stream("", b"BT /GS1 gs 20 260 Td [(TAIL) ) (SECRET)] TJ ET"), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// Hidden text inside a broken `TJ` is never counted: alone, Remove Hidden Information found +/// nothing to remove and Sanitize Document went on; beside other hidden text, the removal went on +/// and left it. The scan now counts the page, and removing hidden text refuses it. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_is_refused() { + let alone = b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET".as_slice(); + let beside = [b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET ".as_slice(), alone].concat(); + for content in [alone, &beside] { + let label = String::from_utf8_lossy(content); + let doc = one_page(content, "", vec![]); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert!(counts[&Hidden::HiddenText] > 0, "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(sanitize(&mut doc), Err(RedactError::Unreadable(1)), "{label}"); + } +} + +/// An inline image whose end the parser guesses, as Apply Redactions already refuses it: what +/// follows it may not be what the page draws, so Remove Hidden Information refuses it too. +#[test] +fn hidden_text_beside_an_inline_image_the_renderer_cannot_end_is_refused() { + let page = b"BT /F1 10 Tf 3 Tr 10 100 Td (HIDDEN) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F".as_slice(); + let mut doc = one_page(page, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1))); +} From a7f0074381c5aa0e3d4d2f85212aaa876b3491e5 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 15:53:40 +0100 Subject: [PATCH 08/46] M8: doc_hidden_info and the parity note say what the refusal covers Review of the refusal. `doc_hidden_info` said it lists what Remove Hidden Information would remove. A page that can't be read in full is counted as one item of hidden text and of hidden layers, so that the removal runs and refuses it: since the refusal before this, that includes a page holding bytes the parser can't read or an inline image whose end is a guess, not only content streams that don't decode. The description now says so, and that with form fields removed too, a field's appearance is read the same way though it isn't counted. The parity note on Apply Redactions lists the new refusal beside the inline image's. Removing hidden layers alone refuses such a page in a document without `/OCProperties` too, which a review asked to narrow: a membership dictionary there (`/AllOff` of a layer that is on) still hides what it marks, for the walk and for the app's renderer. The comment at the refusal in `sanitize.rs` says only what the bytes are shown to hide: text in a broken `TJ`. Tests: `bytes_the_parser_cannot_read_refuse_removing_hidden_layers_alone`; `hidden_text_beside_bytes_the_parser_cannot_read_in_a_field_appearance_is_refused` (removing form fields with hidden text, and Sanitize Document, which went on and left it); and `hidden_text_beside_bytes_the_parser_cannot_read_is_refused` gains hidden text in a broken `TJ` inside a form. The form-away test's doc says the check after the walk would still refuse a form without resources. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/automation/src/tools.rs | 2 +- crates/redact/src/sanitize.rs | 6 +-- crates/redact/src/tests/unparsed.rs | 60 ++++++++++++++++++++++++++++- parity/acrobat-features.toml | 2 +- 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 1cd876d0b1..a37fe49ffe 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -726,7 +726,7 @@ pub fn tools() -> Vec { t( "doc_hidden_info", "Find hidden information", - "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts.", + "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts. A page that can't be read in full (its content streams don't decode, or they or a form they draw hold bytes the parser can't read or an inline image whose end is a guess) counts as one item of hidden-text and of hidden-layers: removing either category then fails on that page. With form-fields removed too, a field's appearance becomes page content and is read the same way, though it isn't counted here.", ) .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index 3da224e8ac..cd38b9eb19 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -660,9 +660,9 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re return Err(error.into()); } // Bytes the parser can't read, or an inline image whose end is a guess: the walk reads past - // them, so hidden text or a hidden layer's content they hold, which other readers may draw, - // is neither found nor removed. As for content that doesn't decode, count the page so the - // removal runs and refuses it. + // them, so what it reads around them may not be what other readers draw (hidden text in a + // broken `TJ` is neither found nor removed). As for content that doesn't decode, count the + // page so the removal runs and refuses it. if out.unparsed || out.ambiguous { if write { return Err(RedactError::Unreadable(pi + 1)); diff --git a/crates/redact/src/tests/unparsed.rs b/crates/redact/src/tests/unparsed.rs index 4fcc9fcb75..90dcb1ae47 100644 --- a/crates/redact/src/tests/unparsed.rs +++ b/crates/redact/src/tests/unparsed.rs @@ -53,8 +53,9 @@ fn bytes_the_parser_cannot_read_in_a_form_under_a_mark_are_refused() { assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); } -/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, so -/// the page is redacted. +/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, and +/// the walk doesn't refuse the page over them. This one has resources of its own, so the check after +/// the walk doesn't read it either (it would refuse one without), and the page is redacted. #[test] fn bytes_the_parser_cannot_read_in_a_form_away_from_the_marks_are_left() { let form = stream( @@ -102,6 +103,61 @@ fn hidden_text_beside_bytes_the_parser_cannot_read_is_refused() { let mut doc = one_page(content, "", vec![]); assert_eq!(sanitize(&mut doc), Err(RedactError::Unreadable(1)), "{label}"); } + // In a form with resources of its own, which the walk enters on every page. + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", alone); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "in a form"); +} + +/// Removing hidden layers alone refuses such bytes too, even in a document without +/// `/OCProperties`: a membership dictionary (`/AllOff` of a layer that is on) still hides what it +/// marks, for the walk as for the app's renderer. With nothing in front of `/OC`, the marked text +/// is removed. Here a string beside a stray `}` is left in front of it, which keeps the block from +/// the walk (it reads a `BDC`'s first operands; a string alone does so too, a matter of its own), +/// and the stray byte has the page refused rather than the text kept. +#[test] +fn bytes_the_parser_cannot_read_refuse_removing_hidden_layers_alone() { + let marked = |before: &[u8]| { + let content = [before, b" /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET"].concat(); + one_page( + &content, + "/Properties << /MC0 8 0 R >>", + vec![b"<< /Type /OCG /Name (Layer) >>".to_vec(), b"<< /Type /OCMD /OCGs [7 0 R] /P /AllOff >>".to_vec()], + ) + }; + let mut doc = marked(b""); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + assert!(!content(&doc, 0).contains("LAYERSECRET")); + let mut doc = marked(b"(x) }"); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::Unreadable(1))); +} + +/// With form fields removed too, a field's appearance is flattened into page content and read the +/// same way, though the scan, which runs before flattening, doesn't count it: hidden text in a +/// broken `TJ` there refuses removing hidden text, and Sanitize Document, which went on and left it. +/// Removing form fields alone doesn't read the content, and goes on. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_in_a_field_appearance_is_refused() { + let fresh = || { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>" + .to_vec(), + stream("", b"BT /F1 10 Tf 10 230 Td (Name:) Tj ET"), + FONT.replace("95 0 R", "8 0 R").into_bytes(), + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /P 3 0 R /Rect [50 225 200 240] /AP << /N 7 0 R >> >>".to_vec(), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 150 15] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 3 Tr 2 3 Td [(TAIL) ) (HIDDEN)] TJ ET", + ), + widths(), + ]) + }; + assert!(remove_hidden(&mut fresh(), &[Hidden::FormFields]).is_ok()); + assert_eq!(remove_hidden(&mut fresh(), &[Hidden::FormFields, Hidden::HiddenText]), Err(RedactError::Unreadable(1))); + assert_eq!(sanitize(&mut fresh()), Err(RedactError::Unreadable(1))); } /// An inline image whose end the parser guesses, as Apply Redactions already refuses it: what diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 5f3dc08f35..bbffa91408 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5920,7 +5920,7 @@ status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] evidence = ["crates/ui-egui/tests/it/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/it/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::the_renderer_check_refuses_text_still_shown_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_does_not_survive_in_info_or_xmp", "crates/redact/src/tests.rs::metadata_is_cleared_when_the_removed_text_was_cut_short", "crates/redact/src/tests/metadata_fail_closed.rs::an_xmp_stream_whose_filter_data_is_corrupt_goes", "crates/redact/src/tests/elsewhere.rs::places_apply_leaves_that_still_hold_the_removed_text_are_counted_and_kept", "crates/engine/src/tests.rs::applied_redactions_report_a_bookmark_that_still_holds_the_removed_text", "crates/automation/tests/it/automation.rs::redact_apply_reports_places_that_still_hold_the_removed_text", "crates/ui-egui/tests/it/redact.rs::applying_says_where_the_removed_text_is_still_held", "crates/redact/src/tests/metadata_matching.rs::a_heading_tracked_with_tj_numbers_matches_the_metadata", "crates/redact/src/tests/metadata_matching.rs::unicode_forms_of_the_removed_text_match", "crates/redact/src/tests/metadata_matching.rs::a_utf16le_string_is_searched_as_utf16le", "crates/redact/src/tests/identification.rs::the_identification_of_a_removed_packet_is_written_back", "crates/automation/tests/it/automation.rs::redact_apply_reports_the_identification_written_back", "crates/ui-egui/tests/it/redact.rs::applying_says_the_pdfa_identification_was_written_back"] -notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" +notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known, or with bytes the content parser can't read in its content or in a form under a mark, which can hide text other readers draw. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" [[feature]] id = "protect.redact-glyph-removal" From d2c87ae78930a9b7a6627196e334573fc24a7305 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 09/46] M8: the redaction walk reads a form's /Matrix entries by reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The walk read a form's `/Matrix` with `Matrix::from_operands` on the array as written, so an entry given by reference made it the identity. That is legal: an array's entries may each be an indirect reference (ISO 32000-2 §7.3.10), and Arlington's `XObjectFormType1` allows the matrix itself to be one. A form moved onto a mark by such a matrix was then tested against the wrong place, never walked, and Apply Redactions reported success with its text still in the file: `/Matrix [1 0 0 1 0 8 0 R]` with `8 0 obj 200` left SECRET, which Poppler's `pdftotext` places under the mark. Remove Hidden Information missed text such a matrix moves off the page the same way. Found in review of the unparsed-bytes refusal. The walk now reads it with the proof's `matrix_of` (`verify/extract.rs`), which resolves the array and each entry. A `/Matrix` that isn't six numbers was the identity too; now where the form draws can't be told, so Apply refuses the page and the scans count it, as for a form past the page's budget. `matrix_of` now reads a null `/Matrix`, given directly or by reference, as the identity: a null entry is no entry (ISO 32000-2 §7.3.9). Tests (`crates/redact/src/tests/form_matrix.rs`): a form moved onto the mark by a matrix written direct, as a reference, and with an entry by reference, redacted; a null matrix, the identity; a matrix that isn't six numbers, refused, and counted by the verify pass and Remove Hidden Information's scan; and hidden text a matrix entry by reference moves off the page, removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 19 ++++- crates/redact/src/tests.rs | 1 + crates/redact/src/tests/form_matrix.rs | 96 ++++++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 11 ++- crates/redact/src/verify/mod.rs | 1 + 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 crates/redact/src/tests/form_matrix.rs diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..04fafd283d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1071,7 +1071,24 @@ fn xobject( } } Some(b"Form") => { - let m = s.dict.get(b"Matrix").and_then(|m| Matrix::from_operands(doc.resolve(m).as_array()?)).unwrap_or_default(); + // Read as the proof reads it, entries by reference included: read as the identity, a + // form moved onto a mark would be tested against the wrong place and never walked. + let Ok(m) = crate::verify::matrix_of(doc, &s.dict) else { + // Not six numbers: where the form draws, and so whether it reaches a mark, can't be + // told. Apply refuses the page, and the scans count it, as for a form past the + // page's budget below. + if scope.mode == Mode::Apply { + out.error = Some(CosError::Syntax { + offset: 0, + detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + }); + } else if scope.everywhere() { + scope.layer_blocks += 1; + } else { + out.residue += 1; + } + return None; + }; let fctm = m.then(&gs.ctm); let bbox = s.dict.get(b"BBox").and_then(|b| { let v: Vec = doc.resolve(b).as_array()?.iter().filter_map(Object::as_f64).collect(); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..47951343fb 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6143,6 +6143,7 @@ fn unused_descriptions_in_shared_or_inherited_resources_are_retired() { mod descriptions; mod elsewhere; +mod form_matrix; mod hidden_scope; mod identification; mod inline_images; diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs new file mode 100644 index 0000000000..c3e386f47d --- /dev/null +++ b/crates/redact/src/tests/form_matrix.rs @@ -0,0 +1,96 @@ +//! A form's `/Matrix` places what it draws. The redaction walk reads it as the proof does, the +//! array and each entry direct or by reference: read as the identity, a form moved onto a mark +//! was tested against the wrong place, never walked, and its text stayed. + +use super::*; + +/// One page drawing a form with resources of its own, whose content shows SECRET at (20, 60) in a +/// `/BBox` of [0 0 300 100], with `matrix` in its dictionary; object 8 is `extra`. +fn moved_form(matrix: &str, extra: &[u8]) -> Document { + one_page( + b"/Fm Do", + "/XObject << /Fm 7 0 R >>", + vec![ + stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ), + extra.to_vec(), + ], + ) +} + +/// Over the top third of the 300 × 300 page, where a move of 200 puts the text. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +#[test] +fn a_form_moved_onto_a_mark_is_redacted_however_its_matrix_is_given() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 200]", &b"null"[..]), + ("/Matrix 8 0 R", b"[1 0 0 1 0 200]"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"200"), + ("/Matrix [8 0 R 0 0 1 0 200]", b"1"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix} with 8 0 obj {}", String::from_utf8_lossy(extra)); + } +} + +/// A `/Matrix` of null, given directly or by a reference to null, is no `/Matrix` (ISO 32000-2 +/// §7.3.9): the identity, under which the text lies below the mark and stays. +#[test] +fn a_form_whose_matrix_is_null_draws_where_it_stands() { + for (matrix, extra) in [("/Matrix null", &b"null"[..]), ("/Matrix 8 0 R", b"null")] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(0), "{matrix}"); + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[[0.0, 0.0, 300.0, 100.0]], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix}, marked where it stands"); + } +} + +/// A `/Matrix` that isn't six numbers: where the form draws can't be told, so neither can whether +/// it reaches the mark, and the page is refused. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_is_refused() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 (up)]", &b"null"[..]), + ("/Matrix [1 0 0 1 0]", b"null"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"(up)"), + ("/Matrix 8 0 R", b"(up)"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix} with 8 0 obj {}: {result:?}", + String::from_utf8_lossy(extra) + ); + } +} + +/// The scans count such a form as content that may remain: the verify pass under a mark, and +/// Remove Hidden Information's scan, whose removal then refuses the page. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_counts_in_the_scans() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + assert_eq!(under(&mut doc, 0, &[MARK]), 1); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenText], 1); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(matches!(&removed, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), "{removed:?}"); +} + +/// Remove Hidden Information reads it the same way: text a matrix entry by reference moves off the +/// page is hidden text, and goes. +#[test] +fn text_a_matrix_entry_by_reference_moves_off_the_page_is_hidden_text() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 8 0 R]", b"1000"); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenText && *n > 0)), "{removed:?}"); +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index fe92e4e78d..8ef3d406cd 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -396,13 +396,18 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); } -/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error -/// (guessing the identity would put its content in the wrong place). -pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { +/// A form's or pattern's `/Matrix`, the array and each entry direct or by reference: absent or null +/// is the identity, present but malformed is an error (guessing the identity would put its content in the +/// wrong place). The redaction walk reads a form's with it too. +pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { match dict.get(b"Matrix") { None => Ok(Matrix::IDENTITY), Some(m) => { let m = doc.resolve(m); + // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. + if matches!(*m, Object::Null) { + return Ok(Matrix::IDENTITY); + } let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) } diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs index 3a36c9e08e..d58d4c307c 100644 --- a/crates/redact/src/verify/mod.rs +++ b/crates/redact/src/verify/mod.rs @@ -70,6 +70,7 @@ mod snapshot; mod sweep; mod util; +pub(crate) use extract::matrix_of; pub use snapshot::Snapshot; /// What the proof established for one region. #[derive(Clone, Copy, Debug, PartialEq, Eq)] From cf26980471a668b52eaa527a3b4212f16c4c187d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 10/46] M8: a long form /Matrix is read cheaply, an unreadable one leaves layer removal alone Review of the walk's read of a form's `/Matrix`. Removing hidden layers alone never asks where content lands, but a form whose `/Matrix` isn't six numbers was counted in its scan and refused, in a document without layers too. Such a form is now walked as it stands there; Apply Redactions, and Remove Hidden Information with hidden text, which tells hidden text by where it lands, still refuse it. `matrix_of` resolved and copied every entry before checking there were six, and the walk reads it at every `Do`. Measured in review (debug build), 20,000 draws of a form whose `/Matrix` is 20,000 entries long took 9 s by reference (16 ms before the walk read entries) and 12 s written directly (2.6 s before, the array being copied by `resolve`). It now reads an array written directly in place, and checks the length before resolving any entry. Tests: `a_form_whose_matrix_is_not_six_numbers_leaves_removing_hidden_layers_alone`, and `tests/matrix_work.rs` (a binary of its own), whose `a_long_form_matrix_costs_no_more_than_a_short_one` times 4,000 draws of a form with a 4,000-entry `/Matrix` against a 7-entry one, written directly and by reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 34 +++++++++------ crates/redact/src/tests/form_matrix.rs | 11 +++++ crates/redact/src/verify/extract.rs | 26 +++++++----- crates/redact/tests/matrix_work.rs | 59 ++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 crates/redact/tests/matrix_work.rs diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 04fafd283d..f08ab2da8d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1073,21 +1073,27 @@ fn xobject( Some(b"Form") => { // Read as the proof reads it, entries by reference included: read as the identity, a // form moved onto a mark would be tested against the wrong place and never walked. - let Ok(m) = crate::verify::matrix_of(doc, &s.dict) else { - // Not six numbers: where the form draws, and so whether it reaches a mark, can't be - // told. Apply refuses the page, and the scans count it, as for a form past the - // page's budget below. - if scope.mode == Mode::Apply { - out.error = Some(CosError::Syntax { - offset: 0, - detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), - }); - } else if scope.everywhere() { - scope.layer_blocks += 1; - } else { - out.residue += 1; + let m = match crate::verify::matrix_of(doc, &s.dict) { + Ok(m) => m, + // Not six numbers. Removing hidden layers alone never asks where content lands, so + // the form is walked as it stands. + Err(()) if scope.everywhere() && scope.hidden_text.is_none() => Matrix::IDENTITY, + // Otherwise where the form draws, and so whether it reaches a mark or leaves the + // page, can't be told. Apply refuses the page, and the scans count it, as for a + // form past the page's budget below. + Err(()) => { + if scope.mode == Mode::Apply { + out.error = Some(CosError::Syntax { + offset: 0, + detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + }); + } else if scope.everywhere() { + scope.layer_blocks += 1; + } else { + out.residue += 1; + } + return None; } - return None; }; let fctm = m.then(&gs.ctm); let bbox = s.dict.get(b"BBox").and_then(|b| { diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs index c3e386f47d..300c2933a1 100644 --- a/crates/redact/src/tests/form_matrix.rs +++ b/crates/redact/src/tests/form_matrix.rs @@ -85,6 +85,17 @@ fn a_form_whose_matrix_is_not_six_numbers_counts_in_the_scans() { assert!(matches!(&removed, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), "{removed:?}"); } +/// Removing hidden layers alone never asks where content lands, so such a form is walked as it +/// stands: nothing is counted or refused over it in a document without layers. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_leaves_removing_hidden_layers_alone() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenLayers], 0); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::NothingToApply)); +} + /// Remove Hidden Information reads it the same way: text a matrix entry by reference moves off the /// page is hidden text, and goes. #[test] diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index 8ef3d406cd..bc46348715 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -400,18 +400,24 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { /// is the identity, present but malformed is an error (guessing the identity would put its content in the /// wrong place). The redaction walk reads a form's with it too. pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { - match dict.get(b"Matrix") { - None => Ok(Matrix::IDENTITY), - Some(m) => { - let m = doc.resolve(m); - // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. - if matches!(*m, Object::Null) { - return Ok(Matrix::IDENTITY); - } - let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); - Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + let Some(m) = dict.get(b"Matrix") else { return Ok(Matrix::IDENTITY) }; + // Read in place when written directly, and the length checked before any entry is resolved: + // the walk reads it at every `Do`, and an array of any length would be copied each time. + let held; + let m = match m { + Object::Ref(_) => { + held = doc.resolve(m); + &*held } + direct => direct, + }; + // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. + if matches!(m, Object::Null) { + return Ok(Matrix::IDENTITY); } + let items = m.as_array().filter(|a| a.len() == 6).ok_or(())?; + let items: Vec = items.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) } /// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. diff --git a/crates/redact/tests/matrix_work.rs b/crates/redact/tests/matrix_work.rs new file mode 100644 index 0000000000..d96447ff71 --- /dev/null +++ b/crates/redact/tests/matrix_work.rs @@ -0,0 +1,59 @@ +//! Work regression for a form's `/Matrix`, which the redaction walk reads at every `Do` of the +//! form: an array of any length costs no more than one of seven entries, written directly or by +//! reference. A test binary of its own, so that none of the crate's other tests shares its process. +//! The test compares two timings of its own, the least of three runs each, not a fixed time. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page drawing a form `draws` times under a mark, the form's `/Matrix` an array of `entries` +/// zeros, written in its dictionary or, `by_reference`, as object 6. +fn document(draws: usize, entries: usize, by_reference: bool) -> Document { + let zeros = format!("[{}]", vec!["0"; entries].join(" ")); + let matrix = if by_reference { "6 0 R".to_string() } else { zeros.clone() }; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Fm 5 0 R >> >> >>".to_vec(), + stream_obj("", "/Fm Do ".repeat(draws).as_bytes()), + stream_obj(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix {matrix}"), b"0 0 10 10 re f"), + zeros.into_bytes(), + ]; + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 0.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// How long redacting `document(draws, entries, by_reference)` takes, the least of three runs. A +/// matrix that isn't six numbers is refused. +fn redaction(draws: usize, entries: usize, by_reference: bool) -> Duration { + (0..3) + .map(|_| { + let mut doc = document(draws, entries, by_reference); + let start = Instant::now(); + let refused = apply(&mut doc, None).is_err(); + let took = start.elapsed(); + assert!(refused, "a /Matrix of {entries} entries is refused"); + took + }) + .min() + .unwrap_or_default() +} + +/// Written when the walk resolved and copied every entry before checking there were six: 4,000 +/// draws of a form whose `/Matrix` is 4,000 entries long read sixteen million entries. +#[test] +fn a_long_form_matrix_costs_no_more_than_a_short_one() { + for by_reference in [false, true] { + let (short, long) = (redaction(4_000, 7, by_reference), redaction(4_000, 4_000, by_reference)); + assert!(long < short * 3 + Duration::from_millis(50), "by reference: {by_reference}: long {long:?}, short {short:?}"); + } +} From 4f107a6f4065e0d2fabbf4ea8318ec60f2f20dee Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 11/46] M8: a form /Matrix through a reference to a reference is refused Review of the walk's read of a form's `/Matrix`. Through `Document::resolve` it followed a reference to a reference, for the matrix and for each entry, as the app's renderer does; Poppler doesn't, and reads such a matrix as the identity and such an entry as 0. With `/Matrix [1 0 0 1 0 8 0 R]`, `8 0 obj 9 0 R` and `9 0 obj 200`, a mark where Poppler shows the text applied with success and left it in the file, where `main`, reading any entry by reference as the identity, removed it. `matrix_of` now follows each reference one hop, and a reference to a reference fails as a malformed matrix: where the content draws can't be told, so Apply refuses the page, as for a matrix that isn't six numbers. The refusal now says "isn't six numbers given directly or through one reference", and `matrix_of`'s doc names the appearances it reads, not patterns. The proof, which reads the matrix of forms and appearances with it, treats such a matrix as malformed too. Test: `a_form_whose_matrix_goes_through_a_reference_to_a_reference_is_refused` (an entry and the whole matrix, with a mark where either reader draws the text). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 6 ++++-- crates/redact/src/tests/form_matrix.rs | 24 ++++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 26 ++++++++++++++++++-------- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index f08ab2da8d..9cfc981381 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1075,7 +1075,7 @@ fn xobject( // form moved onto a mark would be tested against the wrong place and never walked. let m = match crate::verify::matrix_of(doc, &s.dict) { Ok(m) => m, - // Not six numbers. Removing hidden layers alone never asks where content lands, so + // Not six numbers given directly or through one reference. Removing hidden layers alone never asks where content lands, so // the form is walked as it stands. Err(()) if scope.everywhere() && scope.hidden_text.is_none() => Matrix::IDENTITY, // Otherwise where the form draws, and so whether it reaches a mark or leaves the @@ -1085,7 +1085,9 @@ fn xobject( if scope.mode == Mode::Apply { out.error = Some(CosError::Syntax { offset: 0, - detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + detail: + "a form XObject's /Matrix isn't six numbers given directly or through one reference, so where it draws can't be told" + .into(), }); } else if scope.everywhere() { scope.layer_blocks += 1; diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs index 300c2933a1..e4bb33608e 100644 --- a/crates/redact/src/tests/form_matrix.rs +++ b/crates/redact/src/tests/form_matrix.rs @@ -72,6 +72,30 @@ fn a_form_whose_matrix_is_not_six_numbers_is_refused() { } } +/// A reference to a reference, for the matrix or one of its entries: Poppler doesn't follow it (it +/// reads such a matrix as the identity, such an entry as 0), the app's renderer follows it and +/// reads the move, so where the form draws can't be told, and the page is refused, as for a matrix that isn't six numbers. Read as either, +/// a mark where the other reader shows the text would leave it in the file. +#[test] +fn a_form_whose_matrix_goes_through_a_reference_to_a_reference_is_refused() { + for (matrix, target) in [("/Matrix [1 0 0 1 0 8 0 R]", &b"200"[..]), ("/Matrix 8 0 R", b"[1 0 0 1 0 200]")] { + for mark_at in [MARK, [0.0, 0.0, 300.0, 100.0]] { + let form = stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form, b"9 0 R".to_vec(), target.to_vec()]); + mark(&mut doc, 0, &[mark_at], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix}, 8 0 obj 9 0 R, 9 0 obj {}, mark at {mark_at:?}: {result:?}", + String::from_utf8_lossy(target) + ); + } + } +} + /// The scans count such a form as content that may remain: the verify pass under a mark, and /// Remove Hidden Information's scan, whose removal then refuses the page. #[test] diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index bc46348715..63dbeff342 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -396,17 +396,20 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); } -/// A form's or pattern's `/Matrix`, the array and each entry direct or by reference: absent or null +/// A form's or an annotation appearance's `/Matrix`, the array and each entry direct or by reference: absent or null /// is the identity, present but malformed is an error (guessing the identity would put its content in the -/// wrong place). The redaction walk reads a form's with it too. +/// wrong place). A reference to a reference is malformed here: readers disagree on following it +/// (Poppler doesn't, the app's renderer does), so where the content draws can't be told. The +/// redaction walk reads a form's with it too. pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { let Some(m) = dict.get(b"Matrix") else { return Ok(Matrix::IDENTITY) }; - // Read in place when written directly, and the length checked before any entry is resolved: - // the walk reads it at every `Do`, and an array of any length would be copied each time. + // Each reference is followed one hop. The array is read in place when written directly, and + // its length checked before any entry is read: the walk reads it at every `Do`, and an array of + // any length would be copied each time. let held; let m = match m { - Object::Ref(_) => { - held = doc.resolve(m); + Object::Ref(r) => { + held = doc.get(*r); &*held } direct => direct, @@ -416,8 +419,15 @@ pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { return Ok(Matrix::IDENTITY); } let items = m.as_array().filter(|a| a.len() == 6).ok_or(())?; - let items: Vec = items.iter().map(|x| (*doc.resolve(x)).clone()).collect(); - Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + let mut values = [0.0; 6]; + for (value, item) in values.iter_mut().zip(items) { + let n = match item { + Object::Ref(r) => doc.get(*r).as_f64(), + direct => direct.as_f64(), + }; + *value = n.filter(|n| n.is_finite()).ok_or(())?; + } + Ok(Matrix(values)) } /// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. From b16f0c0a7729019a73f5c128da329b384300df8d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 17:25:07 +0100 Subject: [PATCH 12/46] M8: redaction reads an operator's operands from the end, as the renderer does An operator takes its operands from the end, and the renderer ignores a surplus before them: `(x) /OC /MC0 BDC` marks a layer's block, and `(x) /GS1 gs` selects `/GS1`. Redaction read some of them from the front, so a surplus stood in for them and what they named went unseen. Found in review of #6882, where Remove Hidden Information left the text of a hidden layer in `(x) /OC /MC0 BDC (LAYERSECRET) Tj EMC`, which both the app's renderer and Poppler hide. In what Apply Redactions and Remove Hidden Information decide: - the layers walk read a block's tag and property list from the front, and kept a hidden layer's block; - the cleanup of `/ActualText`, `/Alt`, `/E` and `/T` in a block whose paint was removed read (and wrote) the property list from the front, and kept the name that still read the removed word; - the tags cleanup read a block's `MCID` from the front, and left the removed text in its structure element's `/ActualText` and `/Alt`; and a `TJ`'s array, so a block that still drew was taken for empty and its structure element lost the content the page still shows; - the property cleanups read the name of a `BDC` or `DP` property list from the front, and retired a property list the page still draws with; - the walk read the ExtGState a `gs` selects from the front, so a font set only that way was not set, and `cs`/`CS` the same for a pattern space; - the Type 3 cleanup's reads of `cs`, `gs`, `Tf`, `BDC` and `DP`, which decide which resources and fonts are still used. The redaction proof read `Tc`, `Tw`, `Tz`, `TL`, `Ts`, `Tf`, `"`, `TJ` and `Do` from the front too. `pdfcraft_content::Op` gains `back`, `name_back` and `num_back`: operand `k` counted from the end. Tests (`crates/redact/src/tests/operands_from_the_end.rs`): a hidden layer's block, `/ActualText` on removed paint, a tag's `MCID`, a tagged `TJ` that still draws, a property list still drawn with, a font from an ExtGState, the pattern a `/Pattern cs` keeps, and a Type 3 font with the ExtGState, colour space, property list and font its glyph uses, each after a surplus operand. In `tests_verify.rs`, `the_proof_reads_operands_from_the_end` (`Tf`, `TJ`, and `Do` of a form and of images), `the_proof_reads_the_text_states_operands_from_the_end` (`Tc`, `Tw`, `Tz`, `TL`, `Ts` and `"`) and `an_image_a_type3_glyph_draws_after_a_surplus_operand_is_found`. And `operands_are_counted_from_the_end` in the content crate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/content/src/lib.rs | 17 ++ crates/content/src/tests.rs | 11 ++ crates/redact/src/interp.rs | 11 +- crates/redact/src/interp/layers.rs | 6 +- crates/redact/src/interp/prune.rs | 4 +- crates/redact/src/tags.rs | 7 +- crates/redact/src/tests.rs | 1 + .../redact/src/tests/operands_from_the_end.rs | 170 ++++++++++++++++++ crates/redact/src/tests_verify.rs | 72 ++++++++ crates/redact/src/type3.rs | 20 +-- crates/redact/src/verify/extract.rs | 23 +-- crates/redact/src/verify/geometry.rs | 4 +- 12 files changed, 310 insertions(+), 36 deletions(-) create mode 100644 crates/redact/src/tests/operands_from_the_end.rs diff --git a/crates/content/src/lib.rs b/crates/content/src/lib.rs index fb17a903d1..f77c54e1dc 100644 --- a/crates/content/src/lib.rs +++ b/crates/content/src/lib.rs @@ -58,6 +58,23 @@ impl Op { pub fn last_name(&self) -> Option<&[u8]> { self.operands.last().and_then(Object::as_name) } + + /// Operand `k` counted from the end (0 is the last), as the renderer reads an operator's + /// operands: `(x) /OC /MC0 BDC` marks with `/OC /MC0`, and `(x) /F1 10 Tf` selects `/F1`. + pub fn back(&self, k: usize) -> Option<&Object> { + let i = self.operands.len().checked_sub(k)?.checked_sub(1)?; + self.operands.get(i) + } + + /// [`Op::back`] as a name. + pub fn name_back(&self, k: usize) -> Option<&[u8]> { + self.back(k).and_then(Object::as_name) + } + + /// [`Op::back`] as a number. + pub fn num_back(&self, k: usize) -> Option { + self.back(k).and_then(Object::as_f64) + } } /// A parsed stream. diff --git a/crates/content/src/tests.rs b/crates/content/src/tests.rs index 9ca12c4536..89f258853d 100644 --- a/crates/content/src/tests.rs +++ b/crates/content/src/tests.rs @@ -10,6 +10,17 @@ thread_local! { pub(crate) static PIECE_VISITS: std::cell::Cell = const { std::cell::Cell::new(0) }; } +#[test] +fn operands_are_counted_from_the_end() { + let op = parse(b"(x) /OC /MC0 BDC").ops.remove(0); + assert_eq!((op.name_back(1), op.name_back(0)), (Some(&b"OC"[..]), Some(&b"MC0"[..]))); + assert_eq!(op.back(2).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"x".to_vec())); + assert_eq!((op.back(3), op.back(usize::MAX)), (None, None)); + let op = parse(b"/F1 10 Tf").ops.remove(0); + assert_eq!((op.name_back(1), op.num_back(0), op.num_back(1)), (Some(&b"F1"[..]), Some(10.0), None)); + assert_eq!(Op::new("Q", vec![]).back(0), None); +} + #[test] fn splicing_visits_each_piece_a_few_times() { let n = 5_000; diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..b470d7984e 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -571,7 +571,7 @@ impl Walk { Some(op) } b"cs" | b"CS" => { - if op.name(0) != Some(b"Pattern") { + if op.last_name() != Some(b"Pattern") { if o == b"cs" { self.gs.fill_pattern = None; } else { @@ -656,10 +656,10 @@ impl Walk { } b"Tf" => self.select_font(doc, scope, op), _ => { - if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { + if let Some((font, vertical, size)) = op.last_name().and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); // As for `Tf`: a Type 3 font without /Resources paints through this scope's. - self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); + self.gs.type3_inherits = op.last_name().is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } } } @@ -864,7 +864,8 @@ fn clean_replacements( if !op.is("BDC") || !affected.contains(&op.span.start) { continue; } - let property = match op.operands.get(1) { + // The property list is the last operand, as the renderer reads it. + let property = match op.back(0) { Some(Object::Name(name)) => properties.get(name).map(|p| doc.resolve(p)), Some(p) => Some(doc.resolve(p)), None => None, @@ -874,7 +875,7 @@ fn clean_replacements( for key in [&b"ActualText"[..], b"Alt", b"E", b"T"] { stripped |= dict.remove(key).is_some(); } - if stripped && let Some(property) = op.operands.get_mut(1) { + if stripped && let Some(property) = op.operands.last_mut() { *property = Object::Dict(dict); *changed = true; report.replacement_text += 1; diff --git a/crates/redact/src/interp/layers.rs b/crates/redact/src/interp/layers.rs index 88d1c4e677..362cb36620 100644 --- a/crates/redact/src/interp/layers.rs +++ b/crates/redact/src/interp/layers.rs @@ -225,8 +225,10 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve match op.op.as_slice() { b"BDC" | b"BMC" => { let this = op.is("BDC") - && op.name(0) == Some(b"OC") - && match op.operands.get(1) { + // The tag and the property list are the last two operands, as the renderer + // reads them: `(x) /OC /MC0 BDC` is a layer's block too. + && op.name_back(1) == Some(b"OC") + && match op.back(0) { Some(Object::Name(n)) => props.get(n).is_some_and(|o| scope.layer_hidden(doc, o)), Some(o) => scope.layer_hidden(doc, o), None => false, diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 39089601ec..6b0be5d977 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -357,7 +357,7 @@ fn prune_retired_properties(doc: &Document, res: &mut Dict, names: &[Vec], c // Each is one page's content or one form's, with the allowance of a page. for op in resource_ops(doc, &mut DecodeBudget::page(), res, content)? { if matches!(op.op.as_slice(), b"BDC" | b"DP") - && let Some(name) = op.name(1) + && let Some(name) = op.last_name() { used.insert(name.to_vec()); } @@ -520,7 +520,7 @@ pub(crate) fn prune_properties(doc: &Document, budget: &mut DecodeBudget, res: & let ops = resource_ops(doc, budget, res, content)?; let names: Vec> = properties.iter().map(|(n, _)| n.to_vec()).collect(); for name in names { - if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.name(1) == Some(&name)) + if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.last_name() == Some(&name)) && let Some(value) = properties.remove(&name) { retired.push(value); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 648c0c29e5..9cccb93665 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -52,7 +52,8 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { - let id = match op.operands.get(1) { + // The property list is the last operand, as the renderer reads it. + let id = match op.back(0) { Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { let props = doc.resolve(p); let prop = doc.resolve(props.as_dict()?.get(n)?); @@ -86,9 +87,7 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { } let draws = match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().and_then(Object::as_string).is_some_and(|s| !s.bytes.is_empty()), - b"TJ" => { - op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))) - } + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))), b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s" | b"sh" | b"Do" | b"BI" => true, _ => false, }; diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..015a574d4a 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6149,6 +6149,7 @@ mod inline_images; mod limits; mod metadata_fail_closed; mod metadata_matching; +mod operands_from_the_end; mod property_retirement; mod surviving_references; mod text_state; diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs new file mode 100644 index 0000000000..d2d6640121 --- /dev/null +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -0,0 +1,170 @@ +//! An operator takes its operands from the end, as the renderer reads them, and a surplus before +//! them is ignored: `(x) /OC /MC0 BDC` marks a layer's block, `(x) /GS1 gs` selects `/GS1`. Read +//! from the front, the surplus stood in for the operands, and what they named went unseen. + +use super::*; + +/// Hidden-layer text in a block whose tag and property list follow a string: Remove Hidden +/// Information found nothing to remove, and the text stayed. +#[test] +fn a_hidden_layers_block_after_a_surplus_operand_is_removed() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = one_page( + b"(x) /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET", + "/Properties << /MC0 7 0 R >>", + vec![b"<< /Type /OCG /Name (Hidden) >>".to_vec()], + ); + doc.update_dict(ObjRef::new(1, 0), |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([ + (b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + (b"D".to_vec(), Object::Dict(Dict::from_iter([(b"OFF".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))]))]))), + ])), + ); + }) + .unwrap(); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + let c = content(&doc, 0); + assert!(!c.contains("LAYERSECRET") && c.contains("PUBLIC"), "{c}"); +} + +/// Paint under a mark named by `/ActualText` in a block after a surplus operand: the name, which +/// would still read the word, kept it. +#[test] +fn actual_text_after_a_surplus_operand_goes_with_the_paint_it_names() { + let mut doc = one_page(b"(x) /Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h f EMC", "", vec![]); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("SECRET"), "{c}"); +} + +/// A tagged paragraph whose `MCID` follows a surplus operand: its structure element kept the +/// removed text in `/ActualText` and `/Alt`. +#[test] +fn a_tag_after_a_surplus_operand_loses_what_redaction_removed() { + let content = b"(x) /P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the secret) /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); +} + +/// A property list still drawn with, named after a surplus operand, stays in the page's resources +/// when the redaction cleans them; read from the front, it was taken for unused and retired. +#[test] +fn a_property_list_named_after_a_surplus_operand_is_kept() { + let mut doc = one_page( + b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET (x) /Span /Pr1 BDC BT /F1 10 Tf 20 20 Td (KEPT) Tj ET EMC", + "/Properties << /Pr1 << /Lang (en-GB) >> >>", + vec![], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let props = res.get(b"Properties").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(props.contains(b"Pr1"), "{res:?}"); +} + +/// A font set only by an ExtGState selected after a surplus operand: read from the front, no font +/// was set, every glyph stood at the start of the line with no size, and a mark over the end of +/// the word removed nothing. +#[test] +fn a_font_from_an_extgstate_selected_after_a_surplus_operand_places_the_text() { + let mut doc = one_page(b"BT (x) /GS1 gs 20 260 Td (SECRET) Tj ET", "/ExtGState << /GS1 << /Font [5 0 R 10] >> >>", vec![]); + // Over the last two glyphs, E and T at x = 40 to 50. + mark(&mut doc, 0, &[[42.0, 255.0, 55.0, 275.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 2); +} + +/// `/Pattern cs` after a surplus operand, once the pattern's paint under the mark went: it selects +/// the Pattern space and keeps the pattern, so the fill after it still counts as painting with it +/// and the pattern stays. Read from the front, the surplus cleared the pattern, which was pruned. +/// A grey space clears it. +#[test] +fn a_pattern_space_selected_after_a_surplus_operand_keeps_the_pattern() { + for (space, pruned) in [("(x) /Pattern", 0), ("/Pattern", 0), ("(x) /DeviceGray", 1)] { + let mut doc = one_page( + format!("/Pattern cs /P0 scn 10 260 20 20 re f {space} cs 200 20 50 50 re f").as_bytes(), + "/Pattern << /P0 7 0 R >>", + vec![stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f")], + ); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, pruned), "{space}"); + } +} + +/// A Type 3 font selected after a surplus operand, outside the mark, whose glyph selects an +/// ExtGState, a colour space, a property list and a font after one too: the cleanups after the +/// walk took the font for undrawn and those for unused. All stay, with the glyph. +#[test] +fn a_type3_font_and_its_glyphs_resources_named_after_a_surplus_operand_are_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /T3 12 Tf 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] /Resources << /ExtGState << /GS0 << /CA 0.5 >> >> \ + /ColorSpace << /CS0 /DeviceGray >> /Properties << /P0 << >> >> /Font << /F0 5 0 R >> >> >>" + .to_vec(), + stream("", b"10 0 d0 (x) /GS0 gs (x) /CS0 cs (x) /Span /P0 BDC 0 0 10 10 re f EMC BT (x) /F0 1 Tf ET"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let fonts = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); + let font = fonts.get(b"T3").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()); + let font = font.unwrap_or_else(|| panic!("the drawn Type 3 font stays: {fonts:?}")); + let procs = font.get(b"CharProcs").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(procs.contains(b"a"), "its glyph stays: {font:?}"); + let font_res = font.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let states = font_res.get(b"ExtGState").map(|s| doc.resolve(s)).and_then(|s| s.as_dict().cloned()).unwrap_or_default(); + assert!(states.contains(b"GS0"), "the ExtGState its glyph selects stays: {font_res:?}"); + for (kind, name) in [(&b"ColorSpace"[..], &b"CS0"[..]), (b"Properties", b"P0"), (b"Font", b"F0")] { + let held = font_res.get(kind).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default(); + assert!(held.contains(name), "the {} its glyph uses stays: {font_res:?}", String::from_utf8_lossy(name)); + } +} + +/// A tagged paragraph whose `TJ` after a surplus operand still draws once the mark has removed the +/// word beside it: read from the front, that `TJ` drew nothing, the paragraph was taken for empty, +/// and its structure element lost the content the page still shows. +#[test] +fn a_tagged_tj_after_a_surplus_operand_still_draws() { + let content = b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj 10 -100 Td (x) [(KEPT)] TJ ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let paragraph = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert_eq!(paragraph.get(b"K").and_then(Object::as_int), Some(0), "it still tags what the page shows: {paragraph:?}"); +} diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index 136cb2b207..3937fcda4f 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -897,6 +897,78 @@ fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { assert!(proof.entries[0].detail.contains("glyph(s) still show")); } +/// The proof reads an operator's operands from the end, as the renderer does: a surplus before +/// them doesn't hide the text a `Tf`, `TJ` or `Do` after it draws, or the image a `Do` draws. +#[test] +fn the_proof_reads_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", + ); + let image = stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"); + let image_form = + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 8 0 R >> >>", b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q"); + for (content, spec) in [ + (&b"BT (x) /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"[..], Spec::default()), + (b"BT /F1 10 Tf 10 100 Td (x) [(SECRETVALUE)] TJ ET", Spec::default()), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form.clone()], ..Spec::default() }), + // An image under the mark, and one in a form under it. + (b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q", Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image.clone()], ..Spec::default() }), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![image_form.clone(), image.clone()], ..Spec::default() }), + ] { + let mut doc = marked_content(content, spec); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// The proof reads the text state's operands from the end too. In each case the text starts outside +/// the region and only the operand after the surplus brings a glyph into it: char spacing, word +/// spacing, horizontal scaling, leading with `T*`, rise, and the spacing `"` sets. +#[test] +fn the_proof_reads_the_text_states_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"BT /F1 10 Tf -60 100 Td (x) 70 Tc (AB) Tj ET"[..], + b"BT /F1 10 Tf -60 100 Td (x) 70 Tw (A B) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 1500 Tz (AB) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) 100 TL T* (A) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) -100 Ts (A) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 0 70 (AB) \" ET", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws +/// after a surplus operand can't be placed, as one it draws without one can't: the region is +/// unverifiable. +#[test] +fn an_image_a_type3_glyph_draws_after_a_surplus_operand_is_found() { + let spec = Spec { + resources: "/Font << /F1 5 0 R /T3 7 0 R >> /XObject << /Im0 9 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 (x) /Im0 Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), + ], + ..Spec::default() + }; + let mut doc = open(build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET BT /T3 12 Tf 150 20 Td (a) Tj ET", spec)); + mark(&mut doc, &[REGION]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); +} + #[test] fn an_ambiguous_needle_is_still_swept_outside_page_content() { // The word also stays on the page legitimately (so page text, the raw file and decoded diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 4c3f2211e8..0d0ee00559 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -151,7 +151,7 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"cs" | b"CS" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.colorspaces.insert(n.to_vec()); } } @@ -166,17 +166,17 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"gs" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.extgstates.insert(n.to_vec()); } } b"Tf" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.name_back(1) { used.fonts.insert(n.to_vec()); } } b"BDC" | b"DP" => { - if let Some(n) = op.name(1) { + if let Some(n) = op.last_name() { used.properties.insert(n.to_vec()); } } @@ -315,7 +315,7 @@ fn scan_content_stream( } b"gs" => { apply_text_op(ctx, res, &op, &mut current_font); - if let Some(name) = op.name(0) { + if let Some(name) = op.last_name() { scan_soft_mask(ctx, res, holder_ref, name, depth)?; } } @@ -338,12 +338,12 @@ fn scan_content_stream( fn apply_text_op(ctx: &mut ScanContext<'_>, res: &Dict, op: &pdfcraft_content::Op, current_font: &mut Option) { match op.op.as_slice() { b"Tf" => { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { *current_font = resolve_font_in_res(ctx.doc, res, name).map(|(k, _)| k); } } b"gs" => { - if let Some(name) = op.name(0) + if let Some(name) = op.last_name() && let Some((k, _)) = resolve_extgstate_font(ctx.doc, res, name) { *current_font = Some(k); @@ -477,7 +477,7 @@ fn add_soft_mask_ops(doc: &Document, res: &Dict, ops: &mut Vec op - .name(0) + .last_name() .and_then(|n| states.get(n)) .and_then(|s| doc.resolve(s).as_dict().and_then(|s| s.get(b"SMask").cloned())) .and_then(|m| doc.resolve(&m).as_dict().and_then(|m| m.get(b"G").and_then(Object::as_ref))), @@ -584,11 +584,11 @@ pub(crate) fn prune_type3_from_resources( let mut drawn_font_names = HashSet::new(); for op in &ops { if op.is("Tf") { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { drawn_font_names.insert(name.to_vec()); } } else if op.is("gs") - && let Some(name) = op.name(0) + && let Some(name) = op.last_name() { let extg = res_dict(doc, res, b"ExtGState"); if let Some(font_obj) = extg.get(name).and_then(|state_obj| gs_font_object(doc, state_obj)) { diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index fe92e4e78d..294fd92dda 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -16,7 +16,7 @@ pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), - b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), _ => false, } } @@ -552,7 +552,7 @@ impl Extractor<'_> { let o = op.op.as_slice(); // Strings and TJ adjustments, in order. let items: Vec<&Object> = match o { - b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + b"TJ" => op.back(0).and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), _ => op.operands.last().into_iter().collect(), }; let mut any_text = false; @@ -637,14 +637,15 @@ impl Extractor<'_> { tm = Matrix::IDENTITY; tlm = Matrix::IDENTITY; } - b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), - b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), - b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, - b"TL" => gs.tl = op.num(0).unwrap_or(0.0), - b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + // Operands are read from the end, as the renderer reads them. + b"Tc" => gs.tc = op.num_back(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num_back(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num_back(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num_back(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num_back(0).unwrap_or(0.0), b"Tf" => { - gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); - gs.size = op.num(1).unwrap_or(0.0); + gs.font = op.name_back(1).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num_back(0).unwrap_or(0.0); } b"Td" | b"TD" => { if let Some([x, y]) = op.nums::<2>() { @@ -671,7 +672,7 @@ impl Extractor<'_> { tm = tlm; } if o == b"\"" - && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + && let (Some(aw), Some(ac)) = (op.num_back(2), op.num_back(1)) { (gs.tw, gs.tc) = (aw, ac); } @@ -686,7 +687,7 @@ impl Extractor<'_> { self.show_text(font, &op, &gs, &mut tm, placed); } b"Do" => { - let Some(name) = op.name(0) else { continue }; + let Some(name) = op.last_name() else { continue }; let Some(entry) = xobjects.get(name) else { continue }; if let Object::Stream(s) = &*doc.resolve(entry) && s.dict.name(b"Subtype") == Some(b"Form") diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index 7ea0de5699..0d3ef179fc 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -188,7 +188,7 @@ impl Geo<'_> { } } b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let obj = self.doc.resolve(entry); let Object::Stream(s) = &*obj else { continue }; match s.dict.name(b"Subtype") { @@ -344,7 +344,7 @@ impl Geo<'_> { match op.op.as_slice() { b"BI" => return Ok(true), b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; match x.dict.name(b"Subtype") { Some(b"Image") => return Ok(true), From 7c78222612d18d065b54a9319b65f248d3df273a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 18:54:42 +0100 Subject: [PATCH 13/46] M8: path points and an MCID by reference are read as the renderer does Review of the operand reads. Path construction was still read from the front, in the walk (`construct_path`) and in the proof (`extend_path`): after a surplus number, `9 100 100 m 9 140 100 l ...` placed the square at x = 9, so a mark over it removed nothing, and an even surplus placed a square beside a mark under it, which was removed. `m` and `l` now take their last two numbers, `c` six, `v` and `y` four, and the proof's `re` four, as the walk's already did. The tags cleanup read an `MCID` with `Dict::int`, so one given by reference in a named property list, or in a structure element's `/MCR`, was missed and the element kept the removed text in `/ActualText` and `/Alt`. It is read through the reference now. Tests: `a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them` (lines, and curves by `c`, `v` and `y`), `an_mcid_given_by_reference_loses_what_redaction_removed` (and an `/MCR` of a form's stream that stays), and `the_proof_reads_a_paths_points_from_the_end` (lines, `re`, and each of `c`, `v` and `y` from a point outside the region). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 8 ++- crates/redact/src/tags.rs | 5 +- .../redact/src/tests/operands_from_the_end.rs | 68 +++++++++++++++++++ crates/redact/src/tests_verify.rs | 21 ++++++ crates/redact/src/verify/geometry.rs | 17 +++-- 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b470d7984e..b567735d2d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -598,8 +598,12 @@ impl Walk { let o = op.op.as_slice(); let pts: Vec<(f64, f64)> = match o { b"re" => op.nums::<4>().map(|[x, y, w, h]| vec![(x, y), (x + w, y), (x, y + h), (x + w, y + h)]).unwrap_or_default(), - b"h" | b"W" | b"W*" => Vec::new(), - _ => op.operands.iter().filter_map(Object::as_f64).collect::>().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).collect(), + // Each takes its points from the end, as the renderer reads them: a surplus before + // them is ignored, not read as a point. + b"m" | b"l" => op.nums::<2>().map(|[x, y]| vec![(x, y)]).unwrap_or_default(), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![(x1, y1), (x2, y2), (x3, y3)]).unwrap_or_default(), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![(x1, y1), (x2, y2)]).unwrap_or_default(), + _ => Vec::new(), }; for (x, y) in pts { let (u, v) = self.gs.ctm.apply(x, y); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 9cccb93665..15096433d5 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -57,7 +57,8 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { let props = doc.resolve(p); let prop = doc.resolve(props.as_dict()?.get(n)?); - prop.as_dict()?.int(b"MCID") + // A named property list may give its MCID by reference. + prop.as_dict()?.get(b"MCID").and_then(|id| doc.resolve(id).as_int()) }), Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, @@ -138,7 +139,7 @@ fn mcid_of(doc: &Document, o: &Object, page: Option, target: ObjRef, str Object::Dict(m) if m.name(b"Type") == Some(b"MCR") => { let pg = m.get(b"Pg").and_then(Object::as_ref).or(page); let stm = m.get(b"Stm").and_then(Object::as_ref); - (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.int(b"MCID")).flatten() + (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.get(b"MCID").and_then(|id| doc.resolve(id).as_int())).flatten() } _ => None, } diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs index d2d6640121..c18daa4dd8 100644 --- a/crates/redact/src/tests/operands_from_the_end.rs +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -168,3 +168,71 @@ fn a_tagged_tj_after_a_surplus_operand_still_draws() { let paragraph = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); assert_eq!(paragraph.get(b"K").and_then(Object::as_int), Some(0), "it still tags what the page shows: {paragraph:?}"); } + +/// A path's points follow a surplus number: read from the front, each point was shifted by one, +/// so a square under the mark was placed beside it and kept, and one beside the mark was placed +/// under it and removed. +#[test] +fn a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them() { + let under = b"0 0 1 rg 9 100 100 m 9 140 100 l 9 140 140 l 9 100 140 l h f 9 100 100 m 9 140 140 l S".as_slice(); + let mut doc = one_page(under, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 2, "both paths under the mark go"); + let beside = + b"0 0 1 rg 0 200 200 m 250 200 l 250 250 l 200 250 l h f 0 200 200 m 210 210 220 220 230 230 c 240 240 250 250 v 200 250 200 260 y h f" + .as_slice(); + let mut doc = one_page(beside, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "the paths beside the mark stay"); + // Curves whose points, after a surplus, reach under the mark. + for curve in [&b"200 200 m 9 10 10 12 12 15 15 c f"[..], b"200 200 m 9 10 10 15 15 v f", b"200 200 m 9 10 10 15 15 y f"] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1, "{}", String::from_utf8_lossy(curve)); + } +} + +/// An `MCID` given by reference: in a named property list, and in a structure element's `/MCR`. +/// Read directly, it was missed, and the element kept the removed text. +#[test] +fn an_mcid_given_by_reference_loses_what_redaction_removed() { + for (content, properties, kid) in [ + (b"/P /MC0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC".as_slice(), "/Properties << /MC0 << /MCID 10 0 R >> >>", "0"), + (b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", "", "<< /Type /MCR /Pg 3 0 R /MCID 10 0 R >>"), + ] { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> {properties} >> >>").into_bytes(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + format!("<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the SECRET) /K {kid} >>").into_bytes(), + b"0".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1, "{properties}{kid}"); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); + } + // An `/MCR` naming the same MCID in a form's stream (`/Stm`) is other content: it stays. + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (FORMTEXT) /K << /Type /MCR /Pg 3 0 R /Stm 11 0 R /MCID 10 0 R >> >>".to_vec(), + b"0".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"/P <> BDC EMC"), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, None).unwrap(); + let other = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(other.get(b"ActualText").is_some(), "{other:?}"); +} diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index 3937fcda4f..bd9cb909de 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -947,6 +947,27 @@ fn the_proof_reads_the_text_states_operands_from_the_end() { } } +/// The proof reads a path's points from the end too: a square under the region, its points after +/// a surplus number, still paints there. +#[test] +fn the_proof_reads_a_paths_points_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"0 0 1 rg 9 20 98 m 9 60 98 l 9 60 108 l 9 20 108 l h f"[..], + b"0 0 1 rg 9 20 98 40 10 re f", + // Curves from a point outside the region whose own points reach into it. + b"0 0 1 rg 9 200 200 m 9 20 98 40 100 60 108 c h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 v h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 y h f", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + /// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws /// after a surplus operand can't be placed, as one it draws without one can't: the region is /// unverifiable. diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index 0d3ef179fc..7b6833edef 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -372,21 +372,28 @@ impl Geo<'_> { /// carries) by one construction operator. fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { let o = op.op.as_slice(); - let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); let at = |x: f64, y: f64| ctm.apply(x, y); + // Each takes its numbers from the end, as the renderer reads them: a surplus before them is + // ignored, not read as a point. + let points: Option> = match o { + b"l" => op.nums::<2>().map(|[x, y]| vec![at(x, y)]), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![at(x1, y1), at(x2, y2), at(x3, y3)]), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![at(x1, y1), at(x2, y2)]), + _ => None, + }; match o { b"m" => { - if let [x, y] = nums[..] { + if let Some([x, y]) = op.nums::<2>() { sub.push(vec![at(x, y)]); } } b"l" | b"c" | b"v" | b"y" => { - if let Some(cur) = sub.last_mut() { - cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + if let (Some(cur), Some(points)) = (sub.last_mut(), points) { + cur.extend(points); } } b"re" => { - if let [x, y, w, h] = nums[..] { + if let Some([x, y, w, h]) = op.nums::<4>() { sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); } } From 5e2552dcb0e7cef8a24f5e930f4b13a3ddd14a9a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 22:18:19 +0100 Subject: [PATCH 14/46] M8: named property lists and their MCIDs are read once for a stream The tags cleanup resolved the page's `/Properties` at every named `BDC`, on main too: 20,000 of them with 5,000 entries took 28 s, as measured in review (optimized), against 0.07 s with one entry. And the second commit's `MCID` by reference was read again at every `BDC`: the document keeps an object larger than its cache only until it reads another, so reading the `MCID` evicted a property list that large, which was parsed again, all of it, at the next `BDC` (50 `BDC` took 12.3 s against 0.44 s with the `MCID` given directly). Now the named property lists are read once for the stream, in place when they are direct, and each name's `MCID` once. Tests (`crates/redact/tests/operator_work.rs`, a binary of its own whose tests run one at a time), each the least of three runs against one entry or operator, allowing ten times as long plus a second: `named_property_lists_are_read_once_for_a_page` (5,000 `BDC`, 2,000 entries) and `an_oversized_property_list_is_read_once_for_a_stream` (100 `BDC` naming a list too large to keep). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tags.rs | 22 +++++-- crates/redact/tests/operator_work.rs | 97 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 crates/redact/tests/operator_work.rs diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 15096433d5..03573a9743 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -35,6 +35,14 @@ impl Marked { /// left are taken as changed (their descriptions go) rather than compared. const MAX_COMPARED: usize = 1 << 28; +/// `f` of `o`, or of the object it refers to, without copying a direct one. +fn with_held(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> R) -> R { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + o => f(o), + } +} + fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { let mut out = Marked::default(); // Open sequences: the MCID with where its content starts in `ops` and the drawing operators @@ -49,16 +57,22 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { e.1 |= drawn > drawn_before; } }; + // The named property lists, read once, and each name's MCID, read once: a stream can name one + // at every `BDC` of thousands, and reading an object again can parse it again, all of it (the + // document keeps an object larger than its cache only until it reads another). A direct one + // is read where it is. + let held = resources.get(b"Properties").filter(|p| matches!(p, Object::Ref(_))).map(|p| doc.resolve(p)); + let named = held.as_deref().or_else(|| resources.get(b"Properties")).and_then(Object::as_dict); + let mut mcids: HashMap, Option> = HashMap::new(); for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { // The property list is the last operand, as the renderer reads it. let id = match op.back(0) { - Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { - let props = doc.resolve(p); - let prop = doc.resolve(props.as_dict()?.get(n)?); + Some(Object::Name(n)) => *mcids.entry(n.to_vec()).or_insert_with(|| { + let prop = named?.get(n)?; // A named property list may give its MCID by reference. - prop.as_dict()?.get(b"MCID").and_then(|id| doc.resolve(id).as_int()) + with_held(doc, prop, |prop| prop.as_dict()?.get(b"MCID").and_then(|id| with_held(doc, id, Object::as_int))) }), Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, diff --git a/crates/redact/tests/operator_work.rs b/crates/redact/tests/operator_work.rs new file mode 100644 index 0000000000..95bf8e33d5 --- /dev/null +++ b/crates/redact/tests/operator_work.rs @@ -0,0 +1,97 @@ +//! Work regressions for what a redaction looks up at operators a stream can hold thousands of: the +//! tags cleanup's named property lists at each `BDC` are not read whole each time. A test binary of +//! its own, so that none of the crate's other tests shares its process, and its tests run one at a +//! time. Each test compares two timings of its own, the least of three runs each, with many +//! entries or operators and with one, not a fixed time. The bound leaves room for slower machines: +//! each regression took seconds against milliseconds. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page with `resources` and `extra` objects (from object 5 on), drawing a word under a mark +/// (so the redaction rewrites the page) and then `ops`. +fn page(resources: &str, ops: &[u8], extra: Vec>) -> Document { + let content = [b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET ".as_slice(), ops].concat(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> {resources} >> >>" + ) + .into_bytes(), + stream_obj("", content), + ]; + objs.extend(extra); + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 200.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// One page whose `/Properties` holds `entries` entries, drawing `ops`. +fn properties(entries: usize, ops: &[u8]) -> Document { + let lists: String = (0..entries).map(|i| format!("/M{i} << /Lang (en) >> ")).collect(); + page(&format!("/Properties << {lists}>>"), ops, Vec::new()) +} + +/// One page whose property list `/M0` is object 5, which gives its `MCID` as object 6 and holds +/// 400,000 more entries: larger than the document keeps, so that reading it again parses it again, +/// all of it, once another object has been read since. The page draws `ops`. +fn oversized(ops: &[u8]) -> Document { + let more: String = (0..400_000).map(|i| format!("/J{i} /DeviceGray ")).collect(); + page("/Properties << /M0 5 0 R >>", ops, vec![format!("<< /MCID 6 0 R {more}>>").into_bytes(), b"0".to_vec()]) +} + +/// The tests' lock: they run one at a time, so that none is timed while another works. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn one_at_a_time() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// How long redacting a document `make` builds takes, the least of three runs. +fn redaction(make: impl Fn() -> Document) -> Duration { + (0..3) + .map(|_| { + let mut doc = make(); + let start = Instant::now(); + let report = apply(&mut doc, None).unwrap(); + let took = start.elapsed(); + assert_eq!(report.glyphs, 6, "the word under the mark goes"); + took + }) + .min() + .unwrap_or_default() +} + +/// Whether `many` took about as long as `one`. +fn about_as_long(many: Duration, one: Duration) -> bool { + many < one * 10 + Duration::from_secs(1) +} + +/// Written when the tags cleanup resolved the page's `/Properties` at every named `BDC`: 20,000 of +/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). +#[test] +fn named_property_lists_are_read_once_for_a_page() { + let _serial = one_at_a_time(); + let ops = b"/P /M0 BDC EMC ".repeat(5_000); + let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(2_000, &ops))); + assert!(about_as_long(many, one), "2,000 entries {many:?}, one {one:?}"); +} + +/// Written when the tags cleanup read a named property list's `MCID` by reference at each `BDC`: a +/// list larger than the document keeps was parsed again at each, 50 of them taking 12.3 s against +/// 0.44 s with the `MCID` given directly (optimized). +#[test] +fn an_oversized_property_list_is_read_once_for_a_stream() { + let _serial = one_at_a_time(); + let (one, many) = (redaction(|| oversized(b"/P /M0 BDC EMC ")), redaction(|| oversized(&b"/P /M0 BDC EMC ".repeat(100)))); + assert!(about_as_long(many, one), "100 operators {many:?}, one {one:?}"); +} From 042371bda9aa12c8e99e83b637342f5342aef85e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 22:51:42 +0100 Subject: [PATCH 15/46] M8: a test of its own for each operand read the fix changes Review of the whole change found reads it fixes that no test failed without: each was put back to main's read alone, and the tests passed. Each now has a case that fails without it: - the proof's `Tf` size (text starting outside the region reaches into it only at its size) and font (an undecodable font's text leaves the region unverifiable), and the strings a `TJ` carries; - the proof's `l`, `c`, `v` and `y` points after a surplus, from a point outside the region, and the walk's `m`, `l`, `c`, `v` and `y` points beside a mark, each alone; - the property list a second page sharing the resources still names; - a Type 3 font an ExtGState's `/Font` sets, kept by the page and its glyph by the font, and one without resources of its own, which keeps the page's pattern its glyph may paint with; - a Type 3 font only a soft mask's group selects, the mask's ExtGState named after a surplus; - the named property lists read once, with each `BDC` naming another entry, so that it is the lists, not each name's answer, that are kept. And the walk's `cs` says why it tells a Pattern space by the name `/Pattern` alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 5 + .../redact/src/tests/operands_from_the_end.rs | 102 ++++++++++++++++++ .../redact/src/tests/surviving_references.rs | 31 ++++++ crates/redact/src/tests_verify.rs | 20 ++++ crates/redact/tests/operator_work.rs | 9 +- 5 files changed, 163 insertions(+), 4 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b567735d2d..40cf5bc87e 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -571,6 +571,11 @@ impl Walk { Some(op) } b"cs" | b"CS" => { + // A Pattern space is told by the name `/Pattern` alone: one named through the + // resources' `/ColorSpace` clears the pattern, which the app's renderer keeps and + // Poppler doesn't (`cs` resets the colour, ISO 32000-2 §8.6.8). Looking the name up + // at each `cs` wants a cache for the page: a `/ColorSpace` too large for the + // document to keep is parsed again at each read. if op.last_name() != Some(b"Pattern") { if o == b"cs" { self.gs.fill_pattern = None; diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs index c18daa4dd8..0c802763e9 100644 --- a/crates/redact/src/tests/operands_from_the_end.rs +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -190,6 +190,19 @@ fn a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them() { mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1, "{}", String::from_utf8_lossy(curve)); } + // A line, and curves, from a point beside the mark whose points, after a surplus, stay beside + // it: read from the front, the surplus was a point under it. + for curve in [ + &b"100 100 500 m 510 510 l S"[..], + b"500 500 m 100 100 510 l S", + b"100 500 m 100 100 500 110 500 120 500 c f", + b"100 500 m 100 110 500 120 500 v f", + b"100 500 m 100 110 500 120 500 y f", + ] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "{}", String::from_utf8_lossy(curve)); + } } /// An `MCID` given by reference: in a named property list, and in a structure element's `/MCR`. @@ -236,3 +249,92 @@ fn an_mcid_given_by_reference_loses_what_redaction_removed() { let other = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); assert!(other.get(b"ActualText").is_some(), "{other:?}"); } + +/// The resources of page `page`, resolved. +fn resources_of(doc: &Document, page: usize) -> Dict { + let page = pdfcraft_model::pages(doc).swap_remove(page); + page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default() +} + +/// `dict`'s dictionary `key`, resolved. +fn dict_in(doc: &Document, dict: &Dict, key: &[u8]) -> Dict { + dict.get(key).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default() +} + +/// A property list a second page sharing the resources names after a surplus operand, once the +/// first page's block that named it was redacted: read from the front, the second page's name went +/// unseen and the list was retired from under it. +#[test] +fn a_property_list_another_page_names_after_a_surplus_operand_is_kept() { + use pdfcraft_cos::Stream; + let mut doc = one_page( + b"/Span /P0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", + "/Properties << /P0 7 0 R >>", + vec![b"<< /ActualText (SECRET) >>".to_vec()], + ); + let shared = doc.add(Object::Dict(resources_of(&doc, 0))); + doc.update_dict(ObjRef::new(3, 0), |d| d.set(b"Resources".to_vec(), Object::Ref(shared))).unwrap(); + let mut second = doc.get(ObjRef::new(3, 0)).as_dict().cloned().unwrap(); + let contents = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"(x) /Span /P0 BDC BT /F1 10 Tf 10 100 Td (Other) Tj ET EMC".to_vec()))); + second.set(b"Contents".to_vec(), Object::Ref(contents)); + let second = doc.add(Object::Dict(second)); + doc.update_dict(ObjRef::new(2, 0), |d| { + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(3, 0)), Object::Ref(second)])); + d.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, Some(&[0])).unwrap(); + let properties = dict_in(&doc, &resources_of(&doc, 1), b"Properties"); + assert!(properties.contains(b"P0"), "the second page's property list stays: {properties:?}"); +} + +/// A Type 3 font set only by an ExtGState's `/Font`, selected after a surplus operand: read from the +/// front, the cleanups took it for undrawn, dropped it from the page and unlinked its glyph. +#[test] +fn a_type3_font_an_extgstate_selects_after_a_surplus_operand_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /GS1 gs 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R /b 9 0 R >> \ + /Encoding << /Differences [97 /a /b] >> /FirstChar 97 /LastChar 98 /Widths [10 10] >>" + .to_vec(), + stream("", b"10 0 d0 0 0 10 10 re f"), + stream("", b"10 0 d0 0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let fonts = dict_in(&doc, &resources_of(&doc, 0), b"Font"); + assert!(fonts.contains(b"T3"), "the page still lists the font: {fonts:?}"); + let font = doc.get(ObjRef::new(7, 0)).as_dict().cloned().unwrap_or_default(); + let procs = dict_in(&doc, &font, b"CharProcs"); + assert!(procs.contains(b"a"), "the glyph the page draws stays: {procs:?}"); +} + +/// A Type 3 font without resources of its own, set by an ExtGState selected after a surplus operand: +/// its glyph paints through the page's resources, so it may paint with the page's pattern, which +/// stays once the page's own fill with it was redacted (the text itself fills with grey). Read from the front, the font was taken +/// for one with resources, and the pattern was pruned. +#[test] +fn a_type3_font_without_resources_an_extgstate_selects_after_a_surplus_operand_keeps_the_pattern() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Pattern << /P0 9 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 10 260 20 20 re f 0 g BT (x) /GS1 gs 150 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 /Pattern cs /P0 scn 0 0 10 10 re f"), + stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, 0)); +} diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index 0d171b4008..d02e76832e 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -783,6 +783,37 @@ fn a_type3_font_only_a_soft_mask_without_resources_selects_stays_on_the_page() { assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); } +/// The same soft mask, its ExtGState selected after a surplus operand (`(x) /GS1 gs`), which the +/// renderer ignores: read from the front, the cleanups never read the group, and dropped the font +/// it selects and its glyph. +#[test] +fn a_type3_font_only_a_soft_mask_selected_after_a_surplus_operand_selects_stays_on_the_page() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AC) Tj ET (x) /GS1 gs 150 150 100 100 re f"), + b"<< /Font << /T3 6 0 R /T4 7 0 R >> /ExtGState << /GS1 << /Type /ExtGState /SMask << /Type /Mask /S /Luminosity /G 8 0 R >> >> >> >>" + .to_vec(), + type3_abc(9), + type3_abc(12), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency /CS /DeviceGray >>", + b"1 g BT /T4 40 Tf 160 180 Td (B) Tj ET", + ), + stream("", b"% REDACTED_GLYPH_A"), + stream("", b"% UNUSED_GLYPH_B"), + stream("", b"% PAGE_GLYPH_C"), + stream("", b"% UNUSED_T4_GLYPH_A"), + stream("", b"% MASK_GLYPH_B"), + stream("", b"% UNUSED_T4_GLYPH_C"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(full_save_holds(&doc, b"MASK_GLYPH_B"), "the glyph the soft mask shows was unlinked"); + assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); +} + /// The same group, its content behind a predictor whose `/DecodeParms` are given by reference: /// the page's cleanup reads it through them, and keeps the font it selects. #[test] diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index bd9cb909de..6e34fba352 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -912,6 +912,8 @@ fn the_proof_reads_operands_from_the_end() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 8 0 R >> >>", b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q"); for (content, spec) in [ (&b"BT (x) /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"[..], Spec::default()), + // The size: text starting outside the region reaches into it only at its size. + (b"BT (x) /F1 10 Tf -60 100 Td (AAAAAAAAAAAAAAAAAAAAAAAAAAAA) Tj ET", Spec::default()), (b"BT /F1 10 Tf 10 100 Td (x) [(SECRETVALUE)] TJ ET", Spec::default()), (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form.clone()], ..Spec::default() }), // An image under the mark, and one in a form under it. @@ -960,6 +962,11 @@ fn the_proof_reads_a_paths_points_from_the_end() { b"0 0 1 rg 9 200 200 m 9 20 98 40 100 60 108 c h f", b"0 0 1 rg 9 200 200 m 9 20 98 60 108 v h f", b"0 0 1 rg 9 200 200 m 9 20 98 60 108 y h f", + // A segment after a surplus, from a point outside the region. + b"0 0 1 rg 200 200 m 200 20 100 l h f", + b"0 0 1 rg 200 200 m 200 20 100 30 100 40 100 c h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 v h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 y h f", ] { let mut doc = marked_content(content, Spec::default()); doc.require_full_save(); @@ -968,6 +975,19 @@ fn the_proof_reads_a_paths_points_from_the_end() { } } +/// Which font a `Tf` selects, and which strings a `TJ` shows, are read from the end too: text an +/// undecodable font shows after a surplus operand, away from the mark, leaves the region +/// unverifiable. Read from the front, no font was set, or no string was shown, and it passed. +#[test] +fn the_proof_reads_the_font_and_the_strings_from_the_end() { + for content in [&b"BT (x) /F2 10 Tf 10 200 Td <00410042> Tj ET"[..], b"BT /F2 10 Tf 10 200 Td (x) [<00410042>] TJ ET"] { + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + /// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws /// after a surplus operand can't be placed, as one it draws without one can't: the region is /// unverifiable. diff --git a/crates/redact/tests/operator_work.rs b/crates/redact/tests/operator_work.rs index 95bf8e33d5..e4340fc021 100644 --- a/crates/redact/tests/operator_work.rs +++ b/crates/redact/tests/operator_work.rs @@ -77,13 +77,14 @@ fn about_as_long(many: Duration, one: Duration) -> bool { } /// Written when the tags cleanup resolved the page's `/Properties` at every named `BDC`: 20,000 of -/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). +/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). Each `BDC` here +/// names another entry, so that it is the lists, not each name's answer, that are read once. #[test] fn named_property_lists_are_read_once_for_a_page() { let _serial = one_at_a_time(); - let ops = b"/P /M0 BDC EMC ".repeat(5_000); - let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(2_000, &ops))); - assert!(about_as_long(many, one), "2,000 entries {many:?}, one {one:?}"); + let ops: Vec = (0..5_000).map(|i| format!("/P /M{i} BDC EMC ")).collect::().into_bytes(); + let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(5_000, &ops))); + assert!(about_as_long(many, one), "5,000 entries {many:?}, one {one:?}"); } /// Written when the tags cleanup read a named property list's `MCID` by reference at each `BDC`: a From 6b53325f286207fa43b22b4e11fc5761febef618 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 19:09:11 +0100 Subject: [PATCH 16/46] M11: Optimize stores one-bit images and masks past 80 million pixels as monochrome images The image steps leave an image over 80 million pixels alone, for memory: decoded at eight bits a pixel or more, it would hold 80 MB a component and more. An image already of one bit a pixel holds a byte for each 8 samples of a row, so a large black-and-white scan (an A0 page at 600 ppi has about 560 million pixels) was left as stored though its samples would take less than 80 MB. The monochrome step now takes an image of one bit a pixel as stored (an image mask, or one bit in one colour component), a picture's one-bit mask included, while its rows hold no more bytes than 80 million pixels of eight bits (`one_bit_bytes`): up to 640 million pixels when it is wide, and a strip one sample wide no taller than before. One it would pack from eight bits keeps the 80 million limit, as it is decoded at eight bits first. A fax image's code is allowed no more than 80 MB for its pixels however large the page, and four bytes a row, where it was allowed about a byte a pixel (`fax_code_allowance`). When planning how many pictures run at once, a one-bit image counts as the bytes its rows hold and one more a row, as decoding allows each row's end a few bytes besides its samples. The optimize README and the parity note say which images the monochrome step now takes, and the note states JBIG2's limit of 2^28 pixels in all, as the README does. `group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit` now expects its one-bit image of 10,000 by 8,001 stored again, where it was left alone. The packing path counts what it decodes, for the tests, as the one-bit path does. Tests: `one_bit_images_past_80_million_pixels_are_stored_as_group_4` (a one-bit image, a picture's stencil mask and a picture's one-bit soft mask of 9,000 by 8,900), `a_one_bit_image_past_80_million_pixels_is_averaged_with_the_default_settings`, `an_image_too_large_for_the_jbig2_decoder_stays_as_it_is` (2^28 pixels stored as JBIG2, a row more left alone, and averaged), and the unit tests `one_bit_images_are_decoded_up_to_their_own_limit` and `a_one_bit_picture_counts_as_the_bytes_of_its_rows` (each with a strip one sample wide, at its limit and past it) and `a_fax_images_code_is_allowed_80_million_bytes_and_four_a_row` and `a_zero_wide_image_is_not_decoded`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/optimize/README.md | 15 ++-- crates/optimize/src/images.rs | 129 ++++++++++++++++++++++++++++-- crates/optimize/tests/optimize.rs | 95 +++++++++++++++++++--- parity/acrobat-features.toml | 4 +- 4 files changed, 217 insertions(+), 26 deletions(-) diff --git a/crates/optimize/README.md b/crates/optimize/README.md index a6dfd9caed..a2436676bd 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -34,9 +34,13 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A stencil `/Mask` has its own resolution, so the picture changes under it and it keeps its size and its bits (stored again as a picture's mask, below). - An image is replaced only if the result is smaller. Images over 80 million pixels are left - alone by the image steps, here and below, whatever their settings (memory); Clean Up may still - Flate an unfiltered or LZW one. Left alone by these colour and grey + An image is replaced only if the result is smaller. Images over 80 million pixels are left alone + by the image steps, here and below, whatever their settings (memory), except that the monochrome + step below takes an image already of one bit a pixel, a mask included, while its rows hold no more + bytes than 80 million pixels of eight bits (each row starts on a new byte: a byte for each 8 + samples and one for any left over): up to 640 million pixels when it is wide, and a strip one + sample wide no taller than one of eight bits. Clean Up may still Flate an unfiltered or LZW one. + Left alone by these colour and grey settings: CMYK and other colour spaces, image masks, colour-key masks, decode arrays, more than 8 bits, JPEG 2000, JBIG2 and CCITT (black-and-white images, CCITT and JBIG2 among them, are below). A JPEG is read with its colour transform (an Adobe marker's, else `/ColorTransform`, ISO 32000-2 §7.4.8), so one @@ -119,8 +123,9 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going more bits stays as it is, to be resampled with its picture, which wants it at eight. The report counts masks as images. - Not yet: inline images, a picture's mask resampled, or packed to one bit from more, Group 4 - for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side, where the decoder - stops (its 2^28 in all is past the 80 million pixels above). + for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side or 2^28 in all, + where the decoder stops: with JBIG2 chosen, a one-bit image larger than that stays unless + it is averaged. - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller, a dictionary shared by up to 16 images when that is smaller still. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 24fba3ef7a..4037de82f3 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -13,6 +13,15 @@ use crate::{Compression, ImageSettings, MonoCompression, MonoSettings, OptimizeE /// Images with more pixels than this are left alone (memory). pub(crate) const MAX_PIXELS: u64 = 80_000_000; +/// The bytes an image of one bit a pixel holds as stored: each row starts on a new byte, a byte for +/// each 8 samples and one for any left over. The monochrome step takes such an image while these +/// are no more than [`MAX_PIXELS`], as many as that many pixels of eight bits hold: a wide one up +/// to 640 million pixels (an A0 page at 600 ppi is about 560 million), where at eight bits a pixel +/// it would be left alone, and a strip one sample wide no taller than one of eight bits. +fn one_bit_bytes(w: u64, h: u64) -> u64 { + w.div_ceil(8).saturating_mul(h) +} + /// The resources of a page, inherited from the page tree if the page has none. pub(crate) fn page_resources(doc: &Document, page: ObjRef) -> Option { held_page_resources(doc, page).map(|(resources, _)| resources) @@ -829,6 +838,21 @@ fn one_component(doc: &Document, cs: &Object) -> bool { } } +/// What the filters before a fax image's decoder may produce, its code: at most about a byte a +/// pixel and each row's end, as redaction allows, but no more than `MAX_PIXELS` bytes for its +/// pixels (and four a row), so a large page's code can't cost a byte a pixel. +fn fax_code_allowance(w: usize, h: usize) -> Option { + w.checked_mul(h)?.min(usize::try_from(MAX_PIXELS).ok()?).checked_add(h.checked_mul(4)?)?.checked_add(4096) +} + +/// Whether `d` is an image of one bit a pixel as stored: an image mask, or one bit in one colour +/// component. +fn one_bit_as_stored(doc: &Document, d: &Dict) -> bool { + let bits = d.int(b"BitsPerComponent"); + let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); + (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) +} + /// A pixel's samples as a 1 or a 0, or `None` for a colour that is neither. type Pixel = fn(&[u8]) -> Option; @@ -854,19 +878,23 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut let d = &s.dict; // Through u32 first, as everywhere here, so the product can't overflow. let (w, h) = (u32::try_from(d.int(b"Width")?).ok()?, u32::try_from(d.int(b"Height")?).ok()?); - if w == 0 || h == 0 || u64::from(w) * u64::from(h) > MAX_PIXELS { + // An image already of one bit a pixel is taken while its rows hold no more than `MAX_PIXELS` + // bytes (`one_bit_bytes`), one packed from more bits (2, 4 or 8) up to `MAX_PIXELS` pixels, as + // before: its samples are decoded as stored first, eight-bit RGB at three bytes a pixel. + let as_stored = one_bit_as_stored(doc, d); + let held = if as_stored { one_bit_bytes(u64::from(w), u64::from(h)) } else { u64::from(w) * u64::from(h) }; + if w == 0 || h == 0 || held > MAX_PIXELS { return None; } let (wu, hu) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); let stride = wu.div_ceil(8); let bits = d.int(b"BitsPerComponent"); let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); - if (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) { - // No further than its rows can need: a fax image's code (the filters before its decoder - // produce it) takes at most about a byte a pixel and each row's end, as redaction allows. + if as_stored { + // No further than its rows can need. let len = stride.checked_mul(hu)?; let max = match s.filters().last() { - Some((pdfcraft_filters::Filter::CcittFax, _)) => wu.checked_mul(hu)?.checked_add(hu.checked_mul(4)?)?.checked_add(4096)?, + Some((pdfcraft_filters::Filter::CcittFax, _)) => fax_code_allowance(wu, hu)?, // A JBIG2 page, which has to be the image's size, is `len` bytes as rows of a bit. _ => allowance(len, h)?, }; @@ -937,6 +965,7 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut _ => return None, }; let row_len = wu.checked_mul(n)?.checked_mul(bpc)?.div_ceil(8); + count_decoded(u64::from(w) * u64::from(h)); // Through its parameters given by reference too: without a PNG predictor's tag bytes taken // off, rows of 0 and 255 (or palette entries 0 and 1) would still pass below, each shifted. let samples = budget.decode_filters(doc, s, allowance(row_len.checked_mul(hu)?, h)?).ok()?; @@ -1605,12 +1634,21 @@ const PICTURE_MEMORY: u64 = 1 << 30; /// What a picture holds while it is worked out, for each pixel: decoded, and resampled or encoded. const PICTURE_BYTES_PER_PIXEL: u64 = 8; -/// A picture's pixels, if it is small enough to be decoded at all. +/// A picture's pixels, if it is small enough to be decoded at all. One of one bit a pixel as +/// stored is taken while the bytes its rows hold ([`one_bit_bytes`]) are no more than +/// `MAX_PIXELS`, and counts as those and one more a row: decoding allows each row's end a few +/// bytes besides its samples ([`allowance`], [`fax_code_allowance`]), which for an image a few +/// samples wide is as much again as its samples. fn picture_pixels(doc: &Document, r: ObjRef) -> Option { let object = doc.get(r); let Object::Stream(s) = &*object else { return None }; let (w, h) = (u64::try_from(s.dict.int(b"Width")?).ok()?, u64::try_from(s.dict.int(b"Height")?).ok()?); - w.checked_mul(h).filter(|pixels| *pixels <= MAX_PIXELS) + if one_bit_as_stored(doc, &s.dict) { + let bytes = one_bit_bytes(w, h); + return (bytes <= MAX_PIXELS).then(|| bytes.saturating_add(h)); + } + let pixels = w.checked_mul(h)?; + (pixels <= MAX_PIXELS).then_some(pixels) } /// How many pictures of `largest` pixels to work on at once: `workers`, or as many as @@ -1861,6 +1899,83 @@ mod dimension_tests { assert_eq!(threads_for(8, 0), 8); } + /// A one-bit image counts as the bytes its rows hold and one more a row when planning how many + /// pictures run at once, while those it holds are up to `MAX_PIXELS`: about an eighth of its + /// pixels when wide, two bytes a row when a few samples wide. One of eight bits counts whole, + /// up to `MAX_PIXELS`. + #[test] + fn a_one_bit_picture_counts_as_the_bytes_of_its_rows() { + let mut doc = Document::new_empty(); + let mut add = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))) + }; + let (one_bit, past_one_bit, eight_bit) = (add(20_000, 32_000, 1), add(20_000, 32_001, 1), add(10_000, 8_001, 8)); + let (strip, past_strip, narrow) = (add(1, 80_000_000, 1), add(1, 80_000_001, 1), add(8, 10_000_000, 1)); + assert_eq!(picture_pixels(&doc, one_bit), Some(80_000_000 + 32_000)); + assert_eq!(picture_pixels(&doc, past_one_bit), None); + assert_eq!(picture_pixels(&doc, strip), Some(160_000_000), "a byte a row, and one more"); + assert_eq!(picture_pixels(&doc, past_strip), None); + assert_eq!(picture_pixels(&doc, narrow), Some(20_000_000)); + assert_eq!(threads_for(8, 20_000_000), 6); + assert_eq!(threads_for(8, 160_000_000), 1); + assert_eq!(picture_pixels(&doc, eight_bit), None); + assert_eq!(threads_for(8, 80_000_000), 1); + } + + /// A fax image's code may take about a byte a pixel and its rows' ends, but no more than + /// `MAX_PIXELS` bytes for its pixels however large the page, and four a row. + #[test] + fn a_fax_images_code_is_allowed_80_million_bytes_and_four_a_row() { + assert_eq!(fax_code_allowance(1_000, 1_000), Some(1_000_000 + 4_000 + 4_096)); + assert_eq!(fax_code_allowance(20_000, 32_000), Some(80_000_000 + 128_000 + 4_096)); + assert_eq!(fax_code_allowance(1, 80_000_000), Some(80_000_000 + 320_000_000 + 4_096)); + } + + /// The monochrome step decodes an image already of one bit a pixel while its rows hold no more + /// than `MAX_PIXELS` bytes, and one it would pack from eight bits up to `MAX_PIXELS` pixels. + /// Past those, nothing is decoded. + #[test] + fn one_bit_images_are_decoded_up_to_their_own_limit() { + let image = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + Stream::from_raw(dict, Vec::new()) + }; + let decoded = |s: &Stream| { + DECODED_PIXELS.with(|c| c.set(0)); + one_bit(&Document::new_empty(), s, true, false, &mut ImageBudget::with_work(u64::MAX)); + DECODED_PIXELS.with(|c| c.get()) + }; + assert_eq!(decoded(&image(20_000, 32_000, 1)), 640_000_000, "one bit, at the limit"); + assert_eq!(decoded(&image(20_000, 32_001, 1)), 0, "one bit, past it"); + assert_eq!(decoded(&image(1, 80_000_000, 1)), 80_000_000, "one bit a sample wide, a byte a row, at the limit"); + assert_eq!(decoded(&image(1, 80_000_001, 1)), 0, "one bit a sample wide, past it"); + assert_eq!(decoded(&image(10_000, 8_000, 8)), 80_000_000, "eight bits, at theirs"); + assert_eq!(decoded(&image(10_000, 8_001, 8)), 0, "eight bits, past theirs"); + } + + /// A zero-wide image isn't decoded: its rows hold no bytes, but its filters would be allowed + /// three a row. + #[test] + fn a_zero_wide_image_is_not_decoded() { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(0)); + dict.set(b"Height".to_vec(), Object::Int(1_000_000)); + dict.set(b"ImageMask".to_vec(), Object::Bool(true)); + let s = Stream::flate(dict, &vec![0u8; 1 << 20]); + let mut budget = ImageBudget::with_work(1 << 40); + assert!(one_bit(&Document::new_empty(), &s, true, false, &mut budget).is_none()); + assert_eq!(budget.work_left(), 1 << 40); + } + /// Only what redaction can store again without the dictionary shares one: an image mask, or /// DeviceGray named directly, of at most 2^26 pixels. #[test] diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7c3e76f9b9..7da6151deb 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -423,8 +423,10 @@ fn black_and_white_images_become_ccitt_group_4() { /// Which 1-bit images qualify for Group 4: those of one colour component, an ICC profile of one /// component and a DeviceN of one colourant among them, but not one of three components or two /// colourants (more than a bit a pixel); image masks of one bit, as `/BitsPerComponent 1` says or -/// by default, but not one that says 8; and images of up to 80 million pixels, not one row more. -/// The byte totals count exactly the images stored again. +/// by default, but not one that says 8; and images past 80 million pixels too, where an image of +/// eight bits would be left alone (`one_bit_images_are_decoded_up_to_their_own_limit` checks the +/// bytes of rows a one-bit image may hold without decoding so much). The byte totals count exactly +/// the images stored again. #[test] fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mut doc = Document::new_empty(); @@ -457,7 +459,7 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mask = |bits: i64| [("ImageMask", Object::Bool(true)), ("BitsPerComponent", bits.into())]; let mask_1 = mono_image(&mut doc, w, h, &text, None, &mask(1)); let mask_8 = mono_image(&mut doc, w, h, &bilevel(8 * w, h, false), None, &mask(8)); - // 10,000 × 8,000 is the limit itself. + // 10,000 × 8,000 is the limit of an image of eight bits; one of one bit goes past it. let grey = one_bit(Object::name("DeviceGray")); let at_limit = mono_image(&mut doc, 10_000, 8_000, &bilevel(10_000, 8_000, false), None, &grey); let past_limit = mono_image(&mut doc, 10_000, 8_001, &bilevel(10_000, 8_001, false), None, &grey); @@ -474,8 +476,8 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { }) .collect(); page(&mut doc, &names.iter().map(String::as_str).zip(all).collect::>(), &content); - let converted = [icc_1, spot, mask_1, at_limit]; - let left = [icc_3, two_spots, mask_8, past_limit]; + let converted = [icc_1, spot, mask_1, at_limit, past_limit]; + let left = [icc_3, two_spots, mask_8]; let before = |doc: &Document, refs: &[ObjRef]| refs.iter().map(|r| stream(doc, *r).raw).collect::>(); let (old, untouched) = (before(&doc, &converted), before(&doc, &left)); let r = optimize(&mut doc, &Settings::default()).unwrap(); @@ -4925,15 +4927,22 @@ fn indexed_thin_rules_survive_averaging_for_both_palette_orders_and_decodes() { } } -/// An image past what the JBIG2 decoder takes on a side stays as it is: JBIG2 can state the size, -/// but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it drew nothing here. At 65,535 -/// on a side, the same bars do become JBIG2, so the fixture does reach that choice, and the coding -/// stored (symbols, here) reads back through that decoder. +/// An image past what the JBIG2 decoder takes on a side, or past 2^28 pixels in all, stays as it +/// is: JBIG2 can state the size, but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it +/// drew nothing here. At 65,535 on a side, and at 2^28 in all (past 80 million, which the +/// monochrome step takes of a one-bit image), the same bars do become JBIG2, so the fixture does +/// reach that choice, and the coding stored reads back through that decoder. Averaged, an image past +/// 2^28 becomes small enough and is stored as JBIG2 too. #[test] fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { - for (w, h, stored) in - [(65_535u32, 16u32, &b"JBIG2Decode"[..]), (16, 65_535, b"JBIG2Decode"), (65_536, 16, b"FlateDecode"), (16, 65_536, b"FlateDecode")] - { + for (w, h, stored) in [ + (65_535u32, 16u32, &b"JBIG2Decode"[..]), + (16, 65_535, b"JBIG2Decode"), + (65_536, 16, b"FlateDecode"), + (16, 65_536, b"FlateDecode"), + (16_384, 16_384, b"JBIG2Decode"), + (16_384, 16_385, b"FlateDecode"), + ] { let mut doc = Document::new_empty(); let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); page(&mut doc, &[("Im", image)], &format!("q {w} 0 0 {h} 0 0 cm /Im Do Q")); @@ -4958,6 +4967,15 @@ fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { assert!(decoded_image(&doc, &s).unwrap() == bits, "{w} × {h} reads back"); } } + let (w, h) = (16_384u32, 16_385u32); + let mut doc = Document::new_empty(); + let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("Im", image)], &format!("q {pw} 0 0 {ph} 0 0 cm /Im Do Q")); + let settings = Settings { mono: MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }, ..Settings::default() }; + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + assert_eq!(stream(&doc, image).dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "averaged, it is stored as JBIG2"); } /// The colour and grey steps leave an image over 80 million pixels as it is, whatever their settings @@ -5405,6 +5423,59 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { /// the page looks the same to the pixel. A mask two pictures share is stored once, and a stencil /// the page also draws by name is stored as that image. An eight-bit soft mask of only black and /// white stays as it is, to be resampled with its picture. With `mono: Retain` nothing changes. +/// One-bit images past 80 million pixels are stored again, as images of eight bits that large are +/// not: their rows hold a byte for each 8 samples. A picture's stencil mask and one-bit soft mask +/// too. +#[test] +fn one_bit_images_past_80_million_pixels_are_stored_as_group_4() { + let mut doc = Document::new_empty(); + let (w, h) = (9_000, 8_900); + let text = bilevel(w, h, false); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let stencil = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let soft = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let (picture, shaded) = (masked(&mut doc, "Mask", stencil), masked(&mut doc, "SMask", soft)); + page( + &mut doc, + &[("I", one_bit), ("P", picture), ("S", shaded)], + "q 300 0 0 300 20 400 cm /I Do Q q 300 0 0 120 20 100 cm /P Do Q q 300 0 0 120 20 250 cm /S Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let mono = MonoSettings { downsample: false, ..MonoSettings::REDUCE }; + let before = [one_bit, stencil, soft].map(|r| stream(&doc, r)); + let r = optimize(&mut doc, &Settings { color: keep, gray: keep, mono, ..Settings::default() }).unwrap(); + assert_eq!(r.images_recompressed, 3, "{r:?}"); + for (r, old) in [one_bit, stencil, soft].into_iter().zip(&before) { + let new = stream(&doc, r); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{r:?}"); + assert!(new.raw.len() < old.raw.len(), "{r:?}: {} against {}", new.raw.len(), old.raw.len()); + assert_eq!(decoded_image(&doc, &new).unwrap(), text, "{r:?} keeps its bits"); + } +} + +/// With the default settings (Reduce File Size), a one-bit image past 80 million pixels drawn at +/// 600 ppi is averaged to 300 ppi, as a smaller one is: vertical stripes four samples wide. +#[test] +fn a_one_bit_image_past_80_million_pixels_is_averaged_with_the_default_settings() { + let (w, h) = (9_000u32, 8_900u32); + let mut doc = Document::new_empty(); + let stripes = vec![0x0F; (w as usize).div_ceil(8) * h as usize]; + let one_bit = mono_image(&mut doc, w, h, &stripes, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("I", one_bit)], &format!("q {pw} 0 0 {ph} 0 0 cm /I Do Q")); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + let s = stream(&doc, one_bit); + assert_eq!((s.dict.int(b"Width"), s.dict.int(b"Height")), (Some(4_500), Some(4_450))); +} + #[test] fn a_pictures_one_bit_masks_are_stored_again() { let mut doc = Document::new_empty(); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 046c080c62..7d0864676b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7549,8 +7549,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] -evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] -notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." +evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::one_bit_images_past_80_million_pixels_are_stored_as_group_4", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] +notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), except images and masks already of one bit a pixel whose rows hold no more bytes than 80 million pixels of eight bits (a byte for each 8 samples of a row and one for any left over, so up to 640 million pixels when wide, and a strip one sample wide no taller than at eight bits), which the monochrome step averages and stores again as it does smaller ones (with JBIG2 chosen, those past 2^28 pixels stay unless averaged), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side or 2^28 in all, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." [[feature]] id = "optimize.only-if-smaller" From a23b7c7f738e130f05ad8daa4c57f4ef56a356ab Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 09:17:32 -0500 Subject: [PATCH 17/46] fix(redact): apply marks with an indirect /Subtype and judge inline annotations (#5849) * fix(redact): apply marks with an indirect /Subtype and judge inline annotations A Redact annotation whose /Subtype is an indirect reference was passed over by apply although the audit, which resolves it, reports it as a mark: the other marks applied and the content under that one stayed. Annotation dictionaries written in place in /Annots were kept whatever they covered, and an inline reply or pop-up kept a removed comment reachable through /IRT or /Parent. /Subtype is now resolved wherever redaction classifies an annotation. Inline entries are judged by their rectangle and relationships like referenced ones, on the marked page and across pages, and the pop-up an annotation names (/Popup) goes with it. * address review: an inline widget's /Parent field goes even when no referenced widget names it * fmt --- crates/redact/src/lib.rs | 227 ++++++++++++++++++++++++++++--------- crates/redact/src/tests.rs | 115 +++++++++++++++++++ 2 files changed, 289 insertions(+), 53 deletions(-) diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 5938f20a68..7fd34dae12 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -247,6 +247,19 @@ pub(crate) fn annots_of(doc: &Document, page: &Dict) -> Vec { page.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default() } +/// An annotation's `/Subtype`, resolved: it may be an indirect reference (ISO 32000-2 §7.3.10), +/// and the audit reads it that way, so a mark it reports must not be one [`apply`] passes over. +pub(crate) fn subtype(doc: &Document, d: &Dict) -> Option> { + d.get(b"Subtype").and_then(|s| doc.resolve(s).as_name().map(<[u8]>::to_vec)) +} + +/// The pop-up `d` names (`/Popup`), when it is one: it goes with `d`. Anything else it might name +/// (a mark on another page, say) is left alone. +fn named_popup(doc: &Document, d: &Dict) -> Option { + let r = d.reference(b"Popup")?; + doc.get(r).as_dict().is_some_and(|p| subtype(doc, p).as_deref() == Some(b"Popup")).then_some(r) +} + /// Every annotation the pages list. Compared before and after a removal, what is no longer listed /// leaves the structure tree too ([`tags::drop_object_refs`]). pub(crate) fn listed_annotations(doc: &Document) -> HashSet { @@ -267,14 +280,14 @@ pub(crate) fn marks_checked(doc: &Document) -> (Vec, Vec) { let Some(r) = a.as_ref() else { // A mark written in place has no object to find, apply or remove: refuse it // rather than apply the others as if it were not there. - if doc.resolve(&a).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Redact")) { + if doc.resolve(&a).as_dict().is_some_and(|d| subtype(doc, d).as_deref() == Some(b"Redact")) { unreadable.push(pi); } continue; }; let obj = doc.get(r); let Some(d) = obj.as_dict() else { continue }; - if d.name(b"Subtype") != Some(b"Redact") { + if subtype(doc, d).as_deref() != Some(b"Redact") { continue; } let mut rects: Vec<[f64; 4]> = quad_points(doc, d) @@ -580,18 +593,41 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co let annots = annots_of(doc, &p.dict); let mut kept = Vec::with_capacity(annots.len()); for a in &annots { - let Some(r) = a.as_ref() else { - kept.push(a.clone()); - continue; - }; - let obj = doc.get(r); + // An entry written in place is read as one listed by reference: its `/IRT` or + // `/Parent` keeps a removed comment reachable all the same, and it has its own text. + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { kept.push(a.clone()); continue; }; - let is_reply = - d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); - let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); + let is_reply = d.reference(b"IRT").is_some_and(|x| removed_everywhere.contains(&x)) && subtype(doc, d).as_deref() != Some(b"Widget"); + let is_popup = d.reference(b"Parent").is_some_and(|x| removed_everywhere.contains(&x)); + let Some(r) = a.as_ref() else { + if is_reply || is_popup { + grew = true; + if is_reply { + report.annotations += 1; + } + // Its appearance can be shared with an annotation that stays elsewhere. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } + // It has no reference to remove elsewhere; the pop-up it names has. + if let Some(p) = named_popup(doc, d) { + removed_everywhere.insert(p); + } + } else { + kept.push(a.clone()); + } + continue; + }; if removed_everywhere.contains(&r) || is_reply || is_popup { // A reply or pop-up removed here can share its appearance with an annotation // that stays on an unselected page, like one removed under a mark: the @@ -638,6 +674,30 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co report.fields += 1; } } + // A widget written in place on another page names its deleted field through `/Parent`, which + // would keep the field and its value in the file: it goes with the field. + if !doomed_fields.is_empty() { + for p in pdfcraft_model::pages(doc) { + let annots = annots_of(doc, &p.dict); + let kept: Vec = annots + .iter() + .filter(|a| { + a.as_ref().is_some() || !matches!(&**a, Object::Dict(d) if d.reference(b"Parent").is_some_and(|f| doomed_fields.contains(&f))) + }) + .cloned() + .collect(); + if kept.len() != annots.len() { + report.annotations += annots.len() - kept.len(); + doc.update_dict(p.obj, |d| { + if kept.is_empty() { + d.remove(b"Annots"); + } else { + d.set(b"Annots".to_vec(), Object::Array(kept)); + } + })?; + } + } + } // 3a. A tagged PDF's structure tree still points at the removed comments and widgets // (`/OBJR`, #2874): drop those references, or a full rewrite keeps them reachable, the @@ -885,30 +945,66 @@ fn remove_page_annotations( page_marks: &[&Mark], ) -> Result<(), RedactError> { let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; - // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). + // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). An entry + // written in place (a dictionary in `/Annots`, not a reference) is judged like any other: its + // `/Contents` and `/IRT` stay in the file for as long as the page lists it. let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); + /// What decides an entry's fate, read once. + struct Entry { + r: Option, + widget: bool, + irt: Option, + parent: Option, + popup: Option, + } + let entries: Vec> = annots + .iter() + .map(|a| { + let obj = doc.resolve(a); + let d = obj.as_dict()?; + Some(Entry { + r: a.as_ref(), + widget: subtype(doc, d).as_deref() == Some(b"Widget"), + irt: d.reference(b"IRT"), + parent: d.reference(b"Parent"), + popup: named_popup(doc, d), + }) + }) + .collect(); let mut removed: Vec = Vec::new(); - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - let obj = doc.get(r); + // Entries written in place that go, by position in `/Annots`. + let mut inline_gone = vec![false; annots.len()]; + for (i, a) in annots.iter().enumerate() { + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { continue }; - let subtype = d.name(b"Subtype").unwrap_or(b""); - if subtype == b"Redact" { - if page_marks.iter().any(|m| m.obj == r) { + let r = a.as_ref(); + let kind = subtype(doc, d).unwrap_or_default(); + if kind == b"Redact" { + // A mark written in place was refused before anything changed. + if let Some(r) = r + && page_marks.iter().any(|m| m.obj == r) + { removed.push(r); } continue; } - if subtype == b"Popup" { + if kind == b"Popup" { continue; } if rect_of(doc, d.get(b"Rect")).is_some_and(|b| rects.iter().any(|x| pdfcraft_content::overlaps(*x, b, 0.0))) { - removed.push(r); - if subtype == b"Widget" { - if let Some((_, field)) = widget_owner.iter().find(|(w, _)| *w == r) - && !doomed_fields.contains(field) + match r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + if kind == b"Widget" { + // A widget written in place has no reference of its own; its `/Parent` is the field, + // even one no referenced widget belongs to (it then has no entry in `widget_owner`). + // Step 3 deletes only what the form lists as a field, so any `/Parent` is safe here. + let owner = r.and_then(|r| widget_owner.iter().find(|(w, _)| *w == r).map(|(_, f)| *f)).or_else(|| d.reference(b"Parent")); + if let Some(field) = owner + && !doomed_fields.contains(&field) { - doomed_fields.push(*field); + doomed_fields.push(field); } } else { report.annotations += 1; @@ -919,29 +1015,25 @@ fn remove_page_annotations( // those: a reply placed elsewhere would keep its parent — the comment and the QuadPoints // of the redacted words — in the file through its `/IRT` (#2659). Each pass removes at // least one more annotation of this page or stops, so it ends. - let replies: Vec<(ObjRef, ObjRef)> = annots - .iter() - .filter_map(Object::as_ref) - .filter_map(|r| { - let obj = doc.get(r); - let d = obj.as_dict()?; - (d.name(b"Subtype") != Some(b"Widget")).then_some(())?; - Some((r, d.get(b"IRT").and_then(Object::as_ref)?)) - }) - .collect(); loop { - let before = removed.len(); - for (r, parent) in &replies { - if !removed.contains(r) && removed.contains(parent) { - removed.push(*r); - report.annotations += 1; + let mut grew = false; + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone || e.widget || !e.irt.is_some_and(|p| removed.contains(&p)) { + continue; } + match e.r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + report.annotations += 1; + grew = true; } - if removed.len() == before { + if !grew { break; } } - removed_everywhere.extend(removed.iter().copied()); // A removed annotation's appearance streams can be shared with a kept one on another page, // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. for reference in &removed { @@ -961,24 +1053,53 @@ fn remove_page_annotations( } } } - // Pop-ups dropped with their parent below count too: a reply on another page may answer one. - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - if doc.get(r).as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) { - removed_everywhere.insert(r); + // An entry written in place that goes keeps no reference, but its appearances count as well. + for (i, a) in annots.iter().enumerate() { + if *appearance_scan_limit { + break; + } + if let (true, Some(annotation)) = (inline_gone[i], doc.resolve(a).as_dict()) { + match appearance_refs(doc, annotation) { + Some(references) => removed_appearances.extend(references), + None => *appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } } } + // Pop-ups go with their annotation: those that name it (`/Parent`), and those it names + // (`/Popup`), the only link an annotation written in place can have to its pop-up. + let mut going: HashSet = removed.iter().copied().collect(); + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone { + going.extend(e.popup); + } + } + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + if e.parent.is_some_and(|p| removed.contains(&p)) { + match e.r { + Some(r) => { + going.insert(r); + } + None => inline_gone[i] = true, + } + } + } + // Pop-ups dropped with their parent count too: a reply on another page may answer one. + removed_everywhere.extend(going.iter().copied()); let kept: Vec = annots .into_iter() - .filter(|a| { - let Some(r) = a.as_ref() else { return true }; - if removed.contains(&r) { - return false; - } - // Pop-ups of removed annotations go too. - let obj = doc.get(r); - !obj.as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) + .enumerate() + .filter(|(i, a)| match a.as_ref() { + Some(r) => !going.contains(&r), + None => !inline_gone[*i], }) + .map(|(_, a)| a) .collect(); doc.update_dict(page.obj, |d| { if kept.is_empty() { diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index e8ff3cd788..4ad4103e10 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6691,6 +6691,78 @@ fn a_type3_text_layer_is_redacted_like_any_other() { assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 0, "nothing is left under the mark"); } +#[test] +fn a_mark_whose_subtype_is_an_indirect_name_is_applied() { + // `/Subtype` may be an indirect reference (ISO 32000-2 §7.3.10); the audit reads it that way. + // Passing over the mark would leave the text under it while the others report success. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [7 0 R] >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 20 Td (SECRET) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Annot /Subtype 8 0 R /Rect [10 10 90 40] >>".to_vec(), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + assert_eq!(marks(&doc).len(), 1, "the mark is found"); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.marks, 1); + assert!(r.glyphs >= 6, "the text under the mark went: {r:?}"); + assert!(marks(&doc).is_empty(), "the mark itself went"); +} + +#[test] +fn a_mark_written_in_place_with_an_indirect_subtype_is_refused() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype 5 0 R /Rect [10 10 70 40] >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[100.0, 100.0, 120.0, 120.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::UnreadableMark(1)))); +} + +#[test] +fn annotations_written_in_place_go_when_covered_or_hanging_off_a_removed_one() { + // `/Annots` may hold dictionaries rather than references. One under the mark, a reply to a + // removed highlight on the same page and on another, and a reply to that inline comment's + // pop-up must all go; an inline comment elsewhere stays. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [6 0 R \ + << /Type /Annot /Subtype /Text /Rect [30 15 40 25] /Contents (INLINE-UNDER) /Popup 7 0 R >> \ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [250 100 270 120] /Contents (INLINE-KEPT) >> 7 0 R] >>" + .to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [\ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (OTHER-PAGE-INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [200 200 220 220] /IRT 7 0 R /Contents (POPUP-INLINE-REPLY) >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Annot /Subtype /Highlight /Rect [20 20 60 30] /QuadPoints [20 30 60 30 20 20 60 20] /Contents (HIGHLIGHT-NOTE) >>".to_vec(), + b"<< /Type /Annot /Subtype /Popup /Rect [200 150 290 190] /Contents (INLINE-POPUP) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.annotations, 5, "the highlight, the inline comment and the three replies: {r:?}"); + let pages = pdfcraft_model::pages(&doc); + assert_eq!(annots_of(&doc, &pages[0].dict).len(), 1, "only the inline comment elsewhere stays"); + assert!(annots_of(&doc, &pages[1].dict).is_empty(), "the replies leave page 2"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + for gone in [&b"HIGHLIGHT-NOTE"[..], b"INLINE-UNDER", b"INLINE-REPLY", b"OTHER-PAGE-INLINE-REPLY", b"POPUP-INLINE-REPLY", b"INLINE-POPUP"] { + assert!(!bytes.windows(gone.len()).any(|w| w == gone), "{} survived the save", String::from_utf8_lossy(gone)); + } + assert!(bytes.windows(11).any(|w| w == b"INLINE-KEPT")); +} + #[test] fn a_rewritten_form_xobject_drops_the_source_text_of_added_content() { // A form drawing invisible text whose dictionary repeats it in /PCAdded. @@ -6717,6 +6789,49 @@ fn a_redacted_form_xobject_drops_the_source_text_of_added_content() { assert!(!bytes.windows(4).any(|w| w == b"1234"), "redacted text left in the saved file"); } +#[test] +fn an_inline_widget_takes_its_field_even_with_no_referenced_widget() { + // The field's only widget is written in place in `/Annots`, so no referenced widget names it: + // the field (and its value) must still go with the widget under the mark. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 6 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (FIELD-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.fields, 1, "{r:?}"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(12).any(|w| w == b"FIELD-SECRET"), "the field value survived the save"); +} + +#[test] +fn an_inline_widget_of_the_field_on_another_page_goes_with_it() { + // Two pages: the widget under the mark, and a sibling widget in place on the second page with the + // same `/Parent`. Deleting the field must not leave the sibling naming it. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [7 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 7 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [200 200 260 230] /Parent 7 0 R >>] >>" + .to_vec(), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (SIBLING-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + apply(&mut doc, None).unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(14).any(|w| w == b"SIBLING-SECRET"), "the field value survived through the sibling widget"); +} + /// Private data takes an image's prepress data too: its OPI link to the original file and its /// alternate images, which can be the unedited original. #[test] From 0b0519f5df2056b9a9430d72021b88bd7b779cd6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 10:57:05 -0500 Subject: [PATCH 18/46] fix(forms): deleting a field removes ancestors it leaves without kids (#6997) * forms: deleting a field removes ancestors it leaves without kids * forms test: full-save check, widget only in the sibling case --- crates/forms/src/author/lifecycle.rs | 18 +++++++++++++++- crates/forms/src/tests.rs | 32 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/forms/src/author/lifecycle.rs b/crates/forms/src/author/lifecycle.rs index 50d552ccdb..28b56749d9 100644 --- a/crates/forms/src/author/lifecycle.rs +++ b/crates/forms/src/author/lifecycle.rs @@ -262,10 +262,26 @@ pub fn delete_widget(doc: &mut Document, name: &str, widget: usize) -> Result<() fn delete_found_field(doc: &mut Document, f: Field) -> Result<(), FormError> { let widgets: std::collections::HashSet = f.widgets.iter().map(|w| w.obj).collect(); drop_from_pages(doc, &widgets)?; + let mut parent = doc.get(f.obj).as_dict().and_then(|d| d.reference(b"Parent")); detach(doc, f.obj)?; // Out of the calculation order too: a `/CO` entry would keep the field, its value and its // widgets reachable for a full save. - drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r)) + drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r))?; + // An ancestor left with no kids would become a field of its own, and keep the `/V` and `/DV` + // the deleted field inherited from it: it goes too, up the chain. + let mut gone = std::collections::HashSet::new(); + while let Some(p) = parent.filter(|p| gone.len() < 64 && !gone.contains(p)) { + let Some(d) = doc.get(p).as_dict().cloned() else { break }; + let empty = d.get(b"Kids").map(|o| doc.resolve(o)).and_then(|o| o.as_array().map(Vec::is_empty)) == Some(true); + if !empty { + break; + } + parent = d.reference(b"Parent"); + gone.insert(p); + drop_from_pages(doc, &std::iter::once(p).collect())?; + detach(doc, p)?; + } + drop_from_calculation_order(doc, |r| gone.contains(&r)) } /// Take the annotations `widgets` off every page's `/Annots`, held directly or by reference. diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index caeba9491d..75dd075889 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -1664,6 +1664,38 @@ fn properties_rename_and_delete() { assert_eq!(annots.as_array().unwrap().len(), 13 - 3, "the deleted widgets left the page"); } +/// A deleted field's ancestor left with no kids stays in `/Fields` as a field of its own, with the +/// `/V` the deleted field inherited from it: it leaves too. An ancestor a sibling still has stays. +#[test] +fn deleting_the_last_kid_removes_the_emptied_ancestors_and_their_values() { + let form = |kids: &str, annots: &str| { + document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{annots}] >>"), + "<< /Fields [5 0 R] >>".into(), + "<< /FT /Tx /T (outer) /V (SECRET) /DV (SECRET) /Kids [6 0 R] >>".into(), + format!("<< /T (inner) /Parent 5 0 R /Kids [{kids}] >>"), + "<< /Type /Annot /Subtype /Widget /T (a) /Parent 6 0 R /Rect [50 700 250 720] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /T (b) /Parent 6 0 R /Rect [50 650 250 670] /P 3 0 R >>".into(), + ]) + }; + let top = |doc: &Document| { + let af = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"AcroForm").map(|o| doc.resolve(o)).unwrap().as_dict().unwrap().clone(); + doc.resolve(af.get(b"Fields").unwrap()).as_array().unwrap().len() + }; + let mut doc = form("7 0 R", "7 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 0, "the emptied ancestors left /Fields"); + let saved = pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(6).any(|w| w == b"SECRET"), "the inherited value is gone from a full save"); + + let mut doc = form("7 0 R 8 0 R", "7 0 R 8 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 1, "an ancestor with a surviving kid stays"); + assert!(fields(&doc).iter().any(|f| f.name == "outer.inner.b")); +} + /// A parent `MASTER` with kids `a` and `b` (each a field merged with its widget) and `e` (a field /// with two widget kids), and the top-level fields `c` and `d`. fn dotted_form() -> Document { From f44ea8380d996351069835c09e86dd8cd3a8a708 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 10:57:36 -0500 Subject: [PATCH 19/46] redact: a covered clip-and-paint path leaves an empty clip, not its outline (#6996) --- crates/redact/src/interp.rs | 9 ++++++--- crates/redact/src/tests.rs | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index a12fa1c041..80117f451d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -648,9 +648,12 @@ impl Walk { piece.changed = true; scope.report.paths_removed += 1; if scope.covered(b) && clips { - // Keep the clipping, drop the painting. - piece.ops.extend(body); - piece.ops.push(Op::new("n", vec![])); + // Everything this clip lets through is under a mark, so an empty clip hides the same + // content. The original outline is confidential and must not stay in the stream. + for o in ["re", "W", "n"] { + let operands = if o == "re" { vec![num(0.0); 4] } else { vec![] }; + piece.ops.push(Op::new(o, operands)); + } } // Partial vector redaction must not retain underlying drawing commands // behind clipping (F-008). Refuse partial clip retention and remove the path. diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 4ad4103e10..dc94a2c5f5 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -1057,6 +1057,20 @@ fn partially_covered_vector_paths_do_not_leak_drawing_commands() { assert!(c.contains("300 300 m\n350 350 l\nS"), "unaffected vector path must be kept: {c}"); } +#[test] +fn a_covered_clip_and_paint_path_leaves_an_empty_clip_not_its_outline() { + // The outline of a fully covered path that clips and paints is confidential geometry: only + // an empty clip may stay, and what follows is hidden as the covered clip would have hidden it. + let src = b"q 100 100 m 150 180 l 200 100 l h W f 300 300 50 50 re f Q"; + let mut doc = one_page(src, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 210.0, 190.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("150 180 l") && !c.contains("100 100 m"), "outline must not stay: {c}"); + assert!(c.contains("0 0 0 0 re\nW\nn"), "an empty clip stays: {c}"); +} + /// A chain of nested forms deeper than the pass can walk was removed from the page, taking /// whatever else it drew with it even when nothing in it was marked. The limit now reaches as /// deep as the renderer and Edit text read (50), so anything a reader can see can be walked; From 3650f01f233916fc9d14d94637c6a9ce8e228f03 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:58:30 +0100 Subject: [PATCH 20/46] Release: PdfCraft v0.21.0 (#6995) --- Cargo.lock | 78 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 72 ++++++++++++++++++++++++------------------------- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cf008f02e..c4e08c1086 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4263,7 +4263,7 @@ dependencies = [ [[package]] name = "pdfcraft" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "craft-update", @@ -4288,7 +4288,7 @@ dependencies = [ [[package]] name = "pdfcraft-a11y" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4299,7 +4299,7 @@ dependencies = [ [[package]] name = "pdfcraft-annot" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "pdfcraft-audit" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4322,7 +4322,7 @@ dependencies = [ [[package]] name = "pdfcraft-automation" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "flate2", @@ -4346,7 +4346,7 @@ dependencies = [ [[package]] name = "pdfcraft-cli" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "pdfcraft-audit", @@ -4364,11 +4364,11 @@ dependencies = [ [[package]] name = "pdfcraft-compare" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-content" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4376,7 +4376,7 @@ dependencies = [ [[package]] name = "pdfcraft-cos" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "hayro-syntax", @@ -4393,7 +4393,7 @@ dependencies = [ [[package]] name = "pdfcraft-create" -version = "0.18.1" +version = "0.21.0" dependencies = [ "encoding_rs", "hayro-ccitt", @@ -4413,7 +4413,7 @@ dependencies = [ [[package]] name = "pdfcraft-crypt" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aes", "md-5", @@ -4426,7 +4426,7 @@ dependencies = [ [[package]] name = "pdfcraft-edit" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "icu_segmenter", @@ -4445,7 +4445,7 @@ dependencies = [ [[package]] name = "pdfcraft-engine" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "craft-docpath", @@ -4489,14 +4489,14 @@ dependencies = [ [[package]] name = "pdfcraft-export" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", ] [[package]] name = "pdfcraft-filters" -version = "0.18.1" +version = "0.21.0" dependencies = [ "fax", "flate2", @@ -4509,7 +4509,7 @@ dependencies = [ [[package]] name = "pdfcraft-fonts" -version = "0.18.1" +version = "0.21.0" dependencies = [ "harfrust", "pdfcraft-cos", @@ -4519,7 +4519,7 @@ dependencies = [ [[package]] name = "pdfcraft-forms" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4532,11 +4532,11 @@ dependencies = [ [[package]] name = "pdfcraft-geom" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-js" -version = "0.18.1" +version = "0.21.0" dependencies = [ "boa_engine", "chrono", @@ -4544,7 +4544,7 @@ dependencies = [ [[package]] name = "pdfcraft-measure" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4557,14 +4557,14 @@ dependencies = [ [[package]] name = "pdfcraft-model" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-cos", ] [[package]] name = "pdfcraft-ocr" -version = "0.18.1" +version = "0.21.0" dependencies = [ "ocrs", "pdfcraft-fonts", @@ -4579,7 +4579,7 @@ dependencies = [ [[package]] name = "pdfcraft-ocr-worker" -version = "0.18.1" +version = "0.21.0" dependencies = [ "js-sys", "pdfcraft-ocr", @@ -4589,7 +4589,7 @@ dependencies = [ [[package]] name = "pdfcraft-optimize" -version = "0.18.1" +version = "0.21.0" dependencies = [ "image", "pdfcraft-annot", @@ -4606,11 +4606,11 @@ dependencies = [ [[package]] name = "pdfcraft-oracle" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-organize" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4621,7 +4621,7 @@ dependencies = [ [[package]] name = "pdfcraft-platform" -version = "0.18.1" +version = "0.21.0" dependencies = [ "async-channel", "futures-lite", @@ -4632,7 +4632,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "proptest", @@ -4641,7 +4641,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc-render" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "pdfcraft-prc", @@ -4650,7 +4650,7 @@ dependencies = [ [[package]] name = "pdfcraft-preflight" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "pdfcraft-content", @@ -4661,7 +4661,7 @@ dependencies = [ [[package]] name = "pdfcraft-print" -version = "0.18.1" +version = "0.21.0" dependencies = [ "craft-print", "log", @@ -4676,7 +4676,7 @@ dependencies = [ [[package]] name = "pdfcraft-redact" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aho-corasick", "pdfcraft-annot", @@ -4700,7 +4700,7 @@ dependencies = [ [[package]] name = "pdfcraft-render" -version = "0.18.1" +version = "0.21.0" dependencies = [ "bytemuck", "flate2", @@ -4724,7 +4724,7 @@ dependencies = [ [[package]] name = "pdfcraft-sign" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aes", "aws-lc-rs", @@ -4763,11 +4763,11 @@ dependencies = [ [[package]] name = "pdfcraft-testkit" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-ui-egui" -version = "0.18.1" +version = "0.21.0" dependencies = [ "arboard", "base64 0.23.1", @@ -4826,7 +4826,7 @@ dependencies = [ [[package]] name = "pdfcraft-web" -version = "0.18.1" +version = "0.21.0" dependencies = [ "craft-web-api", "craft-web-host", @@ -4841,7 +4841,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfa" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "image", @@ -4855,7 +4855,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfdf" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "pdfcraft-annot", diff --git a/Cargo.toml b/Cargo.toml index d566a69d1f..eaf28264e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor"] default-members = ["crates/*", "apps/pdfcraft", "apps/pdfcraft-cli", "xtask"] [workspace.package] -version = "0.18.1" +version = "0.21.0" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -32,41 +32,41 @@ craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-l # craft-update: app-specific pretend versions apply to startup and manual checks. craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } # internal -pdfcraft-geom = { path = "crates/geom", version = "0.18.1" } -pdfcraft-filters = { path = "crates/filters", version = "0.18.1" } -pdfcraft-cos = { path = "crates/cos", version = "0.18.1" } -pdfcraft-crypt = { path = "crates/crypt", version = "0.18.1" } -pdfcraft-organize = { path = "crates/organize", version = "0.18.1" } -pdfcraft-measure = { path = "crates/measure", version = "0.18.1" } -pdfcraft-annot = { path = "crates/annot", version = "0.18.1" } -pdfcraft-audit = { path = "crates/audit", version = "0.18.1" } -pdfcraft-fonts = { path = "crates/fonts", version = "0.18.1" } -pdfcraft-model = { path = "crates/model", version = "0.18.1" } -pdfcraft-content = { path = "crates/content", version = "0.18.1" } -pdfcraft-redact = { path = "crates/redact", version = "0.18.1" } -pdfcraft-print = { path = "crates/print", version = "0.18.1" } -pdfcraft-xfdf = { path = "crates/xfdf", version = "0.18.1" } -pdfcraft-sign = { path = "crates/sign", version = "0.18.1" } -pdfcraft-optimize = { path = "crates/optimize", version = "0.18.1" } -pdfcraft-a11y = { path = "crates/a11y", version = "0.18.1" } -pdfcraft-oracle = { path = "crates/oracle", version = "0.18.1" } -pdfcraft-testkit = { path = "crates/testkit", version = "0.18.1" } -pdfcraft-ocr = { path = "crates/ocr", version = "0.18.1" } -pdfcraft-js = { path = "crates/js", version = "0.18.1" } -pdfcraft-compare = { path = "crates/compare", version = "0.18.1" } -pdfcraft-preflight = { path = "crates/preflight", version = "0.18.1" } -pdfcraft-export = { path = "crates/export", version = "0.18.1" } -pdfcraft-edit = { path = "crates/edit", version = "0.18.1" } -pdfcraft-create = { path = "crates/create", version = "0.18.1" } -pdfcraft-forms = { path = "crates/forms", version = "0.18.1" } -pdfcraft-xfa = { path = "crates/xfa", version = "0.18.1" } -pdfcraft-render = { path = "crates/render", version = "0.18.1" } -pdfcraft-engine = { path = "crates/engine", version = "0.18.1" } -pdfcraft-platform = { path = "crates/platform", version = "0.18.1" } -pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.18.1" } -pdfcraft-automation = { path = "crates/automation", version = "0.18.1", default-features = false } -pdfcraft-prc = { path = "crates/prc", version = "0.18.1" } -pdfcraft-prc-render = { path = "crates/prc-render", version = "0.18.1" } +pdfcraft-geom = { path = "crates/geom", version = "0.21.0" } +pdfcraft-filters = { path = "crates/filters", version = "0.21.0" } +pdfcraft-cos = { path = "crates/cos", version = "0.21.0" } +pdfcraft-crypt = { path = "crates/crypt", version = "0.21.0" } +pdfcraft-organize = { path = "crates/organize", version = "0.21.0" } +pdfcraft-measure = { path = "crates/measure", version = "0.21.0" } +pdfcraft-annot = { path = "crates/annot", version = "0.21.0" } +pdfcraft-audit = { path = "crates/audit", version = "0.21.0" } +pdfcraft-fonts = { path = "crates/fonts", version = "0.21.0" } +pdfcraft-model = { path = "crates/model", version = "0.21.0" } +pdfcraft-content = { path = "crates/content", version = "0.21.0" } +pdfcraft-redact = { path = "crates/redact", version = "0.21.0" } +pdfcraft-print = { path = "crates/print", version = "0.21.0" } +pdfcraft-xfdf = { path = "crates/xfdf", version = "0.21.0" } +pdfcraft-sign = { path = "crates/sign", version = "0.21.0" } +pdfcraft-optimize = { path = "crates/optimize", version = "0.21.0" } +pdfcraft-a11y = { path = "crates/a11y", version = "0.21.0" } +pdfcraft-oracle = { path = "crates/oracle", version = "0.21.0" } +pdfcraft-testkit = { path = "crates/testkit", version = "0.21.0" } +pdfcraft-ocr = { path = "crates/ocr", version = "0.21.0" } +pdfcraft-js = { path = "crates/js", version = "0.21.0" } +pdfcraft-compare = { path = "crates/compare", version = "0.21.0" } +pdfcraft-preflight = { path = "crates/preflight", version = "0.21.0" } +pdfcraft-export = { path = "crates/export", version = "0.21.0" } +pdfcraft-edit = { path = "crates/edit", version = "0.21.0" } +pdfcraft-create = { path = "crates/create", version = "0.21.0" } +pdfcraft-forms = { path = "crates/forms", version = "0.21.0" } +pdfcraft-xfa = { path = "crates/xfa", version = "0.21.0" } +pdfcraft-render = { path = "crates/render", version = "0.21.0" } +pdfcraft-engine = { path = "crates/engine", version = "0.21.0" } +pdfcraft-platform = { path = "crates/platform", version = "0.21.0" } +pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.21.0" } +pdfcraft-automation = { path = "crates/automation", version = "0.21.0", default-features = false } +pdfcraft-prc = { path = "crates/prc", version = "0.21.0" } +pdfcraft-prc-render = { path = "crates/prc-render", version = "0.21.0" } # pdf hayro = "0.7.1" From 619a239cebe25054ba59486ce2b878b66613c0ce Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 12:12:27 -0500 Subject: [PATCH 21/46] ci: run Harden-Runner on every job (#7003) Adds step-security/harden-runner (audit, pinned to v2.22.4 like craft-common-actions) as the first step of each job that did not have it, so every workflow in the org's public repos reports its egress to StepSecurity. Audit, not block: block-mode allow lists come from each job's observed traffic. --- .github/workflows/ci.yml | 4 ++++ .github/workflows/fuzz.yml | 4 ++++ .github/workflows/packaging-lint.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94a4c545fe..38ee53caf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,10 @@ jobs: permissions: contents: read steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index b7b1d409c8..fe0a0074d7 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -24,6 +24,10 @@ jobs: name: fuzz (mutation, 15 min) runs-on: ubuntu-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index ea61d84303..1771a35fea 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -27,6 +27,10 @@ jobs: name: Windows installation policy (no app build) runs-on: windows-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From d4608308591dc566dd31bd1486801cc48d14209f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 12:44:55 -0500 Subject: [PATCH 22/46] Updates: take craft-update from craft-libs main at 6e03f65bf --- Cargo.lock | 3 ++- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cf008f02e..5d4d1fb2b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1289,7 +1289,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "chrono", "dunce", @@ -1299,6 +1299,7 @@ dependencies = [ "image", "log", "plist", + "rustix 1.1.6", "semver", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index d566a69d1f..d850df512f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1. # Whether a path stored in a document may be opened (GoToR link targets). craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", features = ["egui"] } # internal pdfcraft-geom = { path = "crates/geom", version = "0.18.1" } pdfcraft-filters = { path = "crates/filters", version = "0.18.1" } From 583ae295993ab0c944ebb87d50f7db6acb289d1c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:46:34 -0500 Subject: [PATCH 23/46] fix(sign): a widget removed from a page's inline Annots is not a comment change (#6013) --- crates/sign/tests/it/pdf.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index e765941c9a..fa45ab1ecb 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -325,6 +325,21 @@ fn an_indirect_annots_array_is_judged_by_the_annotations_it_gains() { assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); } +/// A signature widget taken off the page's inline /Annots array is covered by no permission (the +/// indirect-array case is the test above), and stays a change whatever level the signature has. +#[test] +fn removing_a_widget_from_an_inline_annots_array_is_not_a_comment_change() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let page = pdfcraft_cos::ObjRef { num: 3, generation: 0 }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &opts()).unwrap(); + let inline = open(&signed).get(page).as_dict().and_then(|d| d.get(b"Annots").cloned()).unwrap(); + assert!(matches!(inline, Object::Array(ref a) if !a.is_empty()), "the page keeps its widgets inline"); + let stripped = edit_after(&signed, |doc| doc.update_dict(page, |p| p.set(b"Annots".to_vec(), Object::Array(vec![]))).unwrap()); + let s = signatures(&open(&stripped), &stripped, &trust).into_iter().find(|s| s.signed).unwrap(); + assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); +} + /// Issue #1328: a signature field whose /V is null, directly or through a reference, is unsigned: /// it is listed as unsigned and can be signed by name. #[test] From d46d9daea5b0b3bb316bb5889b1757df5e2512ab Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:46:47 -0500 Subject: [PATCH 24/46] fix(edit): editor font enumeration reads a face name without building font metrics (#6086) --- crates/edit/src/text/editor.rs | 4 ++-- crates/fonts/src/pdf.rs | 25 +++++++++++++++---------- crates/fonts/src/pdf/tests.rs | 10 ++++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/crates/edit/src/text/editor.rs b/crates/edit/src/text/editor.rs index 4f4abf3a43..e6df1dbfee 100644 --- a/crates/edit/src/text/editor.rs +++ b/crates/edit/src/text/editor.rs @@ -2,7 +2,7 @@ use super::blocks::{BlockStyle, TextBlock}; use super::fallback::MAX_TYPE3_GLYPHS; use super::lines::{page_dict, text_lines}; use super::rewrite::rewrite_block_from; -use super::{DecodeBudget, Dict, Document, EditError, Metrics, Object, TextLine, TextSession}; +use super::{DecodeBudget, Dict, Document, EditError, Object, TextLine, TextSession}; /// An embedded source font or an approved bundled document-font choice. /// Source bytes stay in memory and are never installed or copied into another document. @@ -45,7 +45,7 @@ pub fn text_editor_fonts(doc: &Document, page: usize) -> Result Option { } } +/// The font's `/BaseFont` (a Type 3 font's descriptor `/FontName` when it has none), read without +/// building its metrics, so it decodes no CMap. +pub fn base_font_name(doc: &Document, font: &Dict) -> String { + font.name(b"BaseFont") + .map(|name| String::from_utf8_lossy(name).into_owned()) + .or_else(|| { + (font.name(b"Subtype") == Some(b"Type3")) + .then(|| dict(doc, font.get(b"FontDescriptor"))) + .flatten() + .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) + }) + .unwrap_or_default() +} + /// Codespace ranges and CID mappings from an embedded CMap stream, at most /// [`MAX_CMAP_RANGES`] of each (every glyph searches them). #[allow(clippy::type_complexity)] @@ -909,16 +923,7 @@ impl Metrics { pub fn from_dict_within(doc: &Document, font: &Dict, budget: &mut DecodeBudget) -> Self { let mut reads = Reads::new(doc, budget); let mut m = Self::read_metrics(doc, font, &mut reads); - let base = font - .name(b"BaseFont") - .map(|name| String::from_utf8_lossy(name).into_owned()) - .or_else(|| { - (font.name(b"Subtype") == Some(b"Type3")) - .then(|| dict(doc, font.get(b"FontDescriptor"))) - .flatten() - .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) - }) - .unwrap_or_default(); + let base = base_font_name(doc, font); m.subset = base.len() > 7 && base.as_bytes()[6] == b'+' && base[..6].bytes().all(|b| b.is_ascii_uppercase()); let (bold, italic) = style_from_name(&base); m.bold |= bold; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 4c5f90044f..74880c94a4 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -1823,6 +1823,16 @@ fn low_codes_resolve_to_the_covering_range() { assert_eq!((u.covering(0xFFFF).is_some(), u.covering(0x1_0001).is_some(), u.covering(0x1_0002).is_none()), (true, true, true)); } +#[test] +fn base_font_name_matches_the_metrics_name() { + let doc = Document::new_empty(); + let mut f = Dict::new(); + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+Body")); + assert_eq!(base_font_name(&doc, &f), "ABCDEF+Body"); + assert_eq!(base_font_name(&doc, &f), Metrics::from_dict(&doc, &f).base_font); + assert_eq!(base_font_name(&doc, &Dict::new()), ""); +} + #[test] fn refused_subset_reads_do_not_encode_unverified_glyphs() { let mut doc = Document::new_empty(); From 683ccc9fcca358568605e7226ae65a9447135fce Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:00 -0500 Subject: [PATCH 25/46] fix(optimize): walk shared resource dictionaries and name-tree kids once (#5956) * fix(optimize): walk shared resource dictionaries and name-tree kids once * test(optimize): restore the file around the appended tests * address review: import ObjRef in the optimize audit module --- crates/optimize/src/audit.rs | 31 ++++++++++----- crates/optimize/src/links.rs | 8 +++- crates/optimize/tests/optimize.rs | 64 +++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/crates/optimize/src/audit.rs b/crates/optimize/src/audit.rs index 789bf2e89d..b06d76d6c7 100644 --- a/crates/optimize/src/audit.rs +++ b/crates/optimize/src/audit.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; -use pdfcraft_cos::{Dict, Document, MAX_DECODED, Object, XrefEntry}; +use pdfcraft_cos::{Dict, Document, MAX_DECODED, ObjRef, Object, XrefEntry}; /// The categories, in Acrobat's order. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -91,6 +91,9 @@ const MAX_FORM_DEPTH: usize = 64; struct Audit<'a> { doc: &'a Document, owner: HashMap, + /// Resource dictionaries (by reference) already walked: one shared by many pages, forms or + /// appearances is walked once, as claiming is the same each time. + walked: HashSet, } impl Audit<'_> { @@ -124,6 +127,18 @@ impl Audit<'_> { o.map(|x| self.doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default() } + /// The resource dictionary `holder` points at (if it does), by kind: [`Self::resources`], + /// once per dictionary given by reference. + fn resources_of(&mut self, holder: Option<&Object>, depth: usize) { + if let Some(Object::Ref(r)) = holder + && !self.walked.insert(*r) + { + return; + } + let res = self.resolve_dict(holder); + self.resources(&res, depth); + } + /// A resource dictionary, by kind; `depth` counts the form XObjects it is nested in. fn resources(&mut self, res: &Dict, depth: usize) { let xobjects = self.resolve_dict(res.get(b"XObject")); @@ -135,8 +150,7 @@ impl Audit<'_> { // A form: its own resources first, by kind, then the rest of it. Owning it // first stops a form that draws itself. self.owner.insert(r.num, SpaceCategory::XObjectForms); - let inner = self.resolve_dict(s.dict.get(b"Resources")); - self.resources(&inner, depth + 1); + self.resources_of(s.dict.get(b"Resources"), depth + 1); self.claim(&Object::Dict(s.dict.clone()), SpaceCategory::XObjectForms); } _ => self.claim(x, SpaceCategory::XObjectForms), @@ -172,8 +186,7 @@ impl Audit<'_> { _ => Vec::new(), }; for dict in streams { - let res = self.resolve_dict(dict.get(b"Resources")); - self.resources(&res, 1); + self.resources_of(dict.get(b"Resources"), 1); } } } @@ -216,19 +229,17 @@ fn object_stream(doc: &Document, num: u32) -> Option { /// Audit the space a document uses; `file_len` is the size of the file on disk (the rest is /// overhead). pub fn audit_space(doc: &Document, file_len: u64) -> Vec { - let mut a = Audit { doc, owner: HashMap::new() }; + let mut a = Audit { doc, owner: HashMap::new(), walked: HashSet::new() }; let cat = doc.root().and_then(|r| doc.get(r).as_dict().cloned()).unwrap_or_default(); let pages = pdfcraft_model::pages(doc); // Order matters: the most specific owners first. Resources by kind before anything that // uses them: a content stream's dictionary may point at what it draws (PdfCraft's /PCAdded // records an added image), and form fields and comments draw with fonts and images too. for p in &pages { - let res = a.resolve_dict(p.dict.get(b"Resources")); - a.resources(&res, 0); + a.resources_of(p.dict.get(b"Resources"), 0); } let form = a.resolve_dict(cat.get(b"AcroForm")); - let form_resources = a.resolve_dict(form.get(b"DR")); - a.resources(&form_resources, 0); + a.resources_of(form.get(b"DR"), 0); for p in &pages { for x in annotations(doc, &p.dict) { a.appearance_resources(&x); diff --git a/crates/optimize/src/links.rs b/crates/optimize/src/links.rs index 5b74ad7023..bef69c43b7 100644 --- a/crates/optimize/src/links.rs +++ b/crates/optimize/src/links.rs @@ -52,8 +52,12 @@ fn tree_entries(doc: &Document, node: &Object) -> Vec<(Vec, Object)> { } } } - if let Some(kids) = d.get(b"Kids").map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) { - queue.extend(kids.into_iter().map(|k| (k, depth + 1))); + // A kids array shared by reference is read once, and only kids not yet seen are queued: + // nodes sharing one array can't multiply the queue. + if let Some(kids) = + d.get(b"Kids").filter(|k| !matches!(k, Object::Ref(r) if !seen.insert(*r))).map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) + { + queue.extend(kids.into_iter().filter(|k| !matches!(k, Object::Ref(r) if seen.contains(r))).map(|k| (k, depth + 1))); } } out diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7da6151deb..eee2e9570c 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -5613,6 +5613,70 @@ fn a_picture_whose_mask_is_not_a_stencil_is_left_alone() { } } +/// A resource dictionary shared by reference between thousands of pages is walked once by the +/// space audit, not once per page. +#[test] +fn space_audit_walks_a_shared_resource_dictionary_once() { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 4, 4, 3, false, None); + let mut x = Dict::new(); + for i in 0..5_000 { + x.set(format!("Im{i}").into_bytes(), Object::Ref(img)); + } + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + let shared = doc.add(Object::Dict(res)); + let pages = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|d| d.reference(b"Pages"))).unwrap(); + let mut kids = Vec::new(); + for _ in 0..2_000 { + let mut p = Dict::new(); + p.set(b"Type".to_vec(), Object::name("Page")); + p.set(b"Parent".to_vec(), Object::Ref(pages)); + p.set(b"MediaBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 612.into(), 792.into()])); + p.set(b"Resources".to_vec(), Object::Ref(shared)); + kids.push(Object::Ref(doc.add(Object::Dict(p)))); + } + doc.update_dict(pages, |d| { + d.set(b"Count".to_vec(), Object::Int(kids.len() as i64)); + d.set(b"Kids".to_vec(), Object::Array(kids)); + }) + .unwrap(); + let started = std::time::Instant::now(); + let audit = audit_space(&doc, 1 << 20); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + assert!(audit.iter().any(|u| u.category == SpaceCategory::Images && u.bytes > 0), "{audit:?}"); +} + +/// A name tree whose nodes all share one kids array (by reference) is read once per node, not +/// once per node per kid: the queue can't grow to the nodes times the kids. +#[test] +fn a_name_tree_sharing_one_kids_array_is_walked_once() { + let mut doc = Document::new_empty(); + let p = page(&mut doc, &[], "BT ET"); + let s = |t: &str| Object::String(pdfcraft_cos::PdfString::literal(t.as_bytes().to_vec())); + let fit = Object::Array(vec![Object::Ref(p), Object::name("Fit")]); + let kids = doc.add(Object::Array(Vec::new())); + let mut nodes = Vec::new(); + for i in 0..3_000 { + let mut node = Dict::new(); + node.set(b"Kids".to_vec(), Object::Ref(kids)); + node.set(b"Names".to_vec(), Object::Array(vec![s(&format!("d{i}")), fit.clone()])); + nodes.push(Object::Ref(doc.add(Object::Dict(node)))); + } + doc.set(kids, Object::Array(nodes)); + let mut tree = Dict::new(); + tree.set(b"Kids".to_vec(), Object::Ref(kids)); + let tree = doc.add(Object::Dict(tree)); + let mut names = Dict::new(); + names.set(b"Dests".to_vec(), Object::Ref(tree)); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Names".to_vec(), Object::Dict(names))).unwrap(); + let started = std::time::Instant::now(); + let report = optimize(&mut doc, &Settings::default()).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed()); + assert_eq!(report.unreferenced_dests, 3_000, "every node's destination was found, once"); +} + /// Pictures run on several threads: what comes out doesn't depend on how many. #[test] fn the_same_file_comes_out_with_any_number_of_threads() { From fba22fc73840986b7e7307d4844680868b29d667 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:15 -0500 Subject: [PATCH 26/46] fix(optimize): the fragment search charges every operator of a group it reads (#6095) * fix(optimize): the fragment search charges every operator of a group it reads * address review: conflict markers removed; the fragment search charges every operator it reads to the production allowance * address review: group reads are charged before a parse failure and count all four operators --- crates/optimize/src/fragments.rs | 98 +++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index aa52c20750..9ccca4791f 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -30,17 +30,21 @@ pub const MIN_FRAGMENTS: usize = 4; /// Most pixels a merged image may have (memory). const MAX_PIXELS: u64 = 80_000_000; -/// Fragment groups a page's search for runs may read per content operator, and at least. A run -/// that is not one picture is searched again from each of its groups, so without a limit images -/// drawn over one another take time quadratic in their number; past it, the rest of the page is -/// left as it is. -const SCAN_PER_OP: usize = 16; +/// Content operators a page's search for runs may read per content operator of the page, and at +/// least. A run that is not one picture is searched again from each of its groups, so without a +/// limit images drawn over one another take time quadratic in their number; past it, the rest of +/// the page is left as it is. Every operator of a group counts (its `cm` operators are composed +/// each time), not just the group; a group of four operators costs four, so the allowance is 16 +/// groups per operator, as before, and groups with many `cm` operators pay for them. +const SCAN_PER_OP: usize = 64; const MIN_SCAN: usize = 1 << 16; #[cfg(test)] thread_local! { /// Fragment groups read, for the test that keeps a page's search linear in its content. static GROUP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Operators group reads walked, for the test that keeps the search linear in operators. + static OP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } /// One fragment: its image, its box in the coordinates the run is drawn in, and its size. @@ -53,21 +57,44 @@ struct Fragment { /// The ops `q`, one or more `cm`, `Do`, `Q` starting at `i`: the index after them, the matrix /// and the name drawn. -fn group_at(ops: &[Op], i: usize) -> Option<(usize, Matrix, Vec)> { +fn group_at(ops: &[Op], i: usize, reads: &std::cell::Cell) -> Option<(usize, Matrix, Vec)> { #[cfg(test)] GROUP_READS.with(|c| c.set(c.get().saturating_add(1))); - if !ops.get(i)?.is("q") { + let mut looked = 0usize; + let found = group_scan(ops, i, &mut looked); + // Every operator looked at is charged to the page's search allowance, whether or not the + // group turns out to be one. + reads.set(reads.get().saturating_add(looked)); + #[cfg(test)] + OP_READS.with(|c| c.set(c.get().saturating_add(looked))); + found +} + +/// [`group_at`]'s search, counting in `looked` each operator it examines. +fn group_scan(ops: &[Op], i: usize, looked: &mut usize) -> Option<(usize, Matrix, Vec)> { + let q = ops.get(i)?; + *looked += 1; + if !q.is("q") { return None; } let mut j = i + 1; let mut m = Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]); while let Some(op) = ops.get(j).filter(|o| o.is("cm")) { + *looked += 1; m = Matrix::from_operands(&op.operands)?.then(&m); j += 1; } - let draw = ops.get(j).filter(|o| o.is("Do"))?; + let draw = ops.get(j)?; + *looked += 1; + if !draw.is("Do") { + return None; + } let name = draw.name(0)?.to_vec(); - ops.get(j + 1).filter(|o| o.is("Q"))?; + let end = ops.get(j + 1)?; + *looked += 1; + if !end.is("Q") { + return None; + } (j > i + 1).then_some((j + 2, m, name)) } @@ -152,7 +179,7 @@ fn stacked(run: &mut [Fragment]) -> Option>> { /// The fragment groups from `i` on (at most `limit`) while they form a picture drawn row by row /// or strip by strip in one direction, up to a gap or a jump. -fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document) -> Vec { +fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document, reads: &std::cell::Cell) -> Vec { let mut run: Vec = Vec::new(); let mut row_x = [0.0f64, 0.0]; let mut cur_x = [0.0f64, 0.0]; @@ -163,7 +190,7 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & let mut complete = 0usize; let mut j = i; while run.len() < limit { - let Some((next, m, name)) = group_at(ops, j) else { break }; + let Some((next, m, name)) = group_at(ops, j, reads) else { break }; let Some((image, width, height, ..)) = fragment_image(doc, xobjects, &name) else { break }; let [a, _, _, d, e, f] = m.0; let f2 = Fragment { image, rect: [e, f, e + a, f + d], width, height }; @@ -240,10 +267,10 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & } /// The op index after the first `count` fragment groups from `i`. -fn run_end(ops: &[Op], i: usize, count: usize) -> usize { +fn run_end(ops: &[Op], i: usize, count: usize, reads: &std::cell::Cell) -> usize { let mut j = i; for _ in 0..count { - match group_at(ops, j) { + match group_at(ops, j, reads) { Some((next, ..)) => j = next, None => break, } @@ -386,9 +413,9 @@ fn merge_page( let mut out: Vec = Vec::with_capacity(ops.len()); let (mut i, mut merged, mut k) = (0, 0, 0); let budget = ops.len().saturating_mul(SCAN_PER_OP).max(MIN_SCAN); - let mut scanned = 0usize; + let reads = std::cell::Cell::new(0usize); while i < ops.len() { - if scanned > budget { + if reads.get() > budget { out.extend(ops.get(i..).unwrap_or_default().iter().cloned()); break; } @@ -396,7 +423,7 @@ fn merge_page( let mut run = Vec::new(); let mut kind: Option<(Object, usize)> = None; let mut j = i; - while let Some((next, m, name)) = group_at(&ops, j) { + while let Some((next, m, name)) = group_at(&ops, j, &reads) { let Some((image, width, height, cs, n)) = fragment_image(doc, &xobjects, &name) else { break }; let [a, b, c, d, e, f] = m.0; if b.abs() > 1e-9 || c.abs() > 1e-9 || !(a > 0.0 && d > 0.0) { @@ -410,13 +437,11 @@ fn merge_page( run.push(Fragment { image, rect: [e, f, e + a, f + d], width, height }); j = next; } - scanned = scanned.saturating_add(run.len()).saturating_add(1); // Not one picture as a whole: a prefix drawn row by row or strip by strip may be. let mut rows = if run.len() >= MIN_FRAGMENTS { stacked(&mut run) } else { None }; if run.len() >= MIN_FRAGMENTS && rows.is_none() { - run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc); - j = run_end(&ops, i, run.len()); - scanned = scanned.saturating_add(run.len().saturating_mul(2)).saturating_add(2); + run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc, &reads); + j = run_end(&ops, i, run.len(), &reads); rows = stacked(&mut run); } let Some(((cs, n), rows)) = kind.zip(rows).filter(|_| run.len() >= MIN_FRAGMENTS) else { @@ -773,4 +798,37 @@ mod tests { assert_eq!((merged, report.merged_images), (0, 0)); } } + + /// The same with many `cm` operators in each group: the search reads a bounded number of operators + /// for each one of the page, however the matrices are spread over the groups. + #[test] + fn groups_with_many_matrices_are_charged_for_each_operator() { + let mut doc = Document::new_empty(); + let mut image = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(1)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + image.set(k.as_bytes().to_vec(), v); + } + image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let image = doc.add(Object::Stream(Stream::from_raw(image, vec![0]))); + let groups = 400; + let content = doc.add(Object::Stream(Stream::flate( + Dict::new(), + format!("q 10 0 0 10 0 0 cm {}/Im Do Q\n", "1 0 0 1 0 0 cm ".repeat(49)).repeat(groups).as_bytes(), + ))); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Contents".to_vec(), Object::Ref(content)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + let page = doc.add(Object::Dict(page)); + OP_READS.with(|c| c.set(0)); + let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true, &mut ImageBudget::with_work(u64::MAX)); + assert_eq!(merged.ok(), Some(0)); + let reads = OP_READS.with(|c| c.get()); + assert!(reads <= 80 * 53 * groups, "{reads} operator reads"); + } } From 87aebd1b24618fa822d994f32209bff08e974056 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:29 -0500 Subject: [PATCH 27/46] fix(preflight): follow every /Next action and bound colour and intent work (#5926) * fix(preflight): follow every /Next action and bound colour and intent work Action chains followed an indirect /Next only when it named an array, and skipped references inside a /Next array, so a forbidden action one indirect hop behind an allowed one survived both verification and conversion; an untyped dictionary with /P and /K was exempt from action checks whatever its /S. Both are now followed and checked, and arrays are borrowed instead of deep-copied per chain. Conversion treats a /Subtype /Widget without /Type as an annotation, and takes forbidden annotations off their pages in one pass. Device-colour inspection decoded each content stream with the 1 GiB per-stream default and no total; it now shares one 1 GiB DecodeBudget and reports unread content instead of treating it as clean. Type 3 CharProcs shared by many fonts are read once, and declared() reads each output intent once (at most 8). * address review: no sRGB intent when colour couldn't all be read; count array reads in action chains; declare every output intent * address review: cut only an over-long identifier; test with a filter that can't be read * review: leave untyped structure elements out of the forbidden-action check * merge main, keep both * address review: a wide direct /Next array is refused before it is queued; queued references are resolved through aliases --- crates/preflight/src/lib.rs | 190 ++++++++++++++++++++++++++-------- crates/preflight/src/tests.rs | 124 ++++++++++++++++++++++ 2 files changed, 269 insertions(+), 45 deletions(-) diff --git a/crates/preflight/src/lib.rs b/crates/preflight/src/lib.rs index 7b6e960006..90c6d9db83 100644 --- a/crates/preflight/src/lib.rs +++ b/crates/preflight/src/lib.rs @@ -203,6 +203,7 @@ pub const WORDINGS: &[&str] = &[ "A graphics state has the rendering intent {}, which isn't a standard one", "A graphics state has a rendering intent that isn't a name", "The form asks viewers to make appearances (NeedAppearances)", + "Some content could not be read for device colour", // Pages, actions, annotations and fields. "Page {} has additional actions", "A {} action isn't allowed", @@ -229,7 +230,8 @@ pub struct Declared { /// `pdfaid:part` and `pdfaid:conformance`, e.g. (2, "B"). pub pdfa: Option<(u8, String)>, pub pdfua: Option, - /// Output intents' condition identifiers. + /// Output intents' condition identifiers: every intent the document lists (each object + /// once), each identifier cut to [`MAX_IDENTIFIER_BYTES`]. pub output_intents: Vec, } @@ -276,10 +278,27 @@ pub fn declared(doc: &Document) -> Declared { d.pdfua = xmp::identifier(&x, xmp::PDFUA_ID, "part").and_then(|p| p.parse().ok()); } let cat = catalog(doc); - if let Some(list) = cat.get(b"OutputIntents").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()) { - for oi in list { - if let Some(od) = doc.resolve(&oi).as_dict() { - let id = od.get(b"OutputConditionIdentifier").and_then(|v| doc.resolve(v).as_string().map(|s| s.to_text())).unwrap_or_default(); + let list = cat.get(b"OutputIntents").map(|o| doc.resolve(o)); + if let Some(list) = list.as_deref().and_then(Object::as_array) { + // One intent object listed again is one declaration: its identifier is not copied once + // per listing. Every distinct one is kept (verification reads them all too), and only + // the start of an identifier longer than any real one. + let mut seen = std::collections::HashSet::new(); + for oi in list.iter().filter(|oi| oi.as_ref().is_none_or(|r| seen.insert(r))) { + if let Some(od) = doc.resolve(oi).as_dict() { + let id = od + .get(b"OutputConditionIdentifier") + .and_then(|v| { + doc.resolve(v).as_string().map(|s| { + if s.bytes.len() <= MAX_IDENTIFIER_BYTES { + s.to_text() + } else { + // An even cut keeps UTF-16 text whole code units. + pdfcraft_cos::PdfString { bytes: s.bytes[..MAX_IDENTIFIER_BYTES & !1].to_vec(), hex: s.hex }.to_text() + } + }) + }) + .unwrap_or_default(); d.output_intents.push(id); } } @@ -287,6 +306,14 @@ pub fn declared(doc: &Document) -> Declared { d } +/// The most bytes of an output condition identifier [`declared`] keeps (a real one is a few +/// dozen). +const MAX_IDENTIFIER_BYTES: usize = 1024; + +/// Decoded bytes one verification reads looking for device colour, all content streams +/// together. Past it, content is not inspected, and the report says so. +const MAX_COLOUR_DECODED: usize = 1 << 30; + /// Every indirect object, with its reference. fn objects(doc: &Document) -> Vec<(ObjRef, std::sync::Arc)> { doc.object_numbers() @@ -361,6 +388,9 @@ struct DeviceColour { /// space, which PDF/A accepts whatever the output intent (ISO 19005-2 6.2.4.3). default_rgb: bool, default_cmyk: bool, + /// Content that could not be read (undecodable, or past [`MAX_COLOUR_DECODED`]) may use + /// device colour that was not seen. + incomplete: bool, } impl DeviceColour { @@ -388,27 +418,35 @@ const MAX_DIRECT_DEPTH: u8 = 8; /// The glyph procedures of every Type 3 font in `d`: `d` itself, or a font held directly inside /// it (a font can sit inline in a resource dictionary). -fn type3_procs(doc: &Document, d: &Dict, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs(doc: &Document, d: &Dict, depth: u8, read: &mut std::collections::HashSet, out: &mut std::collections::HashSet) { if d.name(b"Subtype") == Some(b"Type3") - && let Some(procs) = d.get(b"CharProcs").map(|c| doc.resolve(c)) - && let Some(procs) = procs.as_dict() + && let Some(c) = d.get(b"CharProcs") + // One glyph dictionary shared by many fonts is read once: its entries are the same. + && c.as_ref().is_none_or(|r| read.insert(r)) + && let Some(procs) = doc.resolve(c).as_dict() { out.extend(procs.iter().filter_map(|(_, p)| p.as_ref())); } for (_, v) in d.iter() { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } -fn type3_procs_within(doc: &Document, o: &Object, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs_within( + doc: &Document, + o: &Object, + depth: u8, + read: &mut std::collections::HashSet, + out: &mut std::collections::HashSet, +) { if depth > MAX_DIRECT_DEPTH { return; } match o { - Object::Dict(d) => type3_procs(doc, d, depth, out), + Object::Dict(d) => type3_procs(doc, d, depth, read, out), Object::Array(a) => { for v in a { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } _ => {} @@ -430,9 +468,10 @@ fn referenced_content(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) }; out.extend(contents.iter().filter_map(Object::as_ref)); } + let mut read = std::collections::HashSet::new(); for (_, o) in objs { if let Some(d) = dict_of(o) { - type3_procs(doc, d, 0, &mut out); + type3_procs(doc, d, 0, &mut read, &mut out); } } out @@ -560,6 +599,7 @@ fn mark_cs_object(doc: &Document, cs: &Object, used: &mut DeviceColour, depth: u fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) -> DeviceColour { let mut used = DeviceColour::default(); let referenced = referenced_content(doc, objs); + let mut budget = pdfcraft_cos::DecodeBudget::new(MAX_COLOUR_DECODED, usize::MAX); for (r, o) in objs { let Some(d) = dict_of(o) else { continue }; // A resource dictionary's /ColorSpace, whether it is this object or the page's inline @@ -586,9 +626,14 @@ fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) if !(used.gray && used.rgb && used.cmyk) && let Object::Stream(s) = &**o && (referenced.contains(r) || is_content(&s.dict)) - && let Ok(data) = s.decoded() { - let _ = content_colour(&data, &mut used); + match budget.decode(s) { + Ok(data) => { + let _ = content_colour(&data, &mut used); + } + // Not read: it may draw in a device space nothing above saw. + Err(_) => used.incomplete = true, + } } } used @@ -830,6 +875,9 @@ fn check_output_intent( issue: &mut impl FnMut(&'static str, &'static str, Option, Option, bool), ) { let used = uses_device_colour(doc, objs); + if used.incomplete { + issue("6.2.3", "Some content could not be read for device colour", None, None, false); + } match pdfa_intent_profile(doc) { None if used.any() => { issue("6.2.3", "Device colour is used, but there is no PDF/A output intent", None, None, !used.cmyk); @@ -1215,22 +1263,56 @@ const MAX_ACTION_CHAIN: usize = 1024; fn chain_forbidden(doc: &Document, a: &Dict) -> Option { let mut read = 0; let mut seen = std::collections::HashSet::new(); - // Actions written in an indirect `/Next` array, copied out of it. - let mut held: Vec = Vec::new(); + // Objects `/Next` refers to (an action, or an array of them), each read once and borrowed: + // one array shared by many chains is not copied per chain. + let mut held: Vec> = Vec::new(); if let Some(s) = walk_chain(doc, a, &mut read, &mut seen, &mut held) { return Some(s); } - while let Some(d) = held.pop() { - if let Some(s) = walk_chain(doc, &d, &mut read, &mut seen, &mut held) { - return Some(s); + while let Some(o) = held.pop() { + let dicts: Vec<&Dict> = match &*o { + Object::Dict(d) => vec![d], + Object::Array(list) => { + // An array read is work too, with its entries: a chain of arrays alone must + // still reach the limit. + read = read.saturating_add(1 + list.len().min(MAX_ACTION_CHAIN)); + if read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); + } + // More than the chain limit can't be read anyway. + hold_referenced(doc, list, &mut seen, &mut held); + list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).collect() + } + _ => continue, + }; + for d in dicts { + if let Some(s) = walk_chain(doc, d, &mut read, &mut seen, &mut held) { + return Some(s); + } } } None } -/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. The -/// actions of each indirect `/Next` array (each read once) go to `held`. -fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collections::HashSet, held: &mut Vec) -> Option { +/// Queue the objects the references in a `/Next` array name, each once. +fn hold_referenced(doc: &Document, list: &[Object], seen: &mut std::collections::HashSet, held: &mut Vec>) { + for r in list.iter().filter_map(Object::as_ref).take(MAX_ACTION_CHAIN) { + if seen.insert(r) { + // Resolved through any alias: a reference to a reference names its object's value. + held.push(doc.resolve(&Object::Ref(r))); + } + } +} + +/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. What +/// an indirect `/Next`, or a reference inside a `/Next` array, names goes to `held`. +fn walk_chain( + doc: &Document, + a: &Dict, + read: &mut usize, + seen: &mut std::collections::HashSet, + held: &mut Vec>, +) -> Option { let mut stack = vec![a]; while let Some(d) = stack.pop() { *read += 1; @@ -1242,13 +1324,16 @@ fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collec } match d.get(b"Next") { Some(Object::Dict(next)) => stack.push(next), - Some(Object::Array(list)) => stack.extend(list.iter().filter_map(Object::as_dict)), - Some(Object::Ref(r)) if seen.insert(*r) => { - if let Object::Array(list) = &*doc.get(*r) { - // More than the chain limit can't be read anyway. - held.extend(list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).cloned()); + Some(Object::Array(list)) => { + *read = read.saturating_add(list.len().min(MAX_ACTION_CHAIN)); + // A wide array is refused before any of it is queued. + if *read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); } + hold_referenced(doc, list, seen, held); + stack.extend(list.iter().filter_map(Object::as_dict)); } + Some(Object::Ref(r)) if seen.insert(*r) => held.push(doc.resolve(&Object::Ref(*r))), _ => {} } } @@ -1291,6 +1376,26 @@ fn strip_actions(doc: &Document, d: &mut Dict) -> bool { changed } +/// Take the annotations `gone` off every page that lists them, each page's list rewritten once, and +/// replace their indirect objects with `Null`. +fn remove_from_pages(doc: &mut Document, gone: &std::collections::HashSet) -> Result<(), pdfcraft_cos::CosError> { + if gone.is_empty() { + return Ok(()); + } + for p in pdfcraft_model::pages(doc) { + let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); + let listed = |a: &Object| a.as_ref().is_some_and(|r| gone.contains(&r)); + if annots.iter().any(listed) { + let kept: Vec = annots.into_iter().filter(|a| !listed(a)).collect(); + doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; + } + } + for r in gone { + doc.set(*r, Object::Null); + } + Ok(()) +} + /// Collects every indirect annotation reference listed in any page's `/Annots` array. fn page_annot_refs(doc: &Document) -> std::collections::HashSet { let mut refs = std::collections::HashSet::new(); @@ -1304,19 +1409,6 @@ fn page_annot_refs(doc: &Document) -> std::collections::HashSet { refs } -/// Removes annotation `r` from every page's `/Annots` list and replaces its indirect object with `Null`. -fn remove_annot_from_pages(doc: &mut Document, r: ObjRef) -> Result<(), pdfcraft_cos::CosError> { - for p in pdfcraft_model::pages(doc) { - let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); - if annots.iter().any(|a| a.as_ref() == Some(r)) { - let kept: Vec = annots.into_iter().filter(|a| a.as_ref() != Some(r)).collect(); - doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; - } - } - doc.set(r, Object::Null); - Ok(()) -} - fn fix_annot_or_field(doc: &Document, nd: &mut Dict, is_annot: bool, sub: Option<&[u8]>, actions: &mut usize, flags: &mut usize) -> bool { let mut changed = strip_actions(doc, nd); *actions += changed as usize; @@ -1413,8 +1505,10 @@ pub fn convert(doc: &mut Document, level: Level) -> Result Result::to_vec); let sub = d.name(b"Subtype").map(<[u8]>::to_vec); - let is_annot = matches!(&*o, Object::Dict(_)) && is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)); + let is_annot = matches!(&*o, Object::Dict(_)) + // A widget is an annotation whether or not it says `/Type /Annot` (the form reader + // finds it by its field), so its actions go the same way. + && (is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)) || sub.as_deref() == Some(b"Widget")); if is_annot && sub.as_deref().is_some_and(|s| FORBIDDEN_ANNOTS.contains(&s)) { - remove_annot_from_pages(doc, r)?; + // Taken off its page below, all of them in one pass over the pages. + forbidden_annots.insert(r); removed += 1; continue; } @@ -1489,6 +1588,7 @@ pub fn convert(doc: &mut Document, level: Level) -> Result>", pages, page, &link(&format!("<< /S /GoTo /D [3 0 R /Fit] /Next {js} >>"))], ""), // `/Next` refers to an array that holds the action directly. build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), &format!("[{js}]")], ""), + // `/Next` refers to the action itself, which carries `/P` and `/K` as a structure + // element would: those keys must not exempt it. + build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + pages, + page, + &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), + "<< /S /JavaScript /JS (app.alert(1)) /P 3 0 R /K 0 >>", + ], + "", + ), + // A `/Next` array refers to the action. + build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next [5 0 R] >>"), js], ""), ]; for (i, mut doc) in cases.into_iter().enumerate() { let found = messages(&verify(&doc, Level::A2b)).contains(&"A JavaScript action isn't allowed"); @@ -495,6 +509,116 @@ fn an_output_intent_profile_header_is_read_within_a_size_cap() { } } +/// A widget the form reader finds by its field is an annotation whether or not it writes +/// `/Type /Annot`: conversion takes its additional actions off all the same. +#[test] +fn a_widget_without_a_type_loses_its_additional_actions() { + let pages = "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>"; + let page = "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>"; + let widget = "<< /Subtype /Widget /Rect [0 0 10 10] /F 4 /T (f) /AA << /K << /S /JavaScript /JS (1) >> >> >>"; + let mut doc = build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, widget], ""); + convert(&mut doc, Level::A2b).unwrap(); + let w = doc.get(pdfcraft_cos::ObjRef::new(4, 0)); + assert!(w.as_dict().is_some_and(|d| d.get(b"AA").is_none()), "{w:?}"); +} + +/// Forbidden annotations on one page are taken off it in one pass, all of them. +#[test] +fn forbidden_annotations_leave_their_page_together() { + let n = 300; + let annots: Vec = (0..n).map(|i| format!("{} 0 R", i + 4)).collect(); + let page = format!("<< /Type /Page /Parent 2 0 R /Annots [{} 2 0 R] >>", annots.join(" ")); + let mut objs = + vec!["<< /Type /Catalog /Pages 2 0 R >>".to_string(), "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>".into(), page]; + objs.extend((0..n).map(|_| "<< /Type /Annot /Subtype /Sound /Rect [0 0 0 0] /P 3 0 R >>".to_string())); + let refs: Vec<&str> = objs.iter().map(String::as_str).collect(); + let mut doc = build(&refs, ""); + let r = convert(&mut doc, Level::A2b).unwrap(); + assert!(r.fixed.iter().any(|f| f.starts_with(&format!("{n}: removed multimedia"))), "{:?}", r.fixed); + let left = pdfcraft_model::pages(&doc)[0].dict.get(b"Annots").and_then(|a| doc.resolve(a).as_array().map(Vec::len)); + assert_eq!(left, Some(1)); +} + +/// Content that can't be read may draw in DeviceCMYK: convert then adds no sRGB output intent. +#[test] +fn convert_adds_no_srgb_intent_when_colour_could_not_all_be_read() { + let mut doc = build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>", + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << >> >>", + // A /Filter that isn't a name can't be decoded: the content isn't read. + "<< /Length 9 /Filter 5 >>\nstream\nnot flate\nendstream", + ], + "", + ); + convert(&mut doc, Level::A2b).unwrap(); + assert!(declared(&doc).output_intents.is_empty()); +} + +/// A `/Next` chain of arrays alone (each array naming the next) still reaches the chain limit, +/// so the walk's work is bounded however the arrays are linked. +#[test] +fn an_array_only_next_chain_is_too_deeply_chained() { + let mut doc = Document::new_empty(); + let mut next = Object::Array(Vec::new()); + for _ in 0..2 * MAX_ACTION_CHAIN { + next = Object::Array(vec![Object::Ref(doc.add(next))]); + } + let mut action = Dict::new(); + action.set(b"S".to_vec(), Object::name("URI")); + action.set(b"Next".to_vec(), Object::Ref(doc.add(next))); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // A short chain of arrays is read through. + let mut short = Object::Array(Vec::new()); + for _ in 0..10 { + short = Object::Array(vec![Object::Ref(doc.add(short))]); + } + action.set(b"Next".to_vec(), Object::Ref(doc.add(short))); + assert_eq!(chain_forbidden(&doc, &action), None); +} + +/// A direct `/Next` array wider than the chain limit is refused as soon as it is charged, and a +/// `/Next` that reaches its array through a reference alias is read through it. +#[test] +fn a_wide_direct_next_array_is_refused_and_an_alias_is_followed() { + let uri = || { + let mut d = Dict::new(); + d.set(b"S".to_vec(), Object::name("URI")); + d + }; + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Array((0..=MAX_ACTION_CHAIN).map(|_| Object::Dict(uri())).collect())); + let doc = Document::new_empty(); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // `/Next` -> 1 0 R -> 2 0 R -> [ << /S /JavaScript >> ] + let mut doc = Document::new_empty(); + let mut js = Dict::new(); + js.set(b"S".to_vec(), Object::name("JavaScript")); + let list = doc.add(Object::Array(vec![Object::Dict(js)])); + let alias = doc.add(Object::Ref(list)); + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Ref(alias)); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("JavaScript")); +} + +/// Every distinct output intent is declared, past eight; an identifier longer than any real one +/// keeps its start. +#[test] +fn every_output_intent_is_declared_with_its_identifier_cut_to_the_bound() { + let long = "x".repeat(4 * MAX_IDENTIFIER_BYTES); + let intents: Vec = (0..9) + .map(|i| { + format!("<< /Type /OutputIntent /S /GTS_PDFA1 /OutputConditionIdentifier ({}) >>", if i == 8 { long.clone() } else { format!("c{i}") }) + }) + .collect(); + let catalog = format!("<< /Type /Catalog /Pages 2 0 R /OutputIntents [{}] >>", intents.join(" ")); + let doc = build(&[catalog.as_str(), "<< /Type /Pages /Kids [] /Count 0 >>"], ""); + let d = declared(&doc); + assert_eq!(d.output_intents.len(), 9); + assert_eq!(d.output_intents[8].len(), MAX_IDENTIFIER_BYTES); +} + #[test] fn typeless_annotations_indexed_cmyk_and_indirect_filter_elements_are_handled() { // Annotations in /Annots that omit optional /Type /Annot and /P (while keeping required /Subtype From 13d63673857d10138c0fa4302c3bcc75cf50171c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:42 -0500 Subject: [PATCH 28/46] fix(redact): read each Type 3 glyph program once per cleanup (#5993) clean_type3_font decoded and parsed every surviving glyph program of a font, for every font. Many small font dictionaries sharing one compressed CharProc multiplied that work without multiplying the stored data, with a fresh allowance for each decode. The resource names each indirect glyph program uses are now cached by object across fonts (resource-name interpretation stays per font, since each font prunes its own /Resources), and all decoding spends one cleanup-wide DecodeBudget; past it the cleanup fails closed. --- .../redact/src/tests/surviving_references.rs | 29 ++++++++++++++ crates/redact/src/type3.rs | 39 +++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index d02e76832e..bec850e302 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -390,6 +390,35 @@ fn type3_font_used_in_a_form_xobject_has_its_redacted_glyph_removed() { assert!(!full_save_holds(&doc, b"SECRET_FORM_GLYPH_PAYLOAD"), "redacted glyph program survived"); } +/// Type 3 cleanup decoded a glyph program again for every font listing it: 2 000 fonts sharing +/// one large program read it 2 000 times. It is read once now, and the cleanup still keeps the +/// glyph every font shows. +#[test] +fn a_glyph_program_many_type3_fonts_share_is_read_once() { + const FONTS: usize = 2_000; + let names: String = (0..FONTS).map(|i| format!("/T{i} {} 0 R ", 6 + i)).collect(); + let shows: String = (0..FONTS).map(|i| format!("/T{i} 10 Tf (A) Tj ")).collect(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << {names}>> >> >>").into_bytes(), + stream("", format!("BT 10 100 Td {shows}ET").as_bytes()), + stream("", [b"SHARED_GLYPH ".as_slice(), &b"0 0 m 1 1 l S ".repeat(50_000)].concat()), + ]; + objs.extend((0..FONTS).map(|_| { + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 65 /Widths [0] /Encoding << /Differences [65 /A] >> /CharProcs << /A 5 0 R >> >>".to_vec() + })); + let mut doc = pdf(objs); + // The glyphs are drawn at x 10 with no width, so a mark away from them leaves them all. + mark(&mut doc, 0, &[[200.0, 200.0, 250.0, 250.0]], ""); + let started = std::time::Instant::now(); + apply(&mut doc, None).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(60), "{:?}", started.elapsed()); + let font = doc.get(ObjRef::new(6, 0)); + let procs = doc.resolve(font.as_dict().unwrap().get(b"CharProcs").unwrap()); + assert!(procs.as_dict().unwrap().contains(b"A"), "the glyph every font shows was unlinked"); +} + /// A Type 3 font with glyphs A (its program object 7) and B (object 8), at object 6. const TYPE3: &str = "<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> >>"; diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 11d5a76f6a..1817fb3c39 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::Pieces; -use pdfcraft_cos::{CosError, Dict, Document, ObjRef, Object}; +use pdfcraft_cos::{CosError, DecodeBudget, Dict, Document, ObjRef, Object}; use crate::{RedactError, page_streams}; @@ -130,6 +130,25 @@ struct UsedInGlyphs { properties: HashSet>, } +impl UsedInGlyphs { + fn merge(&mut self, other: &UsedInGlyphs) { + self.xobjects.extend(other.xobjects.iter().cloned()); + self.colorspaces.extend(other.colorspaces.iter().cloned()); + self.patterns.extend(other.patterns.iter().cloned()); + self.shadings.extend(other.shadings.iter().cloned()); + self.extgstates.extend(other.extgstates.iter().cloned()); + self.fonts.extend(other.fonts.iter().cloned()); + self.properties.extend(other.properties.iter().cloned()); + } +} + +/// What one Type 3 cleanup reads of glyph programs: the resource names each indirect program +/// uses, read once however many fonts share it, and one decoding allowance for all of them. +struct GlyphReads { + used: HashMap, + budget: DecodeBudget, +} + fn collect_used_resources_in_stream(data: &[u8], used: &mut UsedInGlyphs) -> Result<(), CosError> { // One operator at a time, so a glyph program of millions of them is never held whole. A byte // that can't be read anywhere in it still refuses, once the loop has read it all. @@ -731,9 +750,10 @@ pub(crate) fn unlink_type3(doc: &mut Document) -> Result<(), RedactError> { } // For each Type 3 font: unlink unused glyphs from CharProcs and prune exclusive resources. + let mut reads = GlyphReads { used: HashMap::new(), budget: DecodeBudget::page() }; for (font_key, font_dict) in &type3_fonts { let surviving = used_glyphs.get(font_key).cloned().unwrap_or_default(); - clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving)?; + clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving, &mut reads)?; } Ok(()) @@ -808,6 +828,7 @@ fn clean_type3_font( font_key: &FontKey, font_dict: &Dict, surviving: &HashSet>, + reads: &mut GlyphReads, ) -> Result<(), RedactError> { let Some(char_procs_val) = font_dict.get(b"CharProcs") else { return Err(CosError::Syntax { offset: 0, detail: "redaction could not establish safe Type 3 font cleanup: missing CharProcs".into() }.into()); @@ -851,6 +872,10 @@ fn clean_type3_font( let mut used_in_glyphs = UsedInGlyphs::default(); for (_, proc_val) in char_procs.iter() { + if let Some(used) = proc_val.as_ref().and_then(|r| reads.used.get(&r)) { + used_in_glyphs.merge(used); + continue; + } let resolved = doc.resolve(proc_val); let Object::Stream(s) = &*resolved else { return Err(CosError::Syntax { @@ -859,8 +884,14 @@ fn clean_type3_font( } .into()); }; - let data = doc.with_direct_parms(s).decoded()?; - collect_used_resources_in_stream(&data, &mut used_in_glyphs)?; + // Past the allowance the cleanup can't be shown complete, and the redaction is refused. + let data = reads.budget.decode(&doc.with_direct_parms(s))?; + let mut used = UsedInGlyphs::default(); + collect_used_resources_in_stream(&data, &mut used)?; + used_in_glyphs.merge(&used); + if let Some(r) = proc_val.as_ref() { + reads.used.insert(r, used); + } } if prune_type3_font_resources(doc, &mut f_res, &used_in_glyphs) { From b52bc303281b58eb0bd28134cc7a1dc764177007 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:56 -0500 Subject: [PATCH 29/46] fix(annot): count ink points as strokes resolve and cap the eraser's samples (#6001) * fix(annot): count ink points as strokes resolve and cap the eraser's samples Resizing and erasing a drawing resolved every /InkList entry without a total, so one stroke array listed many times expanded freely; the eraser densified every segment into up to 2,000 samples with no overall limit. * review: refuse an eraser result that exceeds the stored ink point cap * review: say the erasure would make the drawing too large * address review: the eraser counts each stroke's final point against its sample total --- crates/annot/src/modify.rs | 69 ++++++++++++++++++++++++++------------ crates/annot/src/tests.rs | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 22 deletions(-) diff --git a/crates/annot/src/modify.rs b/crates/annot/src/modify.rs index ae2222b387..6566337ca1 100644 --- a/crates/annot/src/modify.rs +++ b/crates/annot/src/modify.rs @@ -857,19 +857,38 @@ pub fn set_points(doc: &mut Document, page: usize, index: usize, points: &[[f64; /// Scale a drawing's strokes into `rect`. The margin between the strokes and the old `/Rect` (half /// the line width and a little more) stays the same on each side, so the line isn't clipped. +/// Most stroke points a stored drawing may have for resizing or erasing it. Counted as each +/// stroke is resolved, so one stroke array referenced again and again can't expand past it. +const MAX_STORED_INK_POINTS: usize = 1_000_000; +/// Most points the eraser densifies a drawing into before testing them against its path. +const MAX_ERASE_SAMPLES: usize = 4_000_000; + +/// The strokes of an ink annotation's `/InkList`, at most [`MAX_STORED_INK_POINTS`] points. +fn ink_strokes(doc: &Document, d: &Dict) -> Result>, AnnotError> { + let too_many = || AnnotError::Invalid(format!("the drawing has more than {MAX_STORED_INK_POINTS} points")); + let list = d.get(b"InkList").map(|l| doc.resolve(l)); + let mut points = 0usize; + let mut strokes = Vec::new(); + for s in list.as_deref().and_then(Object::as_array).map(Vec::as_slice).unwrap_or_default() { + let stroke = doc.resolve(s); + let Some(a) = stroke.as_array() else { + strokes.push(Vec::new()); + continue; + }; + // Charged before the coordinates are read: a shared array costs each time it is listed. + points = points.saturating_add(a.len() / 2); + if points > MAX_STORED_INK_POINTS { + return Err(too_many()); + } + let v: Vec = a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); + strokes.push(v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect()); + } + Ok(strokes) +} + fn resize_ink(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta) -> Result<(), AnnotError> { let d = annot_dict(doc, r); - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; let Some(inner) = bounds(strokes.iter().flatten().copied()) else { return Err(AnnotError::Invalid("the drawing has no strokes".into())); }; @@ -1475,17 +1494,7 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] if path.is_empty() || !path.iter().all(|p| finite(p)) || !radius.is_finite() || radius <= 0.0 { return Err(AnnotError::Invalid("invalid eraser path".into())); } - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; // Distance from a point to the eraser's path (segments, or a single point). let near = |p: [f64; 2]| -> bool { if path.len() == 1 { @@ -1501,17 +1510,27 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] }; let mut kept: Vec> = Vec::new(); let mut erased = false; + let mut samples = 0usize; for s in &strokes { // Densify so long segments are cut where the eraser crosses them. let mut pts: Vec<[f64; 2]> = Vec::new(); for w in s.windows(2) { let n = ((w[1][0] - w[0][0]).hypot(w[1][1] - w[0][1]) / (radius / 2.0)).ceil().clamp(1.0, 2000.0) as usize; + samples = samples.saturating_add(n); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } for k in 0..n { let t = k as f64 / n as f64; pts.push([w[0][0] + (w[1][0] - w[0][0]) * t, w[0][1] + (w[1][1] - w[0][1]) * t]); } } if let Some(last) = s.last() { + // The final point is a sample too. + samples = samples.saturating_add(1); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } pts.push(*last); } let mut run: Vec<[f64; 2]> = Vec::new(); @@ -1537,6 +1556,12 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] delete_annotation(doc, page, index)?; return Ok(true); } + // The retained runs are stored as the new `/InkList`: past the cap, writing them would leave a + // drawing that later `resize_ink` and `erase_ink` refuse to read. + let points: usize = kept.iter().map(Vec::len).sum(); + if points > MAX_STORED_INK_POINTS { + return Err(AnnotError::Invalid(format!("erasing would leave a drawing with more than {MAX_STORED_INK_POINTS} points"))); + } let width = d.get(b"BS").and_then(|b| b.as_dict()).and_then(|b| b.get(b"W")).and_then(|w| w.as_f64()).unwrap_or(1.0); let rect = grow(bounds(kept.iter().flatten().copied()).unwrap_or_default(), width / 2.0 + 1.0); doc.update_dict(r, |d| { diff --git a/crates/annot/src/tests.rs b/crates/annot/src/tests.rs index ce5a8c1604..980959d4d9 100644 --- a/crates/annot/src/tests.rs +++ b/crates/annot/src/tests.rs @@ -4797,6 +4797,73 @@ fn freehand_highlight_is_compatible_ink_with_marker_blending() { } } +/// A drawing whose /InkList lists one stroke array over and over is refused once its points +/// pass the cap, counted as each stroke is resolved, before they are collected or scaled. +#[test] +fn ink_points_are_counted_as_strokes_resolve() { + let numbers: String = (0..2000).map(|k| format!(" {}", k % 100)).collect(); + let refs = " 5 0 R".repeat(501); + let doc_with = |refs: &str| { + doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + format!("[{numbers}]"), + ]) + }; + // 501 × 1,000 points is under the cap: the points are read (this bare drawing then has no + // appearance it can be redrawn with, which is another matter). + let mut doc = doc_with(&refs); + let under = set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")); + assert!(!matches!(&under, Err(AnnotError::Invalid(m)) if m.contains("points")), "{under:?}"); + // 1,001 × 1,000 points is over it. + let mut doc = doc_with(&" 5 0 R".repeat(1001)); + match set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 1.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// The eraser's densified samples have a total: a tiny eraser over long strokes is refused +/// instead of interpolating millions of points. +#[test] +fn the_eraser_densifies_within_a_total() { + let zigzag: String = (0..2100).map(|k| format!(" {} {}", if k % 2 == 0 { 0 } else { 5000 }, k)).collect(); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 5000 2100] /InkList [[{zigzag}]] >>"), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 0.01, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// Each stroke's final point is a sample too: 500,000 strokes of two points at 8 samples a segment +/// are 4,000,000 densified samples and as many final points again, past the total. +#[test] +fn the_eraser_counts_each_strokes_final_point() { + let refs = " 5 0 R".repeat(500_000); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + "[0 0 8 0]".into(), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 2.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + /// A 3D annotation is PDF 1.6 and a PRC model 1.7 (ISO 32000-2 §13.6): an older document's /// catalog says so, and a newer one is left as it is. #[test] From ee367a4a1c6087c07003cdf9d3999bc4281a997a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:48:10 -0500 Subject: [PATCH 30/46] fix(xfa): bound the name and path bytes data nodes and queued instances copy (#6032) * fix(xfa): bound the name and path bytes data nodes and queued instances copy * review: bound the name bytes of a whole data path and report each cap separately * address review: the queued-instance allowance counts path segment buffers and the dedup copy * address review: the queued path's spare capacity is counted --- crates/xfa/src/data.rs | 16 +++++++++++++--- crates/xfa/src/data/build.rs | 2 +- crates/xfa/src/data/write.rs | 24 +++++++++++++++++++++++- crates/xfa/src/tests.rs | 25 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/xfa/src/data.rs b/crates/xfa/src/data.rs index 34be028d89..4e7de4af74 100644 --- a/crates/xfa/src/data.rs +++ b/crates/xfa/src/data.rs @@ -148,13 +148,21 @@ pub enum DataOp { impl DataNode { /// The node at `path`, made (with empty siblings before it) where missing; `None` past /// the index cap. - fn ensure(&mut self, path: &[(String, usize)]) -> Option<&mut DataNode> { + fn ensure(&mut self, path: &[(String, usize)], budget: &mut usize) -> Option<&mut DataNode> { let mut node = self; for (name, idx) in path { if *idx >= MAX_INDEX { return None; } let have = node.children.iter().filter(|c| &c.name == name).count(); + // Every skipped sibling owns a copy of the name: bounded as `Build::at` bounds a write. + // The budget is the whole path's, not each level's: a deep path that is affordable at + // every level taken alone would otherwise allocate its way past the bound. + let need = (*idx + 1).saturating_sub(have).saturating_mul(name.len()); + if need > *budget { + return None; + } + *budget -= need; for _ in have..=*idx { node.children.push(DataNode { name: name.clone(), ..Default::default() }); } @@ -166,9 +174,11 @@ impl DataNode { /// Apply `op` to this tree (the root is the `xfa:data` element), as [`write_data_ops`] /// will to the packet, so scripts that run before it is written see the change. pub fn apply(&mut self, op: &DataOp) { + // One budget for the op's whole path, as `Build::at` bounds a write. + let mut budget = build::MAX_NEW_NAME_BYTES; match op { DataOp::Value { path, text } => { - if let Some(n) = self.ensure(path) { + if let Some(n) = self.ensure(path, &mut budget) { // A multi-select list's picks, one `value` each (see `value_at`), told apart // as the packet writer does: `value` leaves only, or a group holding none. let picks = if n.children.is_empty() { n.group } else { n.children.iter().all(|c| c.name == "value" && c.children.is_empty()) }; @@ -189,7 +199,7 @@ impl DataNode { if let Some(&last) = count.checked_sub(1).as_ref() { let mut path = parent.clone(); path.push((name.clone(), last.min(MAX_INDEX.saturating_sub(1)))); - let _ = self.ensure(&path); + let _ = self.ensure(&path, &mut budget); } } DataOp::Remove(path) => { diff --git a/crates/xfa/src/data/build.rs b/crates/xfa/src/data/build.rs index 98e34f0f49..e90904d60c 100644 --- a/crates/xfa/src/data/build.rs +++ b/crates/xfa/src/data/build.rs @@ -38,7 +38,7 @@ pub(super) const MAX_INDEX: usize = 10_000; /// Most bytes of names the nodes one write creates hold together: every sibling an index /// skips over gets its own copy of the name. -const MAX_NEW_NAME_BYTES: usize = 4 << 20; +pub(super) const MAX_NEW_NAME_BYTES: usize = 4 << 20; /// What one write may still create: nodes and bytes of their names. pub(super) struct NewNodes { diff --git a/crates/xfa/src/data/write.rs b/crates/xfa/src/data/write.rs index 4667036fe5..554685e1ad 100644 --- a/crates/xfa/src/data/write.rs +++ b/crates/xfa/src/data/write.rs @@ -82,6 +82,11 @@ pub fn remove_data_instance(doc: &mut Document, path: &DataPath) -> Result) let mut run: Vec = Vec::new(); // Instances queued in this run, by parent and name: asking for more only queues the rest. let mut queued: HashMap<(&DataPath, &str), usize> = HashMap::new(); - let (mut ensured, mut capped) = (0usize, false); + let (mut ensured, mut ensured_bytes, mut capped, mut capped_bytes) = (0usize, 0usize, false, false); let flush = |run: &mut Vec, steps: &mut Vec>, removes: &mut Vec>| { if !run.is_empty() { let mut seen = std::collections::HashSet::new(); @@ -117,12 +122,24 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) } DataOp::Instances { parent, name, count } => { let done = queued.entry((parent, name.as_str())).or_insert(0); + // The path (its segments' buffers and their text) is held by the queued step, which a + // clone then `push` can leave with up to twice the room, and cloned again into + // `flush`'s dedup set, beside the label and the step itself. + let segment = std::mem::size_of::<(String, usize)>(); + let path_bytes = + parent.iter().fold(segment.saturating_add(name.len()), |n, (p, _)| n.saturating_add(segment).saturating_add(p.len())); + let per_instance = path_bytes.saturating_mul(3).saturating_add(name.len()).saturating_add(16 + std::mem::size_of::()); while *done < (*count).min(MAX_INDEX) { if ensured >= MAX_ENSURED { capped = true; break; } + if ensured_bytes.saturating_add(per_instance) > MAX_ENSURED_BYTES { + capped_bytes = true; + break; + } ensured += 1; + ensured_bytes += per_instance; #[cfg(test)] ENSURED.with(|n| n.set(n.get() + 1)); let k = *done; @@ -154,6 +171,11 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) if capped { written.warnings.push(format!("XFA data instances not written past {MAX_ENSURED}: the rows were added and removed too often")); } + if capped_bytes { + written + .warnings + .push(format!("XFA data instances not written past {MAX_ENSURED_BYTES} bytes of names: the rows were added and removed too often")); + } Ok(written) } diff --git a/crates/xfa/src/tests.rs b/crates/xfa/src/tests.rs index 682887e673..27e6522f26 100644 --- a/crates/xfa/src/tests.rs +++ b/crates/xfa/src/tests.rs @@ -7114,6 +7114,31 @@ fn a_container_is_as_wide_as_a_turned_draw_stands() { } } +/// Every sibling a SOM index skips owns a copy of the element name: the in-memory builder refuses +/// a path whose copies would pass the name-byte bound the writer holds itself to, and creates +/// nothing for it. +#[test] +fn the_in_memory_data_builder_bounds_copied_name_bytes() { + let mut root = DataNode::default(); + // 1 KiB x 9,999 skipped siblings is about 10 MB, over the 4 MiB bound. + let long = "n".repeat(1 << 10); + root.apply(&DataOp::Value { path: vec![(long.clone(), 9_999)], text: "x".into() }); + assert!(root.children.is_empty()); + // Within the bound, the same shape still builds (and keeps indexes). + root.apply(&DataOp::Value { path: vec![(long, 100)], text: "x".into() }); + assert_eq!(root.children.len(), 101); +} + +/// Each queued instance owns a copy of its parent path: a long ancestor stops the queue at the +/// byte allowance, reported as capped, instead of being multiplied by the instance count. +#[test] +fn queued_instances_are_bounded_by_the_path_bytes_they_copy() { + let mut doc = Document::new_empty(); + let parent: DataPath = vec![("a".repeat(16 << 20), 0), ("b".repeat(16 << 20), 0)]; + let written = crate::data::write_data_ops(&mut doc, &[DataOp::Instances { parent, name: "row".into(), count: 100 }], None).unwrap(); + assert!(written.warnings.iter().any(|w| w.contains("not written past")), "{:?}", written.warnings); +} + /// A page break splits an empty multi-line field that grows with its text (XFA 3.3 pp. 291–292: /// text splits between lines, and a field's `keep intact` is `none`): it starts where it is, its /// widget runs to the bottom of the content area, and the rest of its box, closed on every side, From 1e91e8a93ec8b8107f2c1f4f654c0a7629b07119 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:48:24 -0500 Subject: [PATCH 31/46] fix(forms): a read-only radio button is not picked or left through set_value (#6038) * fix(forms): a read-only radio button is not picked or left through set_value * address review: use /V as the selection in the read-only radio check, keep reset going * address review: radio read-only checks judge each button by the state it would get, and cover FieldValue::Widget --- crates/forms/src/tests.rs | 62 ++++++++++++++++++++++++++++++++++++ crates/forms/src/values.rs | 65 ++++++++++++++++++++++++++++++++------ 2 files changed, 117 insertions(+), 10 deletions(-) diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 75dd075889..a2a87f3a73 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -6802,6 +6802,48 @@ fn locked_button_export_edits_leave_the_document_unchanged() { } } +/// A radio button that is read-only on its own (its own `/Ff`) can't be picked, and a selected one +/// can't be switched away from, though the group as a whole is writable. +#[test] +fn a_read_only_radio_button_is_neither_picked_nor_left() { + let lock = |doc: &mut Document, num: u32| doc.update_dict(pdfcraft_cos::ObjRef::new(num, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + let mut doc = radio_group_form(); + lock(&mut doc, 7); // Medium + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("M".into()))), Err(FormError::ReadOnly(_)))); + set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))).expect("an unlocked button is picked"); + // Small (selected) locked: the group stays on it. + let mut doc = radio_group_form(); + lock(&mut doc, 6); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// The group's `/V` names its selection even when the button's `/AS` says nothing: a read-only +/// button that `/V` selects is not switched away from either. +#[test] +fn a_read_only_radio_button_selected_by_the_group_value_is_not_left() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| { + d.set(b"Ff".to_vec(), Object::Int(1)); + d.set(b"AS".to_vec(), Object::name("Off")); + }) + .unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// Clearing the form skips a group held by a read-only button and still clears the rest. +#[test] +fn clearing_the_form_goes_on_past_a_group_held_by_a_read_only_button() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + doc.update_dict(pdfcraft_cos::ObjRef::new(9, 0), |d| d.set(b"DV".to_vec(), pdfcraft_cos::PdfString::text("Bob"))).unwrap(); + assert_eq!(reset(&mut doc, None).unwrap(), 1); + let all = fields(&doc); + assert_eq!(field(&all, "size").value, ["S"]); + assert_eq!(field(&all, "name").value, ["Bob"]); +} + #[test] /// Cosmetic field updates skip calculations, while a value write still triggers the chain. fn script_cosmetic_mutations_do_not_run_the_calculation_chain() { @@ -6937,6 +6979,26 @@ fn radio_buttons_without_as_sharing_the_value_show_one_on() { assert_eq!(on_widgets(&doc, "r").0, [false, true, false]); } +/// Two buttons share the on state `Yes`; the first is on by its `/AS` and the second is read-only +/// and off. `/V` is `Yes` for both, but only the first is selected: picking another option switches +/// the first off, which is allowed, and the read-only one never changes. +#[test] +fn a_read_only_button_sharing_the_on_state_of_the_selected_one_does_not_block() { + let mut doc = radio_without_as(49152, "/Ff 1 /AS /Off"); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"AS".to_vec(), Object::name("Yes"))).unwrap(); + set_value(&mut doc, "r", &FieldValue::Radio(Some("No".into()))).expect("the writable selected button may be switched off"); + assert_eq!(on_widgets(&doc, "r").0, [false, false, true]); +} + +/// Selecting another button through `FieldValue::Widget` can't switch a read-only selected one off. +#[test] +fn a_read_only_selected_radio_button_is_not_left_by_picking_a_widget() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Widget(2)), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + /// Table 227, ReadOnly: a processor "shall not allow a user to change the value". A button whose own /// dictionary sets it can't be turned on by number. #[test] diff --git a/crates/forms/src/values.rs b/crates/forms/src/values.rs index af2c5e1aca..b60d8aa140 100644 --- a/crates/forms/src/values.rs +++ b/crates/forms/src/values.rs @@ -356,6 +356,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: let Some(state) = f.widgets.get(*i).and_then(|w| w.on_state.clone()) else { return invalid(format!("{:?} has no button {} with an on state", f.name, i.saturating_add(1))); }; + if f.kind == FieldKind::Radio && radio_change_blocked(f, Some(&state), Some(*i)) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, Some(&state), Some(*i)) } (FieldKind::CheckBox, v) => { @@ -406,6 +409,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: if choice.is_none() && f.has(flags::NO_TOGGLE_TO_OFF) && !f.value.is_empty() { return invalid(format!("{:?} must keep one option selected", f.name)); } + if radio_change_blocked(f, choice.as_deref(), None) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, choice.as_deref(), None) } (FieldKind::Combo | FieldKind::List, FieldValue::Choice(_) | FieldValue::Text(_) | FieldValue::ChoiceIndices(_)) => { @@ -541,17 +547,49 @@ pub(crate) fn scroll_widgets(doc: &mut Document, f: &Field, picked: &[usize], co Ok(()) } +/// The one widget of an exclusive radio group that turns on when `on` is chosen (`pick` when it has +/// that state, else the one already on, else the first); `None` when `on` is `None` or the group +/// turns every widget with that state on together. +fn chosen_widget(f: &Field, on: Option<&str>, pick: Option) -> Option { + let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); + on.filter(|_| exclusive).and_then(|c| { + let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); + let n = f.widgets.len(); + pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) + }) +} + +/// Whether setting a radio group to `on` (through widget `pick`, when the caller chose a widget) +/// would turn a read-only button on or off: such a button can't be picked, and a selected one can't +/// be switched off by picking another. Judged per widget, by the state `set_states` would give it, +/// so buttons sharing an on-state name are told apart by their `/AS`; the group's `/V` stands for +/// the selection only when no button's `/AS` says which is on. +fn radio_change_blocked(f: &Field, on: Option<&str>, pick: Option) -> bool { + let chosen = chosen_widget(f, on, pick); + let before = |i: usize, w: &Widget| { + let Some(s) = w.on_state.as_deref() else { return false }; + if f.widget_on(i) { + return true; + } + // No button sharing this on state is on by its `/AS` and the group's `/V` names it: the + // first of them holds the selection, whatever its own `/AS` says (an `Off` among them). + let sharing = |x: &Widget| x.on_state.as_deref() == Some(s); + f.value.first().map(String::as_str) == Some(s) + && !f.widgets.iter().any(|x| sharing(x) && x.state.as_deref() == Some(s)) + && f.widgets.iter().position(sharing) == Some(i) + }; + f.widgets.iter().enumerate().any(|(i, w)| { + let after = on.is_some() && w.on_state.as_deref() == on && chosen.is_none_or(|k| k == i); + w.read_only && before(i, w) != after + }) +} + /// Check boxes and radio buttons: `/V` on the field, `/AS` on each widget. The widgets whose on /// state is `on` turn on together in a check box (ISO 32000-2 §12.7.5.2.3, EXAMPLE 2) and in a /// radio group with RadiosInUnison. Without it, at most one radio button is on (Table 229, /// §12.7.5.2.3): widget `pick` when it has that state, else the one already on, else the first. fn set_states(doc: &mut Document, f: &Field, on: Option<&str>, pick: Option) -> Result<(), FormError> { - let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); - let chosen = on.filter(|_| exclusive).and_then(|c| { - let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); - let n = f.widgets.len(); - pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) - }); + let chosen = chosen_widget(f, on, pick); // `/V` names the on state, the same for every widget that has it: which of them is on is // their `/AS` alone (§12.7.5.2.4). let v = on.unwrap_or("Off"); @@ -919,10 +957,17 @@ pub fn reset_with_formatting( // A reset is not blocked by NoToggleToOff: go through the writer directly. let mut tmp = f.clone(); tmp.flags &= !flags::NO_TOGGLE_TO_OFF; - write_value_impl(doc, &tmp, &v, scripts, true)?; - changed += 1; - if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { - redraw_fields.insert(f.obj); + match write_value_impl(doc, &tmp, &v, scripts, true) { + Ok(()) => { + changed += 1; + if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { + redraw_fields.insert(f.obj); + } + } + // a read-only button of the group holds the value: this field is left as it is, and the + // rest of the form is still cleared + Err(FormError::ReadOnly(_)) => {} + Err(e) => return Err(e), } } let calculation_runner: &mut dyn Scripts = if calculate_with_scripts { scripts } else { &mut NoScripts }; From 2b65ab000bfe0997dec5fbed2769d3da3799355b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:20:18 -0500 Subject: [PATCH 32/46] ci: use StepSecurity's maintained actions (#7026) Each third-party action that StepSecurity publishes a secure drop-in replacement for now uses that replacement, pinned to a commit with its version as a comment. Versions follow what we used: the newest StepSecurity release in the same major, or the closest major they publish. dtolnay/rust-toolchain refs that named a toolchain (stable, nightly, 1.76) become the replacement's `toolchain` input. --- .github/workflows/fuzz.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index fe0a0074d7..78dc1b614b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -31,10 +31,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 with: toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: fuzz - name: Fetch the pdf.js corpus (seeds) From c73cccfb33a9146470060bf692578fbabb543607 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:37:44 -0500 Subject: [PATCH 33/46] fix(redact): hidden-layer removal reads the default state as viewers do (#5867) * fix(redact): hidden-layer removal reads the default state as viewers do hidden_layers ignored an indirect /BaseState and, with /BaseState /OFF, let /ON override /OFF, so a group both lists named survived as visible. The OCMD and OCG classifier also read /Type directly, so a membership dictionary with an indirect /Type was never evaluated and its content kept. The default state is now /BaseState (resolved), then /ON, then /OFF, and /Type is resolved before classifying a group or membership dictionary. * address review: follow chained /Type and /BaseState references, hash-set ON/OFF lists * address review: a visibility expression naming a group through an alias reference fails closed --- crates/redact/src/interp/layers.rs | 22 +++++++-- crates/redact/src/lib.rs | 13 +++++ crates/redact/src/sanitize.rs | 13 ++--- crates/redact/src/tests.rs | 76 ++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 10 deletions(-) diff --git a/crates/redact/src/interp/layers.rs b/crates/redact/src/interp/layers.rs index 362cb36620..7b8ef82120 100644 --- a/crates/redact/src/interp/layers.rs +++ b/crates/redact/src/interp/layers.rs @@ -20,7 +20,7 @@ impl<'a> Scope<'a> { // groups. A membership dictionary's default policy is AnyOn. let d = doc.resolve(o); let Some(d) = d.as_dict() else { return false }; - if d.name(b"Type") != Some(b"OCMD") { + if !has_type(doc, d, b"OCMD") { return false; } let groups: Vec = match d.get(b"OCGs").map(|g| (*doc.resolve(g)).clone()) { @@ -102,7 +102,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let complete = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => { + Object::Dict(d) if has_type(doc, d, b"OCG") => { if let Some(r) = reference && hidden_layers.contains(&r) { @@ -144,6 +144,11 @@ impl<'a> Scope<'a> { } *budget -= 1; let reference = expression.as_ref(); + // A reference to a reference is no listed group: the renderer rejects the alias, so it + // is not taken as visible because the object it names isn't hidden. + if reference.is_some_and(|r| matches!(*doc.get(r), Object::Ref(_))) { + return None; + } if let Some(r) = reference && !path.insert(r) { @@ -151,7 +156,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let result = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), Object::Array(items) if (2..=4097).contains(&items.len()) => { let operator = match items.first() { Some(Object::Name(n)) => n.as_slice(), @@ -195,6 +200,9 @@ impl<'a> Scope<'a> { if self.hidden_layers.contains(&r) { return true; } + if matches!(*doc.get(r), Object::Ref(_)) { + return true; + } if !path.insert(r) { return true; } @@ -202,7 +210,7 @@ impl<'a> Scope<'a> { let resolved = doc.resolve(expression); let found = match &*resolved { Object::Array(items) => items.iter().any(|item| self.expression_mentions_hidden(doc, item, depth + 1, path, budget)), - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), Object::Dict(_) => true, _ => false, }; @@ -256,3 +264,9 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve } (out, removed) } + +/// Is this dictionary's `/Type` the given name? An indirect `/Type` is followed, as a viewer +/// follows it: a group or membership dictionary must not escape classification through one. +fn has_type(doc: &Document, d: &Dict, name: &[u8]) -> bool { + d.get(b"Type").is_some_and(|t| crate::resolve_chain(doc, t).as_name() == Some(name)) +} diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 7fd34dae12..8f54cefdc6 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -1137,6 +1137,19 @@ pub fn clear_marks(doc: &mut Document, pages: Option<&[usize]>) -> Result std::sync::Arc { + let mut o = doc.resolve(o); + for _ in 0..16 { + if !matches!(*o, pdfcraft_cos::Object::Ref(_)) { + break; + } + o = doc.resolve(&o); + } + o +} + /// The name to save a document under after [`apply`]: `_Redacted` before the extension, as /// Acrobat appends it ("contract.pdf" gives "contract_Redacted.pdf"). The `.pdf` extension is /// matched in any letter case. A name that already ends in `_Redacted` keeps it once. diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index cd38b9eb19..61c7cedda4 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -479,12 +479,13 @@ fn hidden_layers(doc: &Document) -> Vec { o.and_then(|g| doc.resolve(g).as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() }; let all = refs(oc.get(b"OCGs")); - let (on, off) = (refs(d.get(b"ON")), refs(d.get(b"OFF"))); - let mut hidden: Vec = if d.name(b"BaseState") == Some(b"OFF") { - all.iter().copied().filter(|g| !on.contains(g)).collect() - } else { - off.into_iter().filter(|g| all.contains(g)).collect() - }; + // sets: the lists are the document's, and a filter over every group would scan them each time + let (on, off): (std::collections::HashSet, std::collections::HashSet) = + (refs(d.get(b"ON")).into_iter().collect(), refs(d.get(b"OFF")).into_iter().collect()); + // The default state is /BaseState, then /ON turns groups on and /OFF turns them off, so a + // group both lists name ends up off (§8.11.4.3, Table 101) — the viewer reads it the same way. + let base_off = d.get(b"BaseState").map(|b| crate::resolve_chain(doc, b)).is_some_and(|b| b.as_name() == Some(b"OFF".as_slice())); + let mut hidden: Vec = all.iter().copied().filter(|g| off.contains(g) || (base_off && !on.contains(g))).collect(); // Each rule, layer list and layer is read once, however often the rules name them: many rules // can share one long list. let known: std::collections::HashSet = all.iter().copied().collect(); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index dc94a2c5f5..08125165b5 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -4151,6 +4151,8 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { (Some("AllOn"), false, Some("not_off"), false), (None, false, Some("malformed_not"), true), (None, false, Some("malformed_without_ocgs"), true), + // `/VE` names the hidden group through an alias object (10 0 R is a reference to 8 0 R). + (None, false, Some("alias"), true), ]; for (policy, malformed_policy, visibility, should_remove) in cases { let mut doc = one_page( @@ -4188,9 +4190,11 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { Some("malformed_without_ocgs") => { membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("UnknownOperator"), Object::Ref(ObjRef::new(8, 0))])) } + Some("alias") => membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("Or"), Object::Ref(ObjRef::new(10, 0))])), _ => {} } doc.set(ObjRef::new(9, 0), Object::Dict(membership)); + doc.set(ObjRef::new(10, 0), Object::Ref(ObjRef::new(8, 0))); let root = doc.root().unwrap(); doc.update_dict(root, |d| { d.set( @@ -6670,6 +6674,78 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// One page drawing `SECRET` in the `/OC /L` block, with layer 7 0 R in the default +/// configuration `config` (objects 8 and up are `extra`). +fn layer_state_page(config: Vec<(&str, Object)>, resource: &str, extra: Vec>) -> Document { + let mut objects = vec![b"<< /Type /OCG /Name (Secret) >>".to_vec()]; + objects.extend(extra); + let mut doc = one_page(b"/OC /L BDC BT /F1 10 Tf 10 100 Td (SECRET) Tj ET EMC", &format!("/Properties << /L {resource} >>"), objects); + let config = Object::Dict(Dict::from_iter(config.into_iter().map(|(k, v)| (k.as_bytes().to_vec(), v)))); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([(b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), (b"D".to_vec(), config)])), + ); + }) + .unwrap(); + doc +} + +fn assert_layer_secret_removed(mut doc: Document) { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let counts = scan(&doc); + assert!(counts.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0), "{counts:?}"); + remove_hidden(&mut doc, &[Hidden::HiddenLayers]).unwrap(); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!pdfcraft_edit::text_lines(&saved, 0).unwrap().iter().any(|line| line.text.contains("SECRET"))); +} + +#[test] +fn an_indirect_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page(vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], "7 0 R", vec![b"/OFF".to_vec()])); +} + +#[test] +fn a_chained_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], + "7 0 R", + vec![b"9 0 R".to_vec(), b"/OFF".to_vec()], + )); +} + +#[test] +fn a_chained_membership_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"10 0 R".to_vec(), b"/OCMD".to_vec()], + )); +} + +#[test] +fn off_wins_over_on_when_the_base_state_is_off() { + assert_layer_secret_removed(layer_state_page( + vec![ + ("BaseState", Object::name("OFF")), + ("ON", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ("OFF", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ], + "7 0 R", + Vec::new(), + )); +} + +#[test] +fn a_membership_dictionary_with_an_indirect_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"/OCMD".to_vec()], + )); +} + /// An invisible OCR-style text layer drawn in a Type3 font (what the OCR output writes for /// characters beyond WinAnsi) is redacted like any other text: its glyphs are hit-tested by /// their real advances, removed, and the verifier passes. From daec898d7c56fc9f283777c99e2f1ca1f52d53c2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:49:35 -0500 Subject: [PATCH 34/46] fix(sign): judge a shared signature value once per listing; dedupe DSS evidence by object number (#6041) * fix(sign): judge a shared signature value once per listing; dedupe DSS evidence by object number * address review: shared signature values are judged by output index without copying the dictionary; a dead evidence reference does not hide a live one * address review: repeated evidence references are skipped before they are resolved * address review: each full evidence reference is tried once --- crates/sign/src/pdf/listing.rs | 16 ++++++++++++- crates/sign/src/pdf/validate.rs | 20 +++++++++++++--- crates/sign/src/pdf_discovery_tests.rs | 13 +++++++++++ crates/sign/tests/it/pdf.rs | 32 ++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/crates/sign/src/pdf/listing.rs b/crates/sign/src/pdf/listing.rs index c2a7183db1..2d41477127 100644 --- a/crates/sign/src/pdf/listing.rs +++ b/crates/sign/src/pdf/listing.rs @@ -431,8 +431,12 @@ pub fn list(doc: &Document, bytes: &[u8], trust: &TrustStore) -> Vec Vec { let pages = annot_pages(doc); - let mut out = Vec::new(); + let mut out: Vec = Vec::new(); let mut field_values: HashSet = HashSet::new(); + // Fields that name one signature value are judged once: validation does not depend on the + // field, and repeating it per field would repeat the digest, chain and evidence work. + // The row of `out` that holds each value's verdict: a later field copies it, not a map of copies. + let mut judged: HashMap = HashMap::new(); let (fields, whole) = walk_sig_fields(doc); for f in fields { if let Some(v_ref) = f.dict.get(b"V").and_then(Object::as_ref) { @@ -445,6 +449,13 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig .filter(|r| r.len() == 4) .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]); let visible = widget.is_some_and(|(_, w)| widget_shown(doc, w)); + // A repeat of a judged value reads nothing of it: the signature dictionary is not copied. + let shared = f.dict.get(b"V").and_then(Object::as_ref).filter(|_| f.dict.get(b"V").is_some_and(|v| doc.resolve(v).as_dict().is_some())); + if let Some(first) = shared.and_then(|r| judged.get(&r)).and_then(|&i| out.get(i)) { + let own = SignatureInfo { field: f.name.clone(), signed: true, page, rect, visible, ..first.clone() }; + out.push(own); + continue; + } let v = f.signature(doc); let mut info = SignatureInfo { field: f.name.clone(), @@ -477,6 +488,9 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig if let Some(v) = v { validate_into(doc, bytes, trust, &v, &mut info, cache); } + if let Some(r) = shared { + judged.insert(r, out.len()); + } out.push(info); } // Standalone document timestamps (ISO 32000-2 §12.8.2.2) live outside AcroForm fields. diff --git a/crates/sign/src/pdf/validate.rs b/crates/sign/src/pdf/validate.rs index 52bf79351f..d3ad411a62 100644 --- a/crates/sign/src/pdf/validate.rs +++ b/crates/sign/src/pdf/validate.rs @@ -371,7 +371,11 @@ pub(crate) const MAX_DSS_BYTES: usize = 64 << 20; pub(crate) struct DssEvidence<'a> { doc: &'a Document, items: std::vec::IntoIter, - seen: HashSet, + /// Object numbers read: `Document::get` resolves a reference by number, so `5 0 R` and `5 1 R` + /// are one stream. + seen: HashSet, + /// Full references already tried, resolved or not. + tried: HashSet, taken: usize, left: usize, /// A cap stopped the array being read in full. @@ -381,7 +385,7 @@ pub(crate) struct DssEvidence<'a> { impl<'a> DssEvidence<'a> { pub(crate) fn new(doc: &'a Document, dss: &Dict, key: &[u8]) -> Self { let items = dss.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()).unwrap_or_default(); - DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } + DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), tried: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } } } @@ -395,11 +399,21 @@ impl Iterator for DssEvidence<'_> { // Repeats are skipped before the caps count anything, so repeating one stream cannot use // them up; the scan of the array itself is bounded by the parser's cap on array length. for item in self.items.by_ref() { - if item.as_ref().is_some_and(|r| !self.seen.insert(r)) { + // Resolved first: a reference to a generation the slot doesn't hold reads as null and + // must not make a later, valid reference to the same object number a repeat. + // A repeat is skipped before anything is resolved: a long chain of repeats costs no + // lookups. The number is recorded only once the reference gave a stream. + // Each full reference is tried once, so one that never reaches a stream (a cycle) costs + // its lookups once, however often it is listed; `seen` keeps object numbers apart so a + // dead generation can't hide a live one. + if item.as_ref().is_some_and(|r| self.seen.contains(&r.num) || !self.tried.insert(r)) { continue; } let o = self.doc.resolve(&item); let Object::Stream(s) = o.as_ref() else { continue }; + if let Some(r) = item.as_ref() { + self.seen.insert(r.num); + } if self.taken >= MAX_DSS_ENTRIES { self.cut = true; return None; diff --git a/crates/sign/src/pdf_discovery_tests.rs b/crates/sign/src/pdf_discovery_tests.rs index 29f115cb07..d3250d62ed 100644 --- a/crates/sign/src/pdf_discovery_tests.rs +++ b/crates/sign/src/pdf_discovery_tests.rs @@ -373,6 +373,19 @@ fn only_grew_compares_huge_arrays_without_quadratic_work() { assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); } +/// A reference to a generation the object doesn't have reads as null: it is no repeat of the valid +/// reference to the same object number that follows it. +#[test] +fn dss_evidence_skips_a_dead_reference_without_hiding_the_live_one() { + let mut doc = fixture(&[]); + let live = doc.add(Object::Stream(Stream::flate(Dict::new(), b"evidence"))); + let dead = ObjRef::new(live.num, live.generation + 1); + let mut dss = Dict::new(); + dss.set(b"CRLs".to_vec(), Object::Array(vec![Object::Ref(dead), Object::Ref(live)])); + let read: Vec> = DssEvidence::new(&doc, &dss, b"CRLs").collect(); + assert_eq!(read, vec![b"evidence".to_vec()]); +} + #[test] fn dss_evidence_decodes_each_stream_once_within_its_caps() { // One small stream listed 1000 times, then streams that inflate past the evidence cap. diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index fa45ab1ecb..5d24d05dce 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -3369,6 +3369,38 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val ); } +/// Fields that name one signature value are listed each, under their own name, with the verdict +/// the value earned once. +#[test] +fn fields_sharing_a_signature_value_are_listed_with_one_verdict() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + let v = open(&signed).get(pdfcraft_cos::ObjRef { num: 6, generation: 0 }).as_dict().and_then(|d| d.get(b"V").cloned()).unwrap(); + let shared = edit_after(&signed, |doc| { + let mut w = pdfcraft_cos::Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"T".to_vec(), PdfString::text("Twin")); + w.set(b"V".to_vec(), v.clone()); + w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4])); + let w = doc.add(Object::Dict(w)); + let root = doc.root().unwrap(); + let af = doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm").cloned()).unwrap(); + let af = doc.resolve(&af).as_dict().cloned().unwrap(); + let mut fields = af.get(b"Fields").map(|f| doc.resolve(f)).and_then(|f| f.as_array().cloned()).unwrap(); + fields.push(Object::Ref(w)); + let mut af = af; + af.set(b"Fields".to_vec(), Object::Array(fields)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(af))).unwrap(); + }); + let list = signatures(&open(&shared), &shared, &trust); + let (a, b) = (list.iter().find(|s| s.field == "Approval").unwrap(), list.iter().find(|s| s.field == "Twin").unwrap()); + assert!(a.signed && b.signed); + assert_eq!((a.status, &a.modification, &a.signer, a.signed_len), (b.status, &b.modification, &b.signer, b.signed_len)); +} + #[test] fn the_catalogs_perms_do_not_change_under_no_changes() { // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later From 77078864f41f07818624ea98c3d1ddee1201e547 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:02 -0500 Subject: [PATCH 35/46] fix(render): charge inspection text (fields, tooltips, layers, bookmarks, blank comments) against shared allowances (#6015) * fix(render): charge field, tooltip, layer, bookmark and blank-comment text against shared allowances * address review: charge comment text once, keep JS and field-action flags past the budgets * review: count an unlisted field's actions through its whole subtree * review: mark multi-select values the allowance dropped * review: count only real actions, walk each field once, keep name selections * address review: revisit shared field descendants from shallower paths, charge value separators, check every widget for actions * merge main, keep both --- crates/render/src/inspect/annotations.rs | 169 +++++++++++++++++++---- crates/render/src/inspect/core.rs | 68 ++++++++- crates/render/src/inspect/navigation.rs | 2 +- crates/render/src/inspect/resources.rs | 7 +- crates/render/src/inspect/tests.rs | 80 +++++++++++ 5 files changed, 289 insertions(+), 37 deletions(-) diff --git a/crates/render/src/inspect/annotations.rs b/crates/render/src/inspect/annotations.rs index 7313716a2e..673d930743 100644 --- a/crates/render/src/inspect/annotations.rs +++ b/crates/render/src/inspect/annotations.rs @@ -144,12 +144,7 @@ impl<'a> Inspector<'a> { info.attachments.push(self.attachment(fs, AttachmentSource::Annotation { page, index }, text_budget, fallback)); } if subtype == "Link" { - if let Some(link) = self.link(page, index, d, info.uri_base.as_deref(), text_budget, &mut unfit_quads) { - if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { - info.has_javascript = true; - } - info.links.push(link); - } + self.collect_link(info, (page, index), d, text_budget, &mut unfit_quads); continue; } // Not comments: form widgets, pop-ups and non-markup annotations such as the @@ -161,6 +156,20 @@ impl<'a> Inspector<'a> { if comments_full { continue; } + // Text that trims to nothing is not kept, so `retained` never sees it: charge what is + // read, so many comments sharing one huge blank string stop being decoded over and over. + let raw: usize = [&b"T"[..], b"Contents", b"M", b"NM", b"State", b"DA"] + .iter() + .filter_map(|k| match d.get(k).map(|o| self.resolve(o)) { + Ok(Object::String(b, _) | Object::Name(b)) => Some(b.len()), + _ => None, + }) + .fold(0, usize::saturating_add); + // (charged once: below, by the larger of these raw bytes and what the annotation keeps) + if kept.saturating_add(raw) > MAX_ANNOTATION_BYTES { + comments_full = true; + continue; + } let rect = rect4(self.resolve(d.get(b"Rect").unwrap_or(&Object::Null))); let c = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => Some(c), @@ -262,7 +271,7 @@ impl<'a> Inspector<'a> { text_box, popup, }; - kept = kept.saturating_add(retained(&annotation)); + kept = kept.saturating_add(retained(&annotation).max(raw)); if kept > MAX_ANNOTATION_BYTES { comments_full = true; continue; @@ -356,6 +365,23 @@ impl<'a> Inspector<'a> { Some(Link { page, index, rect, quads, target }) } + /// Lists the link annotation `d`, and notes the JavaScript it runs. JavaScript is read from the + /// action, not the listed link: once the text budget is spent the action's name is not kept. + fn collect_link(&self, info: &mut DocInfo, at: (usize, usize), d: &Dictionary, text_budget: &mut usize, unfit_quads: &mut HashSet) { + info.has_javascript |= self.runs_javascript(d); + if let Some(link) = self.link(at.0, at.1, d, info.uri_base.as_deref(), text_budget, unfit_quads) { + if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { + info.has_javascript = true; + } + info.links.push(link); + } + } + + /// Whether the link's own action is a JavaScript action. + fn runs_javascript(&self, d: &Dictionary) -> bool { + d.get(b"A").ok().and_then(|a| self.dict(a)).is_some_and(|a| self.name(a, b"S").as_deref() == Some("JavaScript")) + } + /// What an action does when a link or a bookmark runs it. Only a go-to's `/D` is a place in /// this document; a remote go-to's names a page or destination of another file (ISO 32000-2 /// §12.6.4.3). `None` for an action without a type, or a go-to whose destination is not here. @@ -376,7 +402,7 @@ impl<'a> Inspector<'a> { "SetOCGState" => self.layer_state(a), "GoToR" => self.remote_go_to(a).unwrap_or_else(|| LinkTarget::Other("GoToR".into())), "Launch" => self.launch_pdf(a).unwrap_or_else(|| LinkTarget::Other("Launch".into())), - other => LinkTarget::Other(other.to_string()), + other => LinkTarget::Other(spend(other.as_bytes(), text_budget, |b| String::from_utf8_lossy(b).into_owned()).unwrap_or_default()), }) } @@ -492,6 +518,53 @@ impl<'a> Inspector<'a> { // ── form fields ───────────────────────────────────────────────────────────────────────── + /// Whether `o` or one of its descendants under `/Kids` carries an action (`/A` or `/AA`). It + /// walks the same tree `field` does, bounded the same way: a field left out of the listing for + /// want of text allowance can still have its action on a widget several levels down. + pub(super) fn has_action(&self, o: &Object, depth: u32) -> bool { + self.has_action_seen(o, depth, &mut HashMap::new()) + } + + /// The walk behind [`Self::has_action`]. Each indirect descendant is read again only from a + /// shallower depth than before (the depth limit may have cut its subtree): a field tree whose + /// `/Kids` repeat the same objects would otherwise be walked exponentially, and this reads each + /// object at most once per depth. + fn has_action_seen(&self, o: &Object, depth: u32, seen: &mut HashMap) -> bool { + if depth > 32 { + return false; + } + if let Object::Reference(id) = o { + if seen.get(id).is_some_and(|&first| first <= depth) { + return false; + } + seen.insert(*id, depth); + } + let Some(d) = self.dict(o) else { return false }; + if self.carries_action(d) { + return true; + } + let kids = match d.get(b"Kids").map(|k| self.resolve(k)) { + Ok(Object::Array(k)) => k, + _ => return false, + }; + kids.iter().any(|k| self.has_action_seen(k, depth + 1, seen)) + } + + /// Whether `/A` or `/AA` here holds an action. An editor can leave an empty dictionary (or + /// array) behind, and that is no action: a document with none must not report JavaScript. An + /// `/A` is an action when it carries `/S`; an `/AA` is one when an event maps to such an action. + fn carries_action(&self, d: &Dictionary) -> bool { + let action = |o: &Object| matches!(self.resolve(o), Object::Dictionary(a) if a.get(b"S").is_ok()); + [&b"A"[..], &b"AA"[..]].iter().any(|k| match d.get(k) { + Ok(o) => match self.resolve(o) { + Object::Dictionary(x) => x.get(b"S").is_ok() || x.iter().any(|(_, v)| action(v)), + Object::Array(a) => a.iter().any(action), + _ => false, + }, + Err(_) => false, + }) + } + pub(super) fn field( &self, o: &'a Object, @@ -510,19 +583,35 @@ impl<'a> Inspector<'a> { } let Some(d) = self.dict(o) else { return }; let (parent_name, parent_ft, parent_ff) = parent.unwrap_or(("", None, None)); - let partial = self.text(d, b"T"); - let full = match (&partial, parent_name.is_empty()) { - (Some(t), true) => t.clone(), - (Some(t), false) => format!("{parent_name}.{t}"), - (None, _) => parent_name.to_string(), - }; - let ft = self.name(d, b"FT").or(parent_ft); - let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); // Non-terminal: has kids that are fields (they carry /T). Widgets-only kids are terminal. let mut kids: Vec = match d.get(b"Kids").map(|k| self.resolve(k)) { Ok(Object::Array(k)) => k.clone(), _ => Vec::new(), }; + // A field left unlisted for want of text allowance still has its actions counted, through + // its whole subtree: an action can sit on a descendant widget, not only on the field. + let unlisted = || { + if self.has_action(o, 0) { + self.field_actions.set(true); + } + }; + // A /T that does not fit the allowance lists no field (not one under its parent's name). + let Ok(partial) = self.field_text_within(d, b"T") else { + unlisted(); + return; + }; + let full = match (&partial, parent_name.is_empty()) { + (Some(t), true) => t.clone(), + (Some(t), false) => format!("{parent_name}.{t}"), + (None, _) => parent_name.to_string(), + }; + // Every field copies its parent's full name: charge it, and list no more once it is spent. + if !self.field_charge(full.len()) { + unlisted(); + return; + } + let ft = self.name(d, b"FT").or(parent_ft); + let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); let (field_kids, widget_kids): (Vec, Vec) = kids.iter().cloned().partition(|k| self.dict(k).is_some_and(|kd| kd.get(b"T").is_ok())); let field_kids: Vec<_> = field_kids.iter().filter(|k| !self.not_a_child(walk, o, k)).collect(); @@ -570,17 +659,37 @@ impl<'a> Inspector<'a> { _ => FieldKind::Unknown, }; let value = match d.get(b"V").map(|v| self.resolve(v)) { - Ok(Object::Name(n)) => Some(String::from_utf8_lossy(n).into_owned()), - Ok(Object::String(bytes, _)) => Some(text_string(bytes)), - Ok(Object::Array(a)) => Some( - a.iter() - .filter_map(|x| match self.resolve(x) { - Object::String(bytes, _) => Some(text_string(bytes)), - _ => None, - }) - .collect::>() - .join(", "), - ), + Ok(Object::Name(n)) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + Ok(Object::String(bytes, _)) => self.field_spend(bytes, text_string), + Ok(Object::Array(a)) => { + let mut shown = Vec::new(); + let mut dropped = false; + for x in a { + let text = match self.resolve(x) { + Object::String(bytes, _) => self.field_spend(bytes, text_string), + // A name entry (`/V [/Opt1 (b)]`) is a selection too, as the forms reader + // takes it; leaving it out reported a partial selection as complete. + Object::Name(n) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + _ => continue, + }; + // The ", " between selections is charged to the allowance like the text. + match text { + Some(text) if shown.is_empty() || self.field_charge(2) => shown.push(text), + _ => dropped = true, + } + } + // A selected value the allowance could not hold shows as an omission rather than + // as part of a complete selection (its few bytes are charged, fit or not). + let mut value = shown.join(", "); + if dropped { + self.field_charge(5); + if !value.is_empty() { + value.push_str(", "); + } + value.push('…'); + } + Some(value) + } Ok(Object::Dictionary(_)) if kind == FieldKind::Signature => Some("signed".into()), _ => None, }; @@ -602,9 +711,11 @@ impl<'a> Inspector<'a> { .and_then(|p| self.page_of(p)) .or_else(|| self.find_widget_page(o, &kids)) .or_else(|| named.and_then(|id| self.annotation_pages.get_or_init(|| self.annotation_pages()).get(&id).copied())); - let has_actions = d.get(b"AA").is_ok() || widget.is_some_and(|w| w.get(b"AA").is_ok() || w.get(b"A").is_ok()); + let has_actions = self.carries_action(d) + || widget.is_some_and(|w| self.carries_action(w)) + || kids.iter().any(|k| self.dict(k).is_some_and(|w| self.carries_action(w))); let rect = widget.and_then(|w| w.get(b"Rect").ok()).map(|r| rect4(self.resolve(r))); - out.push(Field { name: full, kind, value, page, tooltip: self.text(d, b"TU"), has_actions, rect }); + out.push(Field { name: full, kind, value, page, tooltip: self.field_text(d, b"TU"), has_actions, rect }); } /// Whether `k`, listed in the `/Kids` of field `o`, is not a child of `o` after all: `o` itself diff --git a/crates/render/src/inspect/core.rs b/crates/render/src/inspect/core.rs index 88dc13c34a..adf3edffbd 100644 --- a/crates/render/src/inspect/core.rs +++ b/crates/render/src/inspect/core.rs @@ -155,16 +155,34 @@ pub(super) struct Inspector<'a> { /// has no `/P`. Searching every page's `/Annots` for each such field was O(fields × /// annotations): a second on a form of 8,000 fields. pub(super) annotation_pages: OnceCell>, + /// What is left of [`MAX_FIELD_TEXT_BYTES`]: names, values and tooltips of form fields and + /// layer names, which many entries can share one string of. + pub(super) field_text: std::cell::Cell, + /// Set when a field with an action went unlisted because the text allowance was spent, so the + /// document still reports JavaScript. + pub(super) field_actions: std::cell::Cell, /// The first page widget naming each field as its `/Parent`, built when a field has no widget /// among its `/Kids`. pub(super) parent_widgets: OnceCell>, } +/// Bytes of field and layer text one inspection reads and keeps together. +pub(super) const MAX_FIELD_TEXT_BYTES: usize = 64 << 20; + impl<'a> Inspector<'a> { pub(super) fn new(doc: &'a dyn Structure) -> Self { let page_ids = doc.page_ids(); let page_index = page_ids.iter().copied().enumerate().map(|(index, id)| (id, index)).collect(); - let mut me = Self { doc, page_ids, page_index, named: HashMap::new(), annotation_pages: OnceCell::new(), parent_widgets: OnceCell::new() }; + let mut me = Self { + doc, + page_ids, + page_index, + named: HashMap::new(), + annotation_pages: OnceCell::new(), + field_text: MAX_FIELD_TEXT_BYTES.into(), + field_actions: false.into(), + parent_widgets: OnceCell::new(), + }; let mut named = HashMap::new(); if let Some(tree) = doc.catalog().ok().and_then(|c| c.get(b"Names").ok()).and_then(|o| me.dict(o)).and_then(|n| n.get(b"Dests").ok()).and_then(|o| me.dict(o)) @@ -260,7 +278,7 @@ impl<'a> Inspector<'a> { } } } - info.has_javascript |= info.fields.iter().any(|f| f.has_actions); + info.has_javascript |= info.fields.iter().any(|f| f.has_actions) || self.field_actions.get(); let needs_rendering = catalog.get(b"NeedsRendering").ok().and_then(|o| self.resolve(o).as_bool().ok()).unwrap_or(false); if let Some(xfa) = catalog.get(b"AcroForm").ok().and_then(|o| self.dict(o)).and_then(|form| form.get(b"XFA").ok()) { // A form with no fields is taken to be laid out from its template, unless its @@ -351,11 +369,49 @@ impl<'a> Inspector<'a> { /// Whitespace-trimmed metadata, spent from `budget` as [`spend`] does, so text that trims /// short or to nothing still spends its raw length. + /// [`Self::budgeted_text`] against the shared field and layer text allowance. + pub(super) fn field_text(&self, d: &Dictionary, key: &[u8]) -> Option { + let mut left = self.field_text.get(); + let text = self.budgeted_text(d, key, &mut left); + self.field_text.set(left); + text + } + + /// Converts `bytes` against the field and layer text allowance, as [`spend`] does. + pub(super) fn field_spend(&self, bytes: &[u8], convert: impl FnOnce(&[u8]) -> String) -> Option { + let mut left = self.field_text.get(); + let text = spend(bytes, &mut left, convert); + self.field_text.set(left); + text + } + + /// Spends `n` bytes of the field and layer text allowance; `false` when too few are left. + pub(super) fn field_charge(&self, n: usize) -> bool { + let left = self.field_text.get(); + self.field_text.set(left.saturating_sub(n)); + n <= left + } + pub(super) fn budgeted_text(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Option { - let o = self.resolve(d.get(key).ok()?); - let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return None }; - let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default())?; - (!text.is_empty()).then_some(text) + self.budgeted_text_within(d, key, budget).ok().flatten() + } + + /// [`Self::budgeted_text`], but `Err` when the text is there and does not fit `budget` + /// (against `Ok(None)`: no text, or none that trims to anything). + pub(super) fn budgeted_text_within(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Result, ()> { + let Ok(o) = d.get(key) else { return Ok(None) }; + let o = self.resolve(o); + let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return Ok(None) }; + let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default()).ok_or(())?; + Ok((!text.is_empty()).then_some(text)) + } + + /// [`Self::field_text`], but `Err` when the text does not fit the allowance. + pub(super) fn field_text_within(&self, d: &Dictionary, key: &[u8]) -> Result, ()> { + let mut left = self.field_text.get(); + let text = self.budgeted_text_within(d, key, &mut left); + self.field_text.set(left); + text } pub(super) fn name(&self, d: &Dictionary, key: &[u8]) -> Option { diff --git a/crates/render/src/inspect/navigation.rs b/crates/render/src/inspect/navigation.rs index c5f2a1be86..f680b46246 100644 --- a/crates/render/src/inspect/navigation.rs +++ b/crates/render/src/inspect/navigation.rs @@ -250,7 +250,7 @@ impl<'a> Inspector<'a> { None => {} } let open = d.get(b"Count").ok().and_then(|c| c.as_i64().ok()).is_some_and(|c| c > 0); - let title = self.text(d, b"Title").unwrap_or_default(); + let title = self.field_text(d, b"Title").unwrap_or_default(); let color = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => outline_color(c.iter().map(|v| self.resolve(v).as_float().ok())), _ => None, diff --git a/crates/render/src/inspect/resources.rs b/crates/render/src/inspect/resources.rs index 3dbc1c665f..6d90f715e1 100644 --- a/crates/render/src/inspect/resources.rs +++ b/crates/render/src/inspect/resources.rs @@ -121,9 +121,14 @@ impl<'a> Inspector<'a> { .unwrap_or_default(); let auto = config.map(|c| self.view_auto_state(c)).unwrap_or_default(); let Ok(Object::Array(ocgs)) = props.get(b"OCGs").map(|o| self.resolve(o)) else { return }; + let mut seen = HashSet::new(); for g in ocgs { let Some(d) = self.dict(g) else { continue }; let id = g.as_reference().ok(); + // A group listed twice is one layer (and its name is read once). + if id.is_some_and(|id| !seen.insert(id)) { + continue; + } // The View auto state, when it manages the group, then the configuration. let visible = match id { Some(id) if let Some(&state) = auto.get(&id) => state, @@ -132,7 +137,7 @@ impl<'a> Inspector<'a> { _ => !base_off, }; let Some(id) = id else { continue }; - out.push(Layer { id, name: self.text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); + out.push(Layer { id, name: self.field_text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); } } diff --git a/crates/render/src/inspect/tests.rs b/crates/render/src/inspect/tests.rs index 68486d8097..c908b241cf 100644 --- a/crates/render/src/inspect/tests.rs +++ b/crates/render/src/inspect/tests.rs @@ -1161,6 +1161,50 @@ fn repeated_annotations_are_listed_once_within_a_byte_budget() { assert!(notes.iter().map(retained).sum::() <= MAX_ANNOTATION_BYTES); } +/// Blank text is not kept, so only reading it is charged: many comments sharing one large blank +/// string stop being built at the budget instead of decoding it once each. +#[test] +fn comments_sharing_a_blank_string_stop_at_the_read_budget() { + let small = "<< /Subtype /Text /Rect [0 0 1 1] /Contents 7 0 R >> ".repeat(1200); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Annots [{small}] >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + " ".repeat(64 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(!info.annotations.is_empty() && info.annotations.len() < 1200, "{}", info.annotations.len()); +} + +/// Many fields sharing one large tooltip, and one layer listed many times, are read within a byte +/// allowance rather than once each. +#[test] +fn fields_sharing_a_tooltip_and_repeated_layers_stop_at_the_text_budget() { + let fields = (0..1300).map(|i| format!("<< /FT /Tx /T (f{i}) /TU 7 0 R /V 7 0 R >> ")).collect::(); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> \ + /OCProperties << /OCGs [{}] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 6 0 obj << /Type /OCG /Name (Ink) >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + "6 0 R ".repeat(1000), + "x".repeat(32 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + let kept: usize = info.fields.iter().map(|f| f.tooltip.as_ref().map_or(0, String::len) + f.value.as_ref().map_or(0, String::len)).sum(); + assert!(kept > 0 && kept <= super::core::MAX_FIELD_TEXT_BYTES, "{kept}"); + assert!(info.fields.iter().any(|f| f.tooltip.is_none()), "text past the allowance is left out"); + assert_eq!(info.layers.len(), 1); +} + /// A link reached through an indirect object that holds only a reference to it is the same /// annotation as the link itself, and so is a page array reached the same way: each is listed once. #[test] @@ -2686,6 +2730,42 @@ fn inspection_keeps_no_private_page_data() { assert_eq!(report(with), report(without), "what inspection reports does not change"); } +/// A field with several widgets reports an action on any of them, not only the first, and so does +/// the document's JavaScript flag. +#[test] +fn a_field_reports_an_action_on_a_later_widget() { + let pdf = "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 4 0 obj << /FT /Tx /T (f) /Kids [5 0 R 6 0 R] >> endobj\n\ + 5 0 obj << /Subtype /Widget /Rect [0 0 1 1] >> endobj\n\ + 6 0 obj << /Subtype /Widget /Rect [0 0 1 1] /A << /S /JavaScript /JS (app.alert(1)) >> >> endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF"; + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(info.fields.iter().any(|f| f.name == "f" && f.has_actions), "{:?}", info.fields); + assert!(info.has_javascript); +} + +/// A shared descendant first reached at the depth limit (its action-bearing child cut) is read +/// again from a shallower path, so the action is still found. +#[test] +fn an_action_below_a_shared_field_first_met_deep_is_found_from_a_shallower_path() { + // 1 is the root, 2..=32 a chain of 31 fields, 33 the shared field, 34 holds the action, + // 35 a short path to 33. + let mut bodies = vec!["<< /Kids [2 0 R 35 0 R] >>".to_string()]; + for i in 2..=32 { + bodies.push(format!("<< /Kids [{} 0 R] >>", i + 1)); + } + bodies.push("<< /Kids [34 0 R] >>".into()); + bodies.push("<< /A << /S /JavaScript /JS (x) >> >>".into()); + bodies.push("<< /Kids [33 0 R] >>".into()); + let doc = Document::load_mem(&pdf_of(&bodies)).expect("loads"); + assert!(Inspector::new(&doc).has_action(&Object::Reference((1, 0)), 0)); +} + /// A `/Kids` entry that loops back, or names a root field that doesn't name this one as its /// `/Parent`, is not a child (ISO 32000-2 §12.7.3, Table 224; §12.7.4.1, Table 226): the field /// keeps its other kids and is listed with them. From abec4e3feea96d3566c7c026d5d137172fdf2596 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:29 -0500 Subject: [PATCH 36/46] fix(js): bound array text, month parsing, name registration and list work (#5883) * fix(js): bound array text, month parsing, name registration and list work - strings() (field value and colour setters, resetForm) copied every array entry into its own String with no total: one long string listed a million times was copied a million times. The copies together are now held to the run's MAX_PAYLOAD byte budget. - FormCalc parse_date copied and lowercased the rest of the text for every month token of the picture, quadratic in both; it reads at most the longest month name now. - XFA global registration checked each child name against a Vec of every name defined so far, quadratic in a wide form; it is a HashSet. - FormCalc gather charged one step for a whole collection and list_push charged nothing for a duplicate it dropped, so .all on same-named siblings did quadratic work for one step a pair; every node visited costs a step. - add_instance reserved only the new root's SOM before copying a subtree; the copy's nodes, names and SOMs are charged to the byte budget first. * address review: gather charges each call; check an array entry's length before copying it * Hold an array entry to its widest escaped size before copying it The pre-copy check compared the payload budget against a JS string's UTF-16 length, which is shorter than the escaped UTF-8 form it is copied to: a string of quotes or non-ASCII text could allocate up to six times the budget before the post-check rejected it. The check now uses the widest escaped width, so the copy cannot exceed the budget it is checked against. * review: measure an array entry's escaped width before refusing it * address review: count an array entry's escaped width without copying it * address review: charge retained field text and colours to a run-wide budget * fmt * fmt * merge main, keep both --- crates/js/src/document.rs | 11 +++++--- crates/js/src/formcalc/access.rs | 13 ++++++--- crates/js/src/formcalc/dates.rs | 8 ++++-- crates/js/src/host.rs | 40 ++++++++++++++++++++++++--- crates/js/src/run.rs | 1 + crates/js/src/tests.rs | 46 ++++++++++++++++++++++++++++++++ crates/js/src/xfa/globals.rs | 6 +++-- crates/js/src/xfa/instances.rs | 16 ++++++++++- 8 files changed, 126 insertions(+), 15 deletions(-) diff --git a/crates/js/src/document.rs b/crates/js/src/document.rs index f13c9434e9..cccf89a7a2 100644 --- a/crates/js/src/document.rs +++ b/crates/js/src/document.rs @@ -1,4 +1,6 @@ -use super::host::{FieldKey, arg, colour, error, function, host, read, s, set_value, strings, text, text_line, value_of, write}; +use super::host::{ + FieldKey, arg, charge, colour, error, function, host, kind_of, read, s, strings, text, text_bytes, text_line, value_for, value_of, write, +}; use super::model::{DISPLAY_HIDDEN, DISPLAY_VISIBLE, FieldType, Request}; use super::{Attribute, Context, JsArray, JsObject, JsResult, JsString, JsValue, NativeFunction, ObjectInitializer, js_string}; @@ -41,8 +43,9 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let value_get = getter!(ctx, n, |f, c| value_of(&f, c)); let value_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { - let vals = strings(&arg(args, 0), c)?; - write(c, name, |f| set_value(f, vals))?; + let vals = value_for(kind_of(c, name)?, strings(&arg(args, 0), c)?); + charge(c, name, |f| text_bytes(&f.value), text_bytes(&vals))?; + write(c, name, |f| f.value = vals)?; Ok(JsValue::undefined()) } function(ctx, NativeFunction::from_copy_closure_with_captures(set, n.clone())) @@ -104,6 +107,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let text_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.text_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.text_color = Some(v.into()))?; Ok(JsValue::undefined()) } @@ -113,6 +117,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let fill_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.fill_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.fill_color = Some(v.into()))?; Ok(JsValue::undefined()) } diff --git a/crates/js/src/formcalc/access.rs b/crates/js/src/formcalc/access.rs index 26de7ba9eb..3c323ca088 100644 --- a/crates/js/src/formcalc/access.rs +++ b/crates/js/src/formcalc/access.rs @@ -33,9 +33,12 @@ impl Interp<'_> { Ok(()) } - /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. + /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. Every node visited costs a + /// step, kept or not, and the call itself one more: a search that matches nothing still + /// costs work when it is repeated over a long list. pub(super) fn gather(&mut self, out: &mut Vec, seen: &mut HashSet, more: impl IntoIterator) -> R<()> { for i in more { + self.step()?; if seen.insert(i) { out.push(i); if out.len() > MAX_LIST { @@ -43,7 +46,7 @@ impl Interp<'_> { } } } - self.charge(1) + self.step() } /// The nodes of `v`, each once. @@ -294,8 +297,10 @@ impl Interp<'_> { } } - /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. + /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. A duplicate costs its step + /// too: `all` on a list of same-named siblings makes one for every pair. fn list_push(&mut self, out: &mut Vec, seen: &mut HashSet, v: Val) -> R<()> { + self.step()?; if let Val::Node(i) = v && !seen.insert(i) { @@ -305,7 +310,7 @@ impl Interp<'_> { if out.len() > MAX_LIST { return Err(LIST_TOO_LONG.into()); } - self.charge(1) + Ok(()) } /// Property `name` of form object `i` as FormCalc reads it: its value, attributes, diff --git a/crates/js/src/formcalc/dates.rs b/crates/js/src/formcalc/dates.rs index 4d9befeab2..f201a23525 100644 --- a/crates/js/src/formcalc/dates.rs +++ b/crates/js/src/formcalc/dates.rs @@ -56,6 +56,9 @@ const CENTURY_SPLIT: i64 = 30; const MONTHS: [&str; 12] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; +/// Characters of the longest month name: all a month token needs to read of the text. +const LONGEST_MONTH: usize = 9; + /// Parse `text` as a date by `picture` (`YYYY-MM-DD`, `MM/DD/YYYY`, `MMM D, YYYY`, …). pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { let pic: Vec = picture.chars().collect(); @@ -71,8 +74,9 @@ pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { } i += run; if c == 'M' && run >= 3 { - // Month name. - let rest: String = txt.get(j..).unwrap_or(&[]).iter().collect(); + // Month name: only as much of the text as a name can be, so a picture of many + // month tokens does not copy the rest of a long text for each of them. + let rest: String = txt.get(j..).unwrap_or(&[]).iter().take(LONGEST_MONTH).collect(); let lower = rest.to_ascii_lowercase(); let found = MONTHS .iter() diff --git a/crates/js/src/host.rs b/crates/js/src/host.rs index 5966e63545..d829041ce6 100644 --- a/crates/js/src/host.rs +++ b/crates/js/src/host.rs @@ -27,6 +27,10 @@ pub(crate) struct Host { pub(crate) reset_limit_exceeded: bool, /// Bytes of text kept in alerts, console lines and requests, against [`xfa::MAX_PAYLOAD`]. pub(crate) kept: usize, + /// Bytes of text the script has added to the fields' values and colours, net of what it + /// replaced, against [`xfa::MAX_PAYLOAD`]: each assignment alone is capped, so without this + /// many fields could each hold a maximal array. + pub(crate) retained: usize, /// Alerts, console lines and requests the limits left out. pub(crate) dropped: usize, } @@ -262,12 +266,42 @@ pub(crate) fn value_of(f: &FieldState, ctx: &mut Context) -> JsValue { } } -pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { - f.value = match f.kind { +/// Bytes of text in `v`, counting each string's heap size and its handle. +pub(crate) fn text_bytes(v: &[String]) -> usize { + v.iter().fold(0usize, |n, t| n.saturating_add(t.len()).saturating_add(std::mem::size_of::())) +} + +/// Charge the replacement of the text `old` selects from field `name` with `new_bytes` of text to +/// the run's retained-text budget; an assignment the budget has no room for is an error and +/// charges nothing. +pub(crate) fn charge(ctx: &Context, name: &FieldKey, old: impl FnOnce(&FieldState) -> usize, new_bytes: usize) -> JsResult<()> { + let i = field_index(ctx, name)?; + let mut h = host(ctx)?.borrow_mut(); + let freed = old(&h.fields[i]); + let total = h.retained.saturating_sub(freed).saturating_add(new_bytes); + if total > xfa::MAX_PAYLOAD { + return Err(error(format!("the fields' values and colours hold more than {} bytes of text", xfa::MAX_PAYLOAD))); + } + h.retained = total; + Ok(()) +} + +pub(crate) fn kind_of(ctx: &Context, name: &FieldKey) -> JsResult { + let i = field_index(ctx, name)?; + Ok(host(ctx)?.borrow().fields[i].kind) +} + +/// What a field of `kind` keeps of the values assigned to it. +pub(crate) fn value_for(kind: FieldType, vals: Vec) -> Vec { + match kind { FieldType::CheckBox | FieldType::RadioButton => vals.into_iter().filter(|v| v != "Off" && !v.is_empty()).take(1).collect(), FieldType::ListBox => vals, _ => vals.into_iter().take(1).collect(), - }; + } +} + +pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { + f.value = value_for(f.kind, vals); } #[cfg(test)] diff --git a/crates/js/src/run.rs b/crates/js/src/run.rs index ececa107a3..7019bd768c 100644 --- a/crates/js/src/run.rs +++ b/crates/js/src/run.rs @@ -270,6 +270,7 @@ pub(crate) fn run_in_context( reset_calls: setup.reset_calls, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, })); let mut out = Outcome { rc: true, value: event.value.clone(), change: event.change.clone(), ..Default::default() }; diff --git a/crates/js/src/tests.rs b/crates/js/src/tests.rs index 1a41afeea8..ece4443b7d 100644 --- a/crates/js/src/tests.rs +++ b/crates/js/src/tests.rs @@ -87,6 +87,17 @@ fn calculate_scripts_read_fields_and_set_the_value() { assert!(o.rc); } +/// An array of field values copies each entry: one long string listed many times is held to +/// the run's byte budget, not copied once per listing. +#[test] +fn an_array_of_values_stops_at_the_byte_budget() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < N; i++) a.push(s); getField('qty').value = a;"; + let o = go(&fill.replace('N', "600"), &Event::field("Calculate", "total", "")); + assert!(o.error.as_deref().is_some_and(|e| e.contains("byte limit exceeded")), "{:?}", o.error); + let o = go(&fill.replace('N', "400"), &Event::field("Calculate", "total", "")); + assert_eq!(o.error, None); +} + /// `event.value` reads and writes as a plain value, numbers included, and the outcome says whether /// the script assigned it, even to the value it already had. #[test] @@ -528,6 +539,7 @@ fn app_alert_titles_are_clipped_and_charged() { reset_calls: 0, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, }; h.alert(Message { title: Some("Order form".into()), ..Message::plain("Saved") }); @@ -1113,6 +1125,24 @@ mod xfa_model { assert_eq!(o.result.as_deref(), Some("7")); } + /// A copy of a subform costs its nodes, names and SOMs: instances of one with a long name + /// stop at the run's byte budget, well before the instance cap. + #[test] + fn instances_of_a_subform_with_long_names_stop_at_the_byte_budget() { + let mut f = form(); + let table = &mut f.children[0].children[4]; + assert_eq!(table.name, "table"); + for row in &mut table.children { + row.children.push(field(&"n".repeat(1 << 20), "", "", false)); + } + let ev = XfaEvent { activity: "click".into(), target: "form1[0].page1[0].go[0]".into(), ..Default::default() }; + let o = run_xfa("for (var i = 0; i < 100; i++) table._row.addInstance(1); table._row.count", &ev, &XfaDoc::default(), &f, Limits::default()); + assert_eq!(o.error, None, "{:?}", o.error); + let count: usize = o.result.as_deref().unwrap_or_default().parse().expect("a count"); + assert!((3..50).contains(&count), "{count} instances"); + assert_eq!(o.effects.iter().filter(|e| matches!(e, XfaEffect::AddInstance { .. })).count(), count - 2); + } + /// On screen every page is a sheet of its own: `xfa.layout.sheet` counts it from 0, as /// `absPage` does (forms show `sheet(this) + 1`), and `sheetCount` is the page count. #[test] @@ -1625,6 +1655,10 @@ mod formcalc { #[test] fn date2num_rejects_days_past_the_end_of_the_month() { + // A month name is read from the text's next few characters only, whatever follows. + let serial = calc(r#"Date2Num("2024-09-01")"#); + assert_eq!(calc(r#"Date2Num("September 1, 2024, and a note that runs on", "MMMM D, YYYY")"#), serial); + assert_eq!(calc(r#"Date2Num("Sep 1, 2024", "MMM D, YYYY")"#), serial); assert_eq!(calc(r#"Date2Num("2023-02-30")"#), "0"); assert_eq!(calc(r#"Date2Num("31/04/2024", "DD/MM/YYYY")"#), "0"); assert_eq!(calc(r#"Date2Num("2023-02-29")"#), "0"); @@ -2641,6 +2675,18 @@ fn a_plain_form_sees_the_xfa_version_but_not_installed() { assert_eq!(o.alerts, [Message::plain("old viewer")]); } +/// Arrays assigned to several fields are held to one run-wide budget, and replacing a field's +/// array frees its charge. +#[test] +fn retained_array_text_is_charged_across_fields() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < 320; i++) a.push(s);"; + let one = |tail: &str| go(&format!("{fill} {tail}"), &Event::field("Calculate", "total", "")); + let o = one("getField('qty').textColor = a; getField('qty').textColor = a; getField('zip').textColor = ['RGB'];"); + assert_eq!(o.error, None, "replacing an array frees its charge: {:?}", o.error); + let o = one("getField('qty').textColor = a; getField('zip').fillColor = a;"); + assert!(o.error.as_deref().is_some_and(|e| e.contains("hold more than")), "{:?}", o.error); +} + #[test] fn string_arrays_refuse_aggregate_text_before_request_clipping() { let o = go("resetForm(new Array(65).fill('x'.repeat(1 << 19)));", &Event::doc("Console")); diff --git a/crates/js/src/xfa/globals.rs b/crates/js/src/xfa/globals.rs index de57d7fd49..46dc8a713b 100644 --- a/crates/js/src/xfa/globals.rs +++ b/crates/js/src/xfa/globals.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use super::host::host; use super::instances::instance_manager; use super::model::{MAX_DEPTH, XfaDoc, XfaEffect, XfaEvent, clip, payload, reset_names}; @@ -379,7 +381,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current "DataView", "Intl", ]; - let mut defined: Vec = RESERVED.iter().map(ToString::to_string).collect(); + let mut defined: HashSet = RESERVED.iter().map(ToString::to_string).collect(); for scope in scopes { let kids: Vec<(usize, String, bool)> = { let h = host(ctx)?.borrow(); @@ -398,7 +400,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current let im = instance_manager(ctx, scope, &name)?; ctx.register_global_property(JsString::from(format!("_{name}").as_str()), im, rw)?; } - defined.push(name); + defined.insert(name); } } Ok(()) diff --git a/crates/js/src/xfa/instances.rs b/crates/js/src/xfa/instances.rs index 2985794dc3..c737ea9db9 100644 --- a/crates/js/src/xfa/instances.rs +++ b/crates/js/src/xfa/instances.rs @@ -137,6 +137,17 @@ fn child_som(parent_som: String, name: &str, index: usize) -> String { } } +/// Bytes a copy of `src`'s subtree keeps: a node, its name and its SOM (the copy's SOMs differ +/// in their indices only) for every node [`clone_subtree`] copies. +fn subtree_bytes(h: &XHost, src: usize, depth: usize) -> usize { + let Some(n) = h.nodes.get(src).filter(|_| depth <= MAX_DEPTH) else { return 0 }; + let own = std::mem::size_of::().saturating_add(n.name.len()).saturating_add(n.som.len()); + n.children + .iter() + .filter(|&&k| !h.nodes.get(k).is_some_and(|c| c.gone || c.repeatable && c.index > 0)) + .fold(own, |sum, &k| sum.saturating_add(subtree_bytes(h, k, depth + 1))) +} + /// A copy of `src`'s subtree (values cleared) under `parent`, returning the copy's index. fn clone_subtree(h: &mut XHost, src: usize, parent: Option, depth: usize) -> Option { if depth > MAX_DEPTH || h.nodes.len() >= MAX_NODES { @@ -214,7 +225,10 @@ impl XHost { let name = h.nodes.get(first).map(|n| n.name.to_string()).unwrap_or_default(); let parent_som = h.nodes.get(parent).map(|p| p.som.to_string()).unwrap_or_default(); let som = child_som(parent_som, &name, live.len()); - if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len()) { + // The copy's nodes, their names and SOMs, are charged to the run's byte budget before + // they are made: a subtree with long names must not be copied past it. + let copied = subtree_bytes(h, first, 0); + if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len().saturating_add(copied)) { return None; } let new = clone_subtree(h, first, Some(parent), 0)?; From 69936f1d1f2f589a907941fbc658ed3a75eea56b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:56 -0500 Subject: [PATCH 37/46] fix(engine): run the renderer redaction check on redactions applied inside a batch (#5977) * fix(engine): run the renderer redaction check on redactions applied inside a batch * Resolve redaction marks by page object and measure the edited document The renderer check looked marks up by the page index they had when they were drawn, so a MovePages later in the same batch pointed them at another page's text, and it measured glyphs with page geometry from before the edit. Each mark now carries the page object it was placed on, and the geometry is read back from the document as the batch wrote it. * review: skip marks whose page the batch deleted * review: group marks by page once and refuse an ambiguous page tree * address review: refuse a batch that deletes a marked page while adding pages * merge main, keep both * address review: a batch that applies redactions and then copies a page is refused --- crates/engine/src/document.rs | 47 ++++++++++++++------ crates/engine/src/edit/run.rs | 27 +++++++++++- crates/engine/src/session/editing.rs | 20 ++++++--- crates/engine/src/tests.rs | 65 +++++++++++++++++++++++++++- crates/redact/src/lib.rs | 6 +++ 5 files changed, 142 insertions(+), 23 deletions(-) diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index 602410922f..805e089dd7 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -958,41 +958,60 @@ pub(crate) fn recovered_render_password(cos: &pdfcraft_cos::Document) -> Option< /// the document as it was. pub(crate) fn shown_under_redactions( after: &pdfcraft_cos::Document, - marks: &[pdfcraft_redact::Mark], + marks: &[(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)], overlays: &[pdfcraft_cos::ObjRef], - pages: &[pdfcraft_render::PageInfo], password: Option<&str>, ) -> Result<(), EditError> { use pdfcraft_cos::Object; use pdfcraft_redact::RedactError; - let mut marked: Vec = marks.iter().map(|m| m.page).collect(); + let mut shown = after.clone(); + let model = pdfcraft_model::pages(&shown); + // A batch can move pages after the marks were collected: a mark is found by the page object it + // was placed on, not by the index that page had when it was drawn. A page the batch deleted + // takes its marks with it, so they are not checked against whatever page now holds the index. + // One index, built once: each mark is resolved against it instead of scanning every page, + // and the marks are grouped by resolved page in the same pass. + let mut index: std::collections::HashMap = std::collections::HashMap::new(); + for (i, p) in model.iter().enumerate() { + index.entry(p.obj).or_insert(i); + } + let mut marked: Vec = Vec::new(); + let mut rects_of: std::collections::HashMap> = std::collections::HashMap::new(); + for (obj, m) in marks { + let Some(&pi) = index.get(obj) else { continue }; + if !rects_of.contains_key(&pi) { + marked.push(pi); + } + rects_of.entry(pi).or_default().extend(m.rects.iter().map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])])); + } marked.sort_unstable(); - marked.dedup(); if marked.is_empty() { return Ok(()); } - let mut shown = after.clone(); - let model = pdfcraft_model::pages(&shown); for &pi in &marked { let Some(page) = model.get(pi) else { continue }; let Some(Object::Array(list)) = page.dict.get(b"Contents").map(|c| (*shown.resolve(c)).clone()) else { continue }; let kept: Vec = list.into_iter().filter(|o| !o.as_ref().is_some_and(|r| overlays.contains(&r))).collect(); shown.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(kept))).map_err(|e| EditError::Write(e.to_string()))?; } - let bytes = write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?; + let bytes = Arc::new(write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?); + // Page geometry comes from the document as it is now written: a page moved earlier in the batch + // brings its own size and rotation with it. + let pages = pdfcraft_render::inspect(bytes.clone(), password).map_err(|e| EditError::Write(e.to_string()))?.pages; + // The model counts one entry per `/Kids` entry; the renderer one per page it draws. If they + // disagree, a mark's index could land on another page's text, and the check cannot confirm + // the redaction. + if pages.len() != model.len() { + return Err(RedactError::AmbiguousPages.into()); + } let config = RenderConfig { password: password.map(Arc::from), ..Default::default() }; - let mut renderer = pdfcraft_render::PageRenderer::new(Arc::new(bytes), config); + let mut renderer = pdfcraft_render::PageRenderer::new(bytes, config); let mut residue = 0; for &pi in &marked { let request = pdfcraft_render::RenderRequest { page: pi, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; // A page that cannot be read back cannot be confirmed clean. let (Some(info), Some(text)) = (pages.get(pi), renderer.render(request).text) else { return Err(RedactError::Unreadable(pi + 1).into()) }; - let rects: Vec<[f64; 4]> = marks - .iter() - .filter(|m| m.page == pi) - .flat_map(|m| m.rects.iter()) - .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]) - .collect(); + let rects = rects_of.get(&pi).cloned().unwrap_or_default(); for glyph in text.glyphs.iter().filter(|g| !g.text.trim().is_empty()) { let [x0, y0, x1, y1] = info.view_rect_to_user(glyph.rect).map(f64::from); let area = (x1 - x0) * (y1 - y0); diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 57e5729fc8..0a7a348fca 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -253,6 +253,11 @@ pub(crate) struct EditCtx { /// The overlay streams the edit's applied redactions added, which alone the renderer's check /// leaves out. pub(crate) redaction_overlays: Vec, + /// The marks the edit's applied redactions took out (a batch's included), as they were when + /// each apply ran, for the renderer's check. + /// Each applied mark with the page object it was drawn on, which a later page move changes the + /// index of but not the identity. + pub(crate) redaction_marks: Vec<(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)>, /// What the edit's applied redactions left as it was that still holds the removed text. pub(crate) redaction_elsewhere: Option, /// What became of the PDF/A and PDF/UA identification when the edit's applied redactions @@ -310,6 +315,7 @@ impl EditCtx { rng: getrandom::fill, renamed_fields: Vec::new(), redaction_overlays: Vec::new(), + redaction_marks: Vec::new(), redaction_elsewhere: None, redaction_identification: pdfcraft_redact::Identification::default(), prepared: None, @@ -529,7 +535,17 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_organize::insert_blank_page(doc, *at, *width, *height)?; } Edit::SetInfo { key, value } => pdfcraft_organize::set_info(doc, key, value)?, - Edit::DuplicatePages { pages, at } => pdfcraft_organize::duplicate_pages(doc, pages, *at)?, + Edit::DuplicatePages { pages, at } => { + // A copy of a page cannot be tied back to the marks an earlier redaction of this edit + // applied: text put back under a mark would sit on the copy, unchecked. Fail closed. + if !cx.redaction_marks.is_empty() { + return Err(EditError::Invalid( + "a batch that applies redactions can't also copy pages: the copies can't be checked against the marks; the document was not changed" + .into(), + )); + } + pdfcraft_organize::duplicate_pages(doc, pages, *at)? + } Edit::ReplacePages { pages, name, bytes, src_pages } => { let src = crate::sources::open_source_into(name, bytes, doc)?; pdfcraft_organize::replace_pages(doc, pages, &src, src_pages)?; @@ -991,8 +1007,15 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_edit::add_content(doc, *page, &content)?; } Edit::ApplyRedactions { pages } => { + let model = pdfcraft_model::pages(doc); + cx.redaction_marks.extend( + pdfcraft_redact::marks(doc) + .into_iter() + .filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))) + .filter_map(|m| model.get(m.page).map(|p| (p.obj, m))), + ); let report = pdfcraft_redact::apply(doc, pages.as_deref())?; - cx.redaction_overlays = report.overlays; + cx.redaction_overlays.extend(report.overlays); cx.redaction_elsewhere = Some(report.still_holding); cx.redaction_identification = report.identification; } diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index 4a6f372fb4..a039f56798 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -185,6 +185,7 @@ impl Session { } cx.renamed_fields.clear(); cx.redaction_overlays.clear(); + cx.redaction_marks.clear(); // `next` is a copy: if the edit fails or crashes, the document is unchanged. run_checked_edit(&mut next, &edit, &mut cx)?; // A field's enter or exit scripts that changed nothing leave no edit behind (no undo step, @@ -195,11 +196,20 @@ impl Session { return Ok(None); } // Redaction removed what its own interpreter places under the marks. The marks were drawn - // over the page as the renderer shows it, so the renderer has the last word. - if let Edit::ApplyRedactions { pages } = &edit { - let marks: Vec = - pdfcraft_redact::marks(&editor.cos).into_iter().filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))).collect(); - guard(|| shown_under_redactions(&next, &marks, &cx.redaction_overlays, &doc.info.pages, doc.password.as_deref())) + // over the page as the renderer shows it, so the renderer has the last word: on every + // apply the edit ran, a batch's included. + if !cx.redaction_marks.is_empty() { + // A batch that deletes a marked page and adds pages cannot be tied back to the marks: + // a copy of a page may carry the text the redaction removed from the original only. + // Fail closed. + let before: std::collections::HashSet<_> = pdfcraft_model::pages(&editor.cos).iter().map(|p| p.obj).collect(); + let after: std::collections::HashSet<_> = pdfcraft_model::pages(&next).iter().map(|p| p.obj).collect(); + if cx.redaction_marks.iter().any(|(o, _)| !after.contains(o)) && after.iter().any(|o| !before.contains(o)) { + return Err(EditError::Invalid( + "a batch that deletes a page with a redaction mark and adds pages can't be verified; the document was not changed".into(), + )); + } + guard(|| shown_under_redactions(&next, &cx.redaction_marks, &cx.redaction_overlays, doc.password.as_deref())) .unwrap_or_else(|m| Err(EditError::Invalid(format!("{} failed unexpectedly ({m}); the document was not changed", edit.label()))))?; } // Renamed fields are said, not slipped in: a repair-log note (with the caveat that diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4e24a4f608..91b00df4cc 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -3542,9 +3542,10 @@ fn the_renderer_check_refuses_text_still_shown_under_a_mark() { mark_shown_text(&mut s, id, "SECRET42"); let doc = s.get(id).unwrap(); let mut cos = doc.editor.as_ref().unwrap().cos.clone(); - let marks = pdfcraft_redact::marks(&cos); + let model = pdfcraft_model::pages(&cos); + let marks: Vec<_> = pdfcraft_redact::marks(&cos).into_iter().filter_map(|m| model.get(m.page).map(|p| (p.obj, m))).collect(); let refused = |cos: &pdfcraft_cos::Document| { - let shown = shown_under_redactions(cos, &marks, &[], &doc.info.pages, None); + let shown = shown_under_redactions(cos, &marks, &[], None); matches!(shown, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0) }; assert!(refused(&cos)); @@ -13778,6 +13779,66 @@ fn image_transform_engine_refuses_singular_matrices_without_mutation() { } } +/// The renderer's check covers a redaction applied inside a batch too, with the marks as they +/// were when it ran: one the batch itself added is checked, and text a later edit of the batch +/// puts back under a mark refuses the whole batch. +#[test] +fn redactions_applied_in_a_batch_get_the_renderer_check() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + // Text added where the mark was shows under it again: the renderer sees it and refuses. + let under = AddedText { rect: [150.0, 130.0, 300.0, 160.0], text: "SECRET42 SECRET42".into(), ..Default::default() }; + let refused = s.apply(id, Edit::Batch { label: "Redact then write".into(), edits: vec![apply.clone(), Edit::AddText { page: 0, text: under }] }); + assert!(matches!(refused, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Applied alone in a batch, the redaction goes through, and the text is gone. + s.apply(id, Edit::Batch { label: "Redact".into(), edits: vec![apply] }).unwrap(); + assert!(!page_texts(&s, id)[0].contains("SECRET42"), "{:?}", page_texts(&s, id)); + assert_eq!(s.get(id).unwrap().redaction_marks(), 0); +} + +/// A batch that deletes a page holding a redaction mark and adds pages is refused whole: the +/// marks cannot be tied to copies, so redaction fails closed. Without added pages it goes through. +#[test] +fn a_batch_that_deletes_a_marked_page_and_adds_pages_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let edits = vec![ + Edit::ApplyRedactions { pages: None }, + Edit::InsertBlankPage { at: 1, width: 612.0, height: 792.0 }, + Edit::DeletePages { pages: vec![0] }, + ]; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits }); + assert!(matches!(refused, Err(EditError::Invalid(_))), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); +} + +/// A batch that applies redactions and then copies a page is refused whole: the copy can't be tied +/// to the marks, so text restored under one would be unchecked. Copying first, or alone, is fine. +#[test] +fn a_batch_that_applies_redactions_and_copies_a_page_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + let copy = Edit::DuplicatePages { pages: vec![0], at: None }; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits: vec![apply.clone(), copy.clone()] }); + assert!(matches!(&refused, Err(EditError::Invalid(m)) if m.contains("copy pages")), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Copying before the redaction is applied puts the mark on the copy too, and both are checked. + s.apply(id, Edit::Batch { label: "y".into(), edits: vec![copy, apply] }).unwrap(); + assert!(page_texts(&s, id).iter().all(|t| !t.contains("SECRET42")), "{:?}", page_texts(&s, id)); +} + /// A reset in an opening script reaches a field of a row an earlier opening script added, before /// the form is laid out again with the row's widgets: its data goes back to its default and stays /// so when the form is saved. diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 8f54cefdc6..819cae99cd 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -58,6 +58,12 @@ pub enum RedactError { NothingToApply, #[error("page {0} has content that can't be read, so it can't be redacted safely")] Unreadable(usize), + /// The page tree and the renderer count a different number of pages (a page repeated in + /// `/Kids`, say), so a mark's page can't be matched to the page the renderer drew. + #[error( + "the document's page tree and its rendered pages disagree on how many pages there are, so the redaction could not be verified; nothing was changed" + )] + AmbiguousPages, /// Redact annotations with no readable area (neither usable `/QuadPoints` nor a usable `/Rect`). /// Skipping them would leave the text under them in the file while the others report success. #[error( From 1d7a9f9ffe45247dbc20a2047a4d2b151cb8385c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:51:22 -0500 Subject: [PATCH 38/46] fix(edit,a11y): font aliases share the metrics built for their font (#5998) * fix(edit,a11y): font aliases share the metrics built for their font Text extraction and heading extraction built and kept font metrics per resource name, so many names for one font object each held a copy of its Unicode and width tables. * fmt * address review: font metrics are cached by the object at the end of the reference chain --- crates/a11y/src/headings.rs | 29 ++++++++++++++++++++++++++--- crates/edit/src/tests.rs | 33 +++++++++++++++++++++++++++++++++ crates/edit/src/text.rs | 2 ++ crates/edit/src/text/interp.rs | 22 +++++++++++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/a11y/src/headings.rs b/crates/a11y/src/headings.rs index 235ba4b452..a9de970b65 100644 --- a/crates/a11y/src/headings.rs +++ b/crates/a11y/src/headings.rs @@ -206,7 +206,9 @@ fn sub(doc: &Document, d: &Dict, key: &[u8]) -> Dict { pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { let res = sub(doc, page, b"Resources"); let (fonts, props) = (sub(doc, &res, b"Font"), sub(doc, &res, b"Properties")); - let mut metrics: HashMap, Metrics> = HashMap::new(); + let mut metrics: HashMap, std::rc::Rc> = HashMap::new(); + // Built once per font object, however many resource names alias it. + let mut by_font: HashMap> = HashMap::new(); // The id each open marked-content sequence is in: its own, else the enclosing one's. let mut stack: Vec> = Vec::new(); let mut out: HashMap = HashMap::new(); @@ -255,9 +257,30 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { gs.font = op.name(0).map(<[u8]>::to_vec); if let Some(name) = op.name(0) && !metrics.contains_key(name) - && let Some(f) = fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()) { - metrics.insert(name.to_vec(), Metrics::from_dict(doc, &f)); + // Keyed by the object at the end of the reference chain, so names that reach one + // font through different chains share its metrics (the hop limit is resolve's). + let font = fonts.get(name).and_then(|o| { + let (mut cur, mut last) = (o.clone(), None); + for _ in 0..16 { + let Object::Ref(r) = cur else { break }; + last = Some(r); + cur = (*doc.get(r)).clone(); + } + last + }); + let built = match font.and_then(|r| by_font.get(&r)) { + Some(m) => Some(m.clone()), + None => { + fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().map(|d| std::rc::Rc::new(Metrics::from_dict(doc, d)))) + } + }; + if let Some(m) = built { + if let Some(r) = font { + by_font.insert(r, m.clone()); + } + metrics.insert(name.to_vec(), m); + } } } b"Tc" => gs.char_spacing = op.num(0).unwrap_or(0.0), diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 1f2b925b46..b956cb7079 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -14042,6 +14042,39 @@ fn review_inside_scope_group_move_matches_direct_transform_under_outer_scale() { } } +/// A page that names one font under 300 resource names builds its metrics once: each alias +/// used to build (and keep) a set of its own. +#[test] +fn font_aliases_share_one_set_of_metrics() { + let fonts: String = (0..300).map(|i| format!(" /A{i} 5 0 R")).collect(); + let content: String = format!("BT 72 700 Td {} ET", (0..300).map(|i| format!("/A{i} 12 Tf (x) Tj ")).collect::()); + let objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font <<{fonts} >> >> >>"), + format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + text::METRICS_BUILT.with(|n| n.set(0)); + let lines = text::text_lines(&doc, 0).unwrap(); + assert!(lines.iter().map(|l| l.text.matches('x').count()).sum::() >= 300, "{lines:?}"); + let built = text::METRICS_BUILT.with(std::cell::Cell::get); + assert!(built < 10, "metrics built {built} times for one font"); +} + /// Each `Tj`/`TJ` moves the text matrix by the width of its text. A run shown later in the same /// text object without a `Td` or `Tm` of its own (a raised run in another colour, a column after /// a `TJ` gap) is placed by those advances. Rewriting the paragraph before it must keep them, as diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index b58c41303f..eba8834c2b 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -54,6 +54,8 @@ pub(crate) use fallback::{SOURCE_SPACE_CALLS, take_source_character_measurements #[cfg(test)] use fallback::{add_type3_font, is_type3_fallback}; pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; +#[cfg(test)] +pub(crate) use interp::METRICS_BUILT; use interp::{Origin, Paint, union}; #[cfg(test)] pub(crate) use lines::dropped_advances_within; diff --git a/crates/edit/src/text/interp.rs b/crates/edit/src/text/interp.rs index 0bc9373899..e628e017c4 100644 --- a/crates/edit/src/text/interp.rs +++ b/crates/edit/src/text/interp.rs @@ -664,6 +664,13 @@ impl Carry { /// The advance is an error message when the font's writing mode can't be read. type CachedFont = (Rc, Option, Result, String>); +#[cfg(test)] +thread_local! { + /// Font metrics built by [`interpret`] on this thread, for the test that keeps font aliases + /// sharing one set. + pub(crate) static METRICS_BUILT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// A `Do` met while interpreting: its operator index, XObject name and the state there. type Invocation = (usize, Vec, Ts); @@ -976,6 +983,7 @@ pub(super) fn interpret( let fonts_res = super::resource_dict(doc, resources, b"Font", &mut held_fonts).unwrap_or(&none); let states = super::resource_dict(doc, resources, b"ExtGState", &mut held_states).unwrap_or(&none); let mut out: Vec = Vec::new(); + let mut by_font: HashMap = HashMap::new(); let mut ts = carry.ts.clone(); let mut at_bt = (0usize, ts.clone()); let mut stack: Vec<(Ts, Matrix, Matrix)> = std::mem::take(&mut carry.stack); @@ -1105,11 +1113,23 @@ pub(super) fn interpret( ts.size = op.num(1).unwrap_or(ts.size); if let Some(name) = op.name(0) { let (m, face, vertical) = cache.entry(name.to_vec()).or_insert_with(|| { + // Names that refer to one font object share what was built for it: a + // thousand aliases of one font are one set of metrics, not a thousand. + let font = fonts_res.get(name).and_then(Object::as_ref); + if let Some(built) = font.and_then(|r| by_font.get(&r)) { + return built.clone(); + } + #[cfg(test)] + METRICS_BUILT.with(|n| n.set(n.get() + 1)); let dict = fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); let metrics = dict.as_ref().map_or_else(Metrics::fallback, |d| Metrics::from_dict(doc, d)); let vertical = dict.as_ref().map_or(Ok(None), |d| vertical_advance(doc, d).map_err(|e| e.to_string())); let face = dict.filter(is_type3_fallback).map(|_| metrics.base_font.clone()); - (Rc::new(metrics), face, vertical) + let built: CachedFont = (Rc::new(metrics), face, vertical); + if let Some(r) = font { + by_font.insert(r, built.clone()); + } + built }); ts.font = Some((name.to_vec(), m.clone())); ts.face = face.clone(); From 8c99ec4e084a02af89ab9b20f5502be15ca04a47 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:04:31 +0100 Subject: [PATCH 39/46] CI: nightly release dry run (#7040) release.yml also runs nightly on main with craft-common-actions' dry-run: every release build except macOS, with nothing signed, pushed or drafted. Breakage that only the release build hits (riscv64, Flatpak, FreeBSD, packaging) then shows up the next morning instead of on release day, as it did on 11 Nov 2026. --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b44d79b68..d214d6738b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,10 @@ name: Release on: + # A nightly dry run on main: every release build except macOS, nothing signed or released + # (craft-common-actions release.yml `dry-run`). Finds release-only breakage before release day. + schedule: + - cron: "20 2 * * *" push: branches: [release] workflow_dispatch: @@ -30,6 +34,7 @@ jobs: # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: + dry-run: ${{ github.event_name == 'schedule' }} app-name: pdfcraft app-title: PdfCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 From 9f763636fd46ce487173925c63799cc87774fc24 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:16:31 -0500 Subject: [PATCH 40/46] fix(forms,xfa): find every password field before a save skips scrubbing it (#5864) * fix(forms,xfa): find every password field before a save skips scrubbing it The quick Password-flag check in pdfcraft-forms stopped silently at its depth limit, and visited the tree in an order where a field the full field walk reaches could be cut off on a long branch first. It now answers true (read every field) whenever it cannot look as far as the field walk, and also walks below fields adopted from page widgets. XFA password classification walked AcroForm fields only 32 levels deep, shallower than editing and dataset synchronization, and read the datasets packet of an XFA array without the prefixes its preamble declares, so a typed password could stay in the saved datasets. It now walks as deep as the field walk, refuses to clear (and so to save) when the tree is deeper than it can check, and reads the packet the way the datasets writer does. * address review: check every page widget's /Parent chain for the password flag * address review: password scan follows the parents of nameless /Fields entries; field-walk completeness checked before the datasets packet is read * address review: the field walk's completeness is decided after every root; the subtree scan starts at the last dictionary of the parent chain * address review: a skipped visit does not replace the reach a field was classified with --- crates/forms/src/tests.rs | 85 ++++++++++++++++++++++++++++++++ crates/forms/src/values.rs | 74 +++++++++++++++++++++------ crates/xfa/src/data/passwords.rs | 80 +++++++++++++++++++++--------- crates/xfa/src/data/path.rs | 27 ++++++++++ crates/xfa/src/tests.rs | 68 +++++++++++++++++++++++++ 5 files changed, 298 insertions(+), 36 deletions(-) diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index a2a87f3a73..2ce02626c9 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -5601,6 +5601,75 @@ fn a_page_widget_inheriting_password_flag_beyond_depth_32_is_recognized() { assert!(!has_password_values(&doc)); } +/// A field reachable both near the top of the tree and at the end of a long branch: the field +/// walk reaches it near the top and reads the password field below it, so the quick check must +/// not stop at the long branch's depth limit and conclude there is none. The password field's +/// widget is on the page but names no `/Parent`, so the page widgets alone don't find it either. +#[test] +fn a_password_field_past_the_depth_limit_on_one_branch_is_still_found() { + let chain = 64; + let x = 6 + chain; // the shared field + let y = x + 1; // the password field + let w = y + 1; // its widget + let mut objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{w} 0 R] >>"), + "<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + format!("<< /T (short) /Kids [{x} 0 R] >>"), + ]; + for d in 0..chain { + let next = if d + 1 == chain { x } else { 7 + d }; + objs.push(format!("<< /T (n{d}) /Kids [{next} 0 R] >>")); + } + objs.push(format!("<< /T (shared) /Kids [{y} 0 R] >>")); + objs.push(format!("<< /FT /Tx /T (pin) /Ff 8192 /V (deepsecret) /Kids [{w} 0 R] >>")); + objs.push("<< /Type /Annot /Subtype /Widget /Rect [50 600 150 620] /P 3 0 R >>".into()); + let mut doc = document(&objs); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD)), "the field walk reads the password field"); + assert!(has_password_flag(&doc)); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + +/// A password field below an unlisted field whose other widget is on the page: the field walk +/// adopts the whole unlisted field from that widget, so the quick check looks below it too. +#[test] +fn a_password_field_below_an_adopted_page_field_is_found() { + let mut doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [6 0 R] >>".into(), + "<< /Fields [] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (grp) /Kids [6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /Parent 5 0 R /Rect [50 700 150 720] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (pin) /Ff 8192 /V (secret8) /Parent 5 0 R /Rect [0 0 0 0] >>".into(), + ]); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD)), "the field walk adopts the password field"); + assert!(has_password_flag(&doc)); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + +/// A nameless widget listed in `/Fields` and on no page, whose `/Parent` is an unlisted password +/// field: the field walk adopts the parent, so the quick check follows the entry's `/Parent` too. +#[test] +fn a_password_parent_of_a_nameless_fields_entry_is_found() { + let mut doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R >>".into(), + "<< /Fields [6 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (pin) /FT /Tx /Ff 8192 /V (secret8) /Kids [6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /Parent 5 0 R /Rect [0 0 0 0] >>".into(), + ]); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD) && f.value.iter().any(|v| v == "secret8")), "the field walk reads the parent"); + assert!(has_password_flag(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); +} + /// #4018: a value that doesn't fit its box at the field's font size is reported (it is drawn cut /// at the box). An auto-sized field shrinks to fit, and a short or empty value fits. #[test] @@ -6802,6 +6871,22 @@ fn locked_button_export_edits_leave_the_document_unchanged() { } } +/// A widget reached from `/Fields` as an unnamed kid of a plain field, whose own `/Parent` is an +/// unlisted password field: the quick check must still read that `/Parent` chain. +#[test] +fn a_page_widget_seen_under_fields_still_has_its_parent_chain_checked() { + let doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [6 0 R] >>".into(), + "<< /Fields [5 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (grp) /Kids [6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /Parent 7 0 R /Rect [50 700 150 720] /P 3 0 R >>".into(), + "<< /FT /Tx /T (pin) /Ff 8192 /V (secret9) /Kids [6 0 R] >>".into(), + ]); + assert!(has_password_flag(&doc)); +} + /// A radio button that is read-only on its own (its own `/Ff`) can't be picked, and a selected one /// can't be switched away from, though the group as a whole is writable. #[test] diff --git a/crates/forms/src/values.rs b/crates/forms/src/values.rs index b60d8aa140..3600695c03 100644 --- a/crates/forms/src/values.rs +++ b/crates/forms/src/values.rs @@ -758,29 +758,69 @@ fn password_fields(doc: &Document) -> Vec { /// again only makes the answer cautious. It reads only `/Ff`, `/Kids` and `/Parent`, so a save /// or recovery snapshot of a form without such a field doesn't build every [`Field`]. Like /// [`fields`], it looks at the form's field tree and at the widgets on the pages, which may be -/// missing from `/Fields`. Each dictionary is visited once and at most 64 levels deep. +/// missing from `/Fields`, together with the whole tree above and below them that [`fields`] +/// adopts. Each dictionary is visited once. +/// +/// It must never answer `false` for a form whose [`fields`] include a password field, or a typed +/// password would be saved. So where it can't look as far as [`fields`] may (a branch deeper than +/// the field walk's limit, which depends on the order the tree is visited in, or a `/Parent` +/// chain too long to follow), it answers `true` and the caller reads every field. pub(crate) fn has_password_flag(doc: &Document) -> bool { + const MAX_FIELD_DEPTH: usize = 64; let sets_it = |r: ObjRef| doc.get(r).as_dict().and_then(|d| field_flags(doc, d)).is_some_and(|ff| ff & flags::PASSWORD != 0); - let top = acroform(doc).and_then(|af| af.get(b"Fields").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned())).unwrap_or_default(); - let mut stack: Vec<(Object, usize)> = top.into_iter().map(|o| (o, 0)).collect(); + let kids_of = + |r: ObjRef| doc.get(r).as_dict().and_then(|d| d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned())).unwrap_or_default(); let mut seen = std::collections::HashSet::new(); - while let Some((o, depth)) = stack.pop() { - let Object::Ref(r) = o else { continue }; - if depth > 64 || !seen.insert(r) { - continue; + // Whether a dictionary below `start` sets the flag, or the walk was cut short. + let subtree = |start: Vec, seen: &mut std::collections::HashSet| { + let mut stack: Vec<(Object, usize)> = start.into_iter().map(|o| (o, 0)).collect(); + while let Some((o, depth)) = stack.pop() { + let Object::Ref(r) = o else { continue }; + if !seen.insert(r) { + continue; + } + if depth > MAX_FIELD_DEPTH || sets_it(r) { + return true; + } + stack.extend(kids_of(r).into_iter().map(|k| (k, depth + 1))); } - if sets_it(r) { + false + }; + let top = acroform(doc).and_then(|af| af.get(b"Fields").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned())).unwrap_or_default(); + let top_refs: Vec = top.iter().filter_map(Object::as_ref).collect(); + if subtree(top, &mut seen) { + return true; + } + // A `/Fields` entry may itself name a field above it through `/Parent` (one `fields` adopts when + // the entry has no name of its own), whether or not a page lists it. + for r in top_refs { + let chain = field_chain(doc, r); + if chain.len() >= MAX_FIELD_DEPTH || chain.iter().copied().any(sets_it) { + return true; + } + if subtree(vec![Object::Ref(field_root(doc, &chain, r))], &mut seen) { return true; } - let kids = doc.get(r).as_dict().and_then(|d| d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned())).unwrap_or_default(); - stack.extend(kids.into_iter().map(|k| (k, depth + 1))); } - // Widgets on the pages, and the fields above them. - page_refs(doc).into_iter().any(|p| { + // Widgets on the pages, the fields above them, and the fields below the topmost one. + for p in page_refs(doc) { let annots = doc.get(p).as_dict().and_then(|d| d.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned())).unwrap_or_default(); - annots.iter().filter_map(Object::as_ref).filter(|r| !seen.contains(r)).any(|r| field_chain(doc, r).into_iter().any(sets_it)) - }) + // Every widget's `/Parent` chain is read, even one already reached from `/Fields`: the + // field walk can adopt it under a different parent there. Each chain is at most + // MAX_FIELD_DEPTH long, and `subtree` returns at once for a root it has seen. + for r in annots.iter().filter_map(Object::as_ref) { + let chain = field_chain(doc, r); + if chain.len() >= MAX_FIELD_DEPTH || chain.iter().copied().any(sets_it) { + return true; + } + let root = field_root(doc, &chain, r); + if subtree(vec![Object::Ref(root)], &mut seen) { + return true; + } + } + } + false } /// Whether any text field with the Password flag holds a value in `doc`. @@ -806,6 +846,12 @@ pub fn restore_password_values(doc: &mut Document, values: &[(String, String)]) Ok(()) } +/// The topmost dictionary of `chain` (from [`field_chain`]): a `/Parent` that is not a dictionary +/// ends the field tree, so the walk below it starts at the last one that is. +fn field_root(doc: &Document, chain: &[ObjRef], r: ObjRef) -> ObjRef { + chain.iter().rev().copied().find(|c| doc.get(*c).as_dict().is_some()).unwrap_or(r) +} + /// `r` and its field ancestors (`/Parent`), nearest first, at most 64 deep and without repeats. fn field_chain(doc: &Document, r: ObjRef) -> Vec { let mut chain = vec![r]; diff --git a/crates/xfa/src/data/passwords.rs b/crates/xfa/src/data/passwords.rs index 0f98a3f2b4..896d513f2c 100644 --- a/crates/xfa/src/data/passwords.rs +++ b/crates/xfa/src/data/passwords.rs @@ -1,26 +1,38 @@ use super::import::datasets_packet; use super::merge::Op; -use super::path::{data_path, parse_datasets}; +use super::path::{data_path, parse_datasets_packet}; use super::values::{Want, path_of, text_of, with_template}; use super::write::{datasets_stream_ref, rewrite}; use super::{DataNode, DataPath, Document, MAX_DEPTH, Node, ObjRef, Object, SomIndices, Template, Ui, XfaError}; +/// How deep the AcroForm field tree is walked for password fields: as deep as the form's own +/// field walk (`pdfcraft_forms::fields`), which is what editing and dataset synchronization see. +const MAX_ACRO_DEPTH: usize = 64; + /// The SOM path and default value of every AcroForm text field with the Password flag (bit 14, /// `1 << 13`) in `doc`, for static XFA forms whose password fields set the flag in the AcroForm -/// dictionary. -fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { +/// dictionary. `None` when the walk was cut short at [`MAX_ACRO_DEPTH`]: which branch is cut +/// depends on the order the tree is visited in, so a field the form's own walk reaches may have +/// been missed, and the caller can't tell that no password field is left. +fn acro_password_fields(doc: &Document) -> Option> { const PASSWORD_FLAG: i64 = 1 << 13; - let Some(root) = doc.root() else { return Vec::new() }; + let Some(root) = doc.root() else { return Some(Vec::new()) }; let catalog = doc.get(root); - let Some(acro) = catalog.as_dict().and_then(|c| c.get(b"AcroForm")).map(|a| doc.resolve(a)) else { return Vec::new() }; + let Some(acro) = catalog.as_dict().and_then(|c| c.get(b"AcroForm")).map(|a| doc.resolve(a)) else { return Some(Vec::new()) }; let Some(top) = acro.as_dict().and_then(|a| a.get(b"Fields")).map(|f| doc.resolve(f)).and_then(|f| f.as_array().cloned()) else { - return Vec::new(); + return Some(Vec::new()); }; struct AcroWalk<'a> { doc: &'a Document, - seen: std::collections::HashSet, + /// Each field reached, with what it was reached with: its parent's name, type, flags and + /// default, which decide what it classifies as. + seen: std::collections::HashMap, + /// Fields past the depth limit, with what they were reached with. One reached again from + /// another root in the same way was classified there; any other is not fully read. + cut: Vec<(ObjRef, Reach)>, out: Vec<(String, String)>, } + type Reach = (String, Option>, i64, Option); impl AcroWalk<'_> { fn walk( &mut self, @@ -31,12 +43,19 @@ fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { inherited_dv: Option<&str>, depth: usize, ) { - if depth > 32 { + if depth > MAX_ACRO_DEPTH { + let reach: Reach = (parent_name.to_string(), inherited_ft.map(<[u8]>::to_vec), inherited_ff, inherited_dv.map(str::to_string)); + self.cut.extend(items.iter().filter_map(Object::as_ref).map(|r| (r, reach.clone()))); return; } for item in items { let Some(r) = item.as_ref() else { continue }; - if !self.seen.insert(r) { + let reach: Reach = (parent_name.to_string(), inherited_ft.map(<[u8]>::to_vec), inherited_ff, inherited_dv.map(str::to_string)); + // The reach a field was classified with stays: a later visit that skips it classified + // nothing, so it must not stand in for the cut-off one. + if let std::collections::hash_map::Entry::Vacant(slot) = self.seen.entry(r) { + slot.insert(reach); + } else { continue; } let obj = self.doc.get(r); @@ -65,20 +84,35 @@ fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { } } } - let mut w = AcroWalk { doc, seen: std::collections::HashSet::new(), out: Vec::new() }; + let mut w = AcroWalk { doc, seen: std::collections::HashMap::new(), cut: Vec::new(), out: Vec::new() }; w.walk(&top, "", None, 0, None, 0); - w.out + // Decided once every root has been visited: a branch cut deep down may be reached shallower + // from a later root, and is then read in full. + let incomplete = w.cut.iter().any(|(r, reach)| w.seen.get(r) != Some(reach)); + (!incomplete).then_some(w.out) } /// The `Want` entries needed to restore password fields in `doc`'s datasets packet to their /// defaults (or empty when they have no default). -fn password_dataset_wants(doc: &Document) -> Vec { +/// +/// The packet is read as the datasets writer reads it ([`parse_datasets_packet`]): a packet the +/// writer can't read holds no value it wrote, so there is nothing typed in it to clear. An error +/// when the password fields can't all be found (a field tree deeper than the field walk reaches): +/// the save is refused rather than written with a typed password left in the data. +fn password_dataset_wants(doc: &Document) -> Result, XfaError> { if datasets_stream_ref(doc).is_none() { - return Vec::new(); + return Ok(Vec::new()); } - let Ok(Some(packet)) = datasets_packet(doc) else { return Vec::new() }; - let Some(data) = parse_datasets(&packet) else { return Vec::new() }; - with_template(doc, None, |tpl| { + // Checked before the packet is read: a packet that can't be parsed leaves nothing to clear, but + // a form whose fields can't all be found is refused whatever the packet holds. + let Some(acro_fields) = acro_password_fields(doc) else { + return Err(XfaError::Malformed( + "the form's fields nest too deeply to find every password field, so typed passwords could not be removed from its XFA data".into(), + )); + }; + let Ok(Some(packet)) = datasets_packet(doc) else { return Ok(Vec::new()) }; + let Some(data) = parse_datasets_packet(&packet) else { return Ok(Vec::new()) }; + Ok(with_template(doc, None, |tpl| { let mut candidates: Vec<(String, DataPath, String)> = Vec::new(); let mut seen = std::collections::HashSet::new(); if let Some(t) = tpl { @@ -153,7 +187,7 @@ fn password_dataset_wants(doc: &Document) -> Vec { } } } - for (som, default) in acro_password_fields(doc) { + for (som, default) in acro_fields { let path = path_of(tpl, &som); if !path.is_empty() && seen.insert(path.clone()) { candidates.push((som, path, default)); @@ -166,27 +200,29 @@ fn password_dataset_wants(doc: &Document) -> Vec { (!cur.is_empty() && cur != default).then_some(Want { som, path, text: default, ensure: false }) }) .collect() - }) + })) } /// Whether any XFA password field (`` or an AcroForm text field with the Password -/// flag) holds a value in `doc`'s datasets packet beyond its default (#3944). +/// flag) holds a value in `doc`'s datasets packet beyond its default (#3944). A form whose +/// password fields can't all be found counts as holding some, so that clearing them +/// ([`clear_password_values`]) reports why the document can't be written without them. pub fn has_password_values(doc: &Document) -> bool { - !password_dataset_wants(doc).is_empty() + !password_dataset_wants(doc).is_ok_and(|w| w.is_empty()) } /// Clear typed password values from `doc`'s XFA datasets packet (restoring a template default /// where the field has one, else emptying the element), replacing the datasets stream in place /// (#3944). Returns how many password fields were cleared in the datasets. pub fn clear_password_values(doc: &mut Document) -> Result { - let wants = password_dataset_wants(doc); + let wants = password_dataset_wants(doc)?; if wants.is_empty() { return Ok(0); } let count = wants.len(); let reuse = datasets_stream_ref(doc); let report = rewrite(doc, &[Op::Values(&wants)], reuse)?; - if !report.written || !password_dataset_wants(doc).is_empty() { + if !report.written || !password_dataset_wants(doc)?.is_empty() { let reason = report.warnings.first().map_or("the XFA datasets password values could not be cleared", String::as_str); return Err(XfaError::Malformed(reason.to_string())); } diff --git a/crates/xfa/src/data/path.rs b/crates/xfa/src/data/path.rs index 23c5e5ba5d..7fdba77071 100644 --- a/crates/xfa/src/data/path.rs +++ b/crates/xfa/src/data/path.rs @@ -24,6 +24,33 @@ pub fn parse_datasets(xdp: &str) -> Option { Some(to_node(data, 0)) } +/// The `xfa:data` element of a datasets packet as the datasets writer reads it before changing it: +/// a packet of an array may use the `xfa` and `xdp` prefixes its XDP envelope declares, so it is +/// read inside an element declaring them when it doesn't parse alone, and the datasets element in +/// the XFA data namespace is preferred over another with the same local name. Reading the packet +/// any other way could miss the values the writer put in it. +pub(super) fn parse_datasets_packet(text: &str) -> Option { + if crate::xml_nesting_depth(text) > crate::MAX_XML_DEPTH { + return None; + } + let opts = || roxmltree::ParsingOptions { allow_dtd: false, nodes_limit: MAX_NODES, ..Default::default() }; + let wrapped; + let doc = match roxmltree::Document::parse_with_options(text, opts()) { + Ok(d) => d, + Err(_) => { + wrapped = format!("{}{text}{}", super::merge::WRAP_OPEN, super::merge::WRAP_CLOSE); + roxmltree::Document::parse_with_options(&wrapped, opts()).ok()? + } + }; + let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name; + let datasets = doc + .descendants() + .find(|n| is(n, "datasets") && n.tag_name().namespace() == Some(super::XFA_DATA_NS)) + .or_else(|| doc.descendants().find(|n| is(n, "datasets")))?; + let data = datasets.children().find(|n| is(n, "data"))?; + Some(to_node(data, 0)) +} + /// The data path a SOM expression names: `form[0].#subform[0].name[1]` → `[(form, 0), (name, 1)]`. /// Unnamed containers (`#subform`) have no data node. pub fn som_to_path(som: &str) -> DataPath { diff --git a/crates/xfa/src/tests.rs b/crates/xfa/src/tests.rs index 27e6522f26..c1abcce190 100644 --- a/crates/xfa/src/tests.rs +++ b/crates/xfa/src/tests.rs @@ -7114,6 +7114,74 @@ fn a_container_is_as_wide_as_a_turned_draw_stands() { } } +/// The static fixture with its text field (`form1[0].page1[0].name[0]`) flagged Password and +/// `levels` unnamed fields between `form1[0]` and `page1[0]`, which add no SOM segment. +fn static_password_doc(levels: usize, data: &str) -> Document { + let mut doc = Document::open(Arc::new(static_shell(data))).unwrap(); + doc.update_dict(pdfcraft_cos::ObjRef::new(8, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1 << 13))).unwrap(); + let mut next = pdfcraft_cos::ObjRef::new(7, 0); + for _ in 0..levels { + let mut d = Dict::new(); + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(next)])); + next = doc.add(Object::Dict(d)); + } + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(next)]))).unwrap(); + doc +} + +/// #3944: a Password-flagged AcroForm field of a static XFA form nested deeper than 32 levels (as +/// deep as the form's own field walk reaches) has its typed value cleared from the datasets. +#[test] +fn a_deeply_nested_acroform_password_field_is_cleared_from_the_datasets() { + let data = "hunter2"; + let mut doc = static_password_doc(40, data); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); + assert!(!String::from_utf8_lossy(&xfa_bytes(&doc)).contains("hunter2")); +} + +/// A field tree deeper than the walk reaches can't be shown to hold no password field: clearing +/// refuses (so the save does) rather than leave a typed password in the datasets. +#[test] +fn a_field_tree_too_deep_to_check_for_passwords_refuses_to_clear() { + let mut doc = static_password_doc(80, "hunter2"); + assert!(has_password_values(&doc)); + assert!(clear_password_values(&mut doc).is_err()); + // A shallow form without a typed value has nothing to clear. + let mut doc = static_password_doc(0, ""); + assert!(!has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 0); +} + +/// A datasets packet of an XFA array may use the `xfa` prefix its preamble declares: the typed +/// password in it is still found and cleared, as the datasets writer reads that packet too. +#[test] +fn a_password_in_a_datasets_packet_using_the_preamble_prefix_is_cleared() { + let mut doc = static_password_doc(0, ""); + let pre = r#""#; + let tpl = r#""#; + let ds = "hunter2"; + let parts: Vec = [("preamble", pre), ("template", tpl), ("datasets", ds), ("postamble", "")] + .into_iter() + .flat_map(|(name, text)| { + let r = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), text.as_bytes()))); + [Object::String(pdfcraft_cos::PdfString::literal(name.as_bytes().to_vec())), Object::Ref(r)] + }) + .collect(); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + let mut acro = c.get(b"AcroForm").and_then(Object::as_dict).cloned().unwrap(); + acro.set(b"XFA".to_vec(), Object::Array(parts)); + c.set(b"AcroForm".to_vec(), Object::Dict(acro)); + }) + .unwrap(); + assert!(crate::data::parse_datasets(ds).is_none(), "the packet doesn't parse alone"); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + /// Every sibling a SOM index skips owns a copy of the element name: the in-memory builder refuses /// a path whose copies would pass the name-byte bound the writer holds itself to, and creates /// nothing for it. From 80e26e674d1aa33078e83c8139875047f3289eb6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:16:48 -0500 Subject: [PATCH 41/46] fix(redact): remove a stroked path by the box its stroke paints (#5876) * fix(redact): remove a stroked path by the box its stroke paints Path redaction tested a path's construction box against the marks. A horizontal or vertical stroke has a zero-area box, which overlaps nothing, so its drawing commands stayed beneath the overlay; a wide stroke whose points lie beside a mark but whose ink reaches into it stayed too. The interpreter now tracks the line width, miter limit and join style (`w`, `M`, `j`, and `LW`, `ML`, `LJ` through `gs`) and, for a stroking operator, grows the box by half the width (at least one unit), times the miter limit when the path has mitred joins, scaled by the CTM's largest stretch, before the hit test. The bound is conservative: a stroke near a mark may now go where before it stayed. * address review: pad hairlines by half a unit, rectangles for their corners, square caps by sqrt 2 * review: test a kept clip against the path as constructed * address review: a line state set between a path and its stroke no longer lets the stroke skip the redaction test * address review: line-state operators read the operand before the operator --- crates/redact/src/interp.rs | 69 +++++++++++++++++++++++++++++++++++-- crates/redact/src/tests.rs | 69 +++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 3 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 80117f451d..12636dc88f 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -93,6 +93,12 @@ struct Gs { scale: f64, leading: f64, rise: f64, + /// The stroke parameters that widen a painted path beyond its construction points. + line_width: f64, + miter_limit: f64, + line_join: i64, + /// `J` / ExtGState `/LC`: a projecting square cap (2) reaches past the endpoint's corner. + line_cap: i64, /// The tiling or shading pattern selected as the fill colour (`/Name scn`), and as the stroke /// colour (`/Name SCN`). Part of the graphics state, so `q`/`Q` save and restore them with /// everything else. A pattern's cell is a content stream of its own and this pass does not @@ -394,6 +400,10 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< scale: 1.0, leading: 0.0, rise: 0.0, + line_width: 1.0, + miter_limit: 10.0, + line_join: 0, + line_cap: 0, fill_pattern: None, stroke_pattern: None, type3_inherits: false, @@ -438,6 +448,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], } if !walk.path.is_empty() { done.ops.append(&mut walk.path); + done.ops.append(&mut walk.pending); } processed.push((done.ops, done.changed)); } @@ -505,6 +516,10 @@ struct Walk { path: Vec, path_box: Option<[f64; 4]>, clip: bool, + /// Line-state operators (`w`, `M`, `j`, `J`, `d`, `gs`) met while a path is open: they apply to + /// the stroke that ends it, so the path stays whole for the hit test, and they are written + /// after it, kept even when the path goes. + pending: Vec, /// Painting operators written again under a clip that cuts the regions out: unchanged as /// operators, but what they paint is no longer all there. clipped: HashSet, @@ -526,6 +541,7 @@ impl Walk { path: Vec::new(), path_box: None, clip: false, + pending: Vec::new(), clipped: HashSet::new(), out: Output::default(), } @@ -542,14 +558,20 @@ impl Walk { self.paint_path(scope, op, piece); return; } + if !self.path.is_empty() && matches!(o, b"w" | b"M" | b"j" | b"J" | b"d" | b"gs") { + self.graphics_state(doc, scope, &op); + self.pending.push(op); + return; + } if !self.path.is_empty() { // A path not ended by a painting operator: pass it through. piece.ops.append(&mut self.path); + piece.ops.append(&mut self.pending); self.path_box = None; self.clip = false; } let kept = match o { - b"q" | b"Q" | b"cm" | b"Tf" | b"gs" => { + b"q" | b"Q" | b"cm" | b"Tf" | b"gs" | b"w" | b"M" | b"j" | b"J" => { self.graphics_state(doc, scope, &op); Some(op) } @@ -627,11 +649,35 @@ impl Walk { self.path.push(op); } + /// The box a stroke of the current path can paint, grown from its construction box `b` by + /// what the pen adds: half the line width (a width of 0 is one device pixel, taken as one + /// unit) scaled by the CTM's largest stretch, and never less than half a unit of page space + /// (a renderer paints a hairline, or a line a CTM shrinks, about a device pixel wide); times + /// the miter limit when the path has joins and they are mitred (a rectangle has four), or + /// √2 for a projecting square cap (its outer corner). Caps and joins never reach further, so + /// a hit test on it is conservative, and a horizontal or vertical line no longer has a + /// zero-area box. + fn stroke_box(&self, b: [f64; 4]) -> [f64; 4] { + let m = self.gs.ctm.0; + let scale = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2] + m[3] * m[3]).sqrt(); + let joins = self.path.iter().filter(|op| matches!(op.op.as_slice(), b"l" | b"c" | b"v" | b"y" | b"h" | b"re")).count() >= 2 + || self.path.iter().any(|op| op.op.as_slice() == b"re"); + let miter = if joins && self.gs.line_join == 0 { self.gs.miter_limit.max(1.0) } else { 1.0 }; + let reach = if self.gs.line_cap == 2 { miter.max(std::f64::consts::SQRT_2) } else { miter }; + let pad = (self.gs.line_width.max(1.0) / 2.0 * scale).max(0.5) * reach; + [b[0] - pad, b[1] - pad, b[2] + pad, b[3] + pad] + } + /// A painting operator ends the path; one under a region is removed (or kept as a clip). fn paint_path(&mut self, scope: &mut Scope<'_>, op: Op, piece: &mut Piece) { + let strokes = matches!(op.op.as_slice(), b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*"); + // A `W`/`W*` makes the path as constructed the clip; the stroke expansion only describes + // what the painting itself covers. + let path_box = self.path_box.take(); + let bbox = path_box.map(|b| if strokes { self.stroke_box(b) } else { b }); let body = std::mem::take(&mut self.path); - let bbox = self.path_box.take(); let clips = std::mem::replace(&mut self.clip, false); + let pending = std::mem::take(&mut self.pending); let hit = scope.mode == Mode::Apply && op.op.as_slice() != b"n" && bbox.is_some_and(|b| scope.hits(b)); if !hit { // The paint survives, so the pattern it draws with is still in use. `n` paints @@ -641,25 +687,30 @@ impl Walk { let fills = matches!(o, b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*"); uses_patterns(&mut self.out, &self.gs, fills, strokes); piece.ops.extend(body); + piece.ops.extend(pending); piece.ops.push(op); return; } let b = bbox.unwrap_or_default(); piece.changed = true; scope.report.paths_removed += 1; - if scope.covered(b) && clips { + if scope.covered(path_box.unwrap_or(b)) && clips { // Everything this clip lets through is under a mark, so an empty clip hides the same // content. The original outline is confidential and must not stay in the stream. + piece.ops.extend(pending); for o in ["re", "W", "n"] { let operands = if o == "re" { vec![num(0.0); 4] } else { vec![] }; piece.ops.push(Op::new(o, operands)); } + } else { + piece.ops.extend(pending); } // Partial vector redaction must not retain underlying drawing commands // behind clipping (F-008). Refuse partial clip retention and remove the path. } fn graphics_state(&mut self, doc: &Document, scope: &mut Scope<'_>, op: &Op) { + let last = op.nums::<1>().map(|[v]| v); match op.op.as_slice() { b"q" => self.stack.push((self.gs.clone(), self.tm, self.tlm)), b"Q" => { @@ -673,12 +724,24 @@ impl Walk { } } b"Tf" => self.select_font(doc, scope, op), + // The operand before the operator, as the renderer reads it: `1 12 w` is a width of 12. + b"w" => self.gs.line_width = last.unwrap_or(self.gs.line_width), + b"M" => self.gs.miter_limit = last.unwrap_or(self.gs.miter_limit), + b"j" => self.gs.line_join = last.map_or(self.gs.line_join, |j| j as i64), + b"J" => self.gs.line_cap = last.map_or(self.gs.line_cap, |j| j as i64), _ => { if let Some((font, vertical, size)) = op.last_name().and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); // As for `Tf`: a Type 3 font without /Resources paints through this scope's. self.gs.type3_inherits = op.last_name().is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } + if let Some(state) = op.name(0).and_then(|n| self.ext_gstates.get(n)).and_then(|s| doc.resolve(s).as_dict().cloned()) { + let num = |key: &[u8]| state.get(key).and_then(|v| doc.resolve(v).as_f64()); + self.gs.line_width = num(b"LW").unwrap_or(self.gs.line_width); + self.gs.miter_limit = num(b"ML").unwrap_or(self.gs.miter_limit); + self.gs.line_join = num(b"LJ").map_or(self.gs.line_join, |j| j as i64); + self.gs.line_cap = num(b"LC").map_or(self.gs.line_cap, |j| j as i64); + } } } } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 08125165b5..43b6caaadc 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -837,6 +837,22 @@ fn vectors_are_removed_or_clipped_and_inline_images_go() { assert!(c.contains("150 150 m"), "the line elsewhere stays"); } +/// A horizontal stroke's construction box has no area, so it overlapped nothing and stayed under +/// the mark. The painted stroke is as wide as the line: a 12 pt line centred 5 pt above the mark +/// reaches into it and goes; a 1 pt line there does not and stays. +#[test] +fn a_stroke_is_removed_by_its_painted_width() { + let src = b"12 w 10 60 m 100 60 l S 1 w 10 60 m 100 60 l S 4 w 10 50 m 100 50 l S"; + let mut doc = one_page(src, "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 2, "{r:?}"); + let c = content(&doc, 0); + assert!(!c.contains("12 w\n10 60 m"), "the wide line reaches the mark: {c}"); + assert!(c.contains("1 w\n10 60 m\n100 60 l\nS"), "the thin line beside it stays: {c}"); + assert!(!c.contains("10 50 m"), "the line through the mark goes: {c}"); +} + /// A tiling pattern's cell is a content stream of its own and can draw text. Redacting a shape /// filled with such a pattern removed the fill from the page but left the pattern in the page's /// resources, so its cell -- with the text that was under the mark -- stayed in the saved file @@ -6674,6 +6690,59 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// A hairline under a shrinking CTM still paints about a device pixel: its pad must not shrink +/// with the CTM, or a mark on that ink misses it. +#[test] +fn a_hairline_under_a_shrinking_ctm_is_padded_by_half_a_unit() { + // Device line from (10, 10) to (50, 10); the mark lies on its painted pixel, above its axis. + let mut doc = one_page(b"0.01 0 0 0.01 0 0 cm 0 w 1000 1000 m 5000 1000 l S", "", vec![]); + mark(&mut doc, 0, &[[20.0, 10.2, 30.0, 10.4]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// One `re` has four mitred corners: it is padded by the miter limit like any joined path. +#[test] +fn a_rectangle_stroke_is_padded_for_its_mitred_corners() { + let mut doc = one_page(b"2 w 100 100 50 50 re S", "", vec![]); + // Past half the line width from the rectangle, inside the miter reach. + mark(&mut doc, 0, &[[155.0, 120.0, 160.0, 130.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// A projecting square cap's outer corner reaches lw/2 x sqrt(2) from the endpoint. Under a shear +/// it maps past a pad of lw/2 times the CTM's stretch (the review's example: corner at x = 12). +#[test] +fn a_square_cap_corner_under_a_shear_is_inside_the_pad() { + let mut doc = one_page(b"1 0 1 1 0 0 cm 2 J 2 w 0 0 m 10 0 l S", "", vec![]); + mark(&mut doc, 0, &[[11.8, 0.8, 11.9, 0.95]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// A line width set between the path and its `S` still applies to that stroke, so the stroke is +/// tested at the width it is painted with; the width itself stays, the path goes. +#[test] +fn a_width_set_between_the_path_and_its_stroke_does_not_hide_it_from_the_test() { + let mut doc = one_page(b"10 60 m 100 60 l 12 w S 5 5 m 9 9 l S", "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1, "{r:?}"); + let c = content(&doc, 0); + assert!(!c.contains("10 60 m") && !c.contains("100 60 l"), "the wide line reaches the mark: {c}"); + assert!(c.contains("12 w"), "the width stays: {c}"); + assert!(c.contains("5 5 m"), "the line elsewhere stays: {c}"); +} + +/// The renderer takes a line-state operator's operand from the end: `1 12 w` strokes 12 points wide, +/// so the stroke is tested at that width. +#[test] +fn a_surplus_operand_before_the_line_width_does_not_hide_the_stroke() { + let mut doc = one_page(b"1 12 w 10 60 m 100 60 l S", "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1, "{r:?}"); + assert!(!content(&doc, 0).contains("100 60 l")); +} + /// One page drawing `SECRET` in the `/OC /L` block, with layer 7 0 R in the default /// configuration `config` (objects 8 and up are `extra`). fn layer_state_page(config: Vec<(&str, Object)>, resource: &str, extra: Vec>) -> Document { From 1d2af9907ba772b368f6b54811359805fa5428e9 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 18:38:09 -0500 Subject: [PATCH 42/46] Fix Flatpak updates and bump to 0.21.1 (#7020) * fix(flatpak): allow release checks and self-update helpers * test: keep Flatpak permission parsing within strict Clippy rules * chore: bump version to 0.21.1 for the Flatpak update fix --- Cargo.lock | 78 +++++++++---------- Cargo.toml | 2 +- crates/ui-egui/tests/flatpak_updates.rs | 27 +++++++ .../ai.storyteller.pdfcraft.bundle.yml | 2 + .../linux/flatpak/ai.storyteller.pdfcraft.yml | 2 + 5 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 crates/ui-egui/tests/flatpak_updates.rs diff --git a/Cargo.lock b/Cargo.lock index cecedc12c9..0e4c910427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4264,7 +4264,7 @@ dependencies = [ [[package]] name = "pdfcraft" -version = "0.21.0" +version = "0.21.1" dependencies = [ "chrono", "craft-update", @@ -4289,7 +4289,7 @@ dependencies = [ [[package]] name = "pdfcraft-a11y" -version = "0.21.0" +version = "0.21.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4300,7 +4300,7 @@ dependencies = [ [[package]] name = "pdfcraft-annot" -version = "0.21.0" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4312,7 +4312,7 @@ dependencies = [ [[package]] name = "pdfcraft-audit" -version = "0.21.0" +version = "0.21.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4323,7 +4323,7 @@ dependencies = [ [[package]] name = "pdfcraft-automation" -version = "0.21.0" +version = "0.21.1" dependencies = [ "base64 0.23.1", "flate2", @@ -4347,7 +4347,7 @@ dependencies = [ [[package]] name = "pdfcraft-cli" -version = "0.21.0" +version = "0.21.1" dependencies = [ "base64 0.23.1", "pdfcraft-audit", @@ -4365,11 +4365,11 @@ dependencies = [ [[package]] name = "pdfcraft-compare" -version = "0.21.0" +version = "0.21.1" [[package]] name = "pdfcraft-content" -version = "0.21.0" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4377,7 +4377,7 @@ dependencies = [ [[package]] name = "pdfcraft-cos" -version = "0.21.0" +version = "0.21.1" dependencies = [ "chrono", "hayro-syntax", @@ -4394,7 +4394,7 @@ dependencies = [ [[package]] name = "pdfcraft-create" -version = "0.21.0" +version = "0.21.1" dependencies = [ "encoding_rs", "hayro-ccitt", @@ -4414,7 +4414,7 @@ dependencies = [ [[package]] name = "pdfcraft-crypt" -version = "0.21.0" +version = "0.21.1" dependencies = [ "aes", "md-5", @@ -4427,7 +4427,7 @@ dependencies = [ [[package]] name = "pdfcraft-edit" -version = "0.21.0" +version = "0.21.1" dependencies = [ "hayro-syntax", "icu_segmenter", @@ -4446,7 +4446,7 @@ dependencies = [ [[package]] name = "pdfcraft-engine" -version = "0.21.0" +version = "0.21.1" dependencies = [ "chrono", "craft-docpath", @@ -4490,14 +4490,14 @@ dependencies = [ [[package]] name = "pdfcraft-export" -version = "0.21.0" +version = "0.21.1" dependencies = [ "flate2", ] [[package]] name = "pdfcraft-filters" -version = "0.21.0" +version = "0.21.1" dependencies = [ "fax", "flate2", @@ -4510,7 +4510,7 @@ dependencies = [ [[package]] name = "pdfcraft-fonts" -version = "0.21.0" +version = "0.21.1" dependencies = [ "harfrust", "pdfcraft-cos", @@ -4520,7 +4520,7 @@ dependencies = [ [[package]] name = "pdfcraft-forms" -version = "0.21.0" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4533,11 +4533,11 @@ dependencies = [ [[package]] name = "pdfcraft-geom" -version = "0.21.0" +version = "0.21.1" [[package]] name = "pdfcraft-js" -version = "0.21.0" +version = "0.21.1" dependencies = [ "boa_engine", "chrono", @@ -4545,7 +4545,7 @@ dependencies = [ [[package]] name = "pdfcraft-measure" -version = "0.21.0" +version = "0.21.1" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4558,14 +4558,14 @@ dependencies = [ [[package]] name = "pdfcraft-model" -version = "0.21.0" +version = "0.21.1" dependencies = [ "pdfcraft-cos", ] [[package]] name = "pdfcraft-ocr" -version = "0.21.0" +version = "0.21.1" dependencies = [ "ocrs", "pdfcraft-fonts", @@ -4580,7 +4580,7 @@ dependencies = [ [[package]] name = "pdfcraft-ocr-worker" -version = "0.21.0" +version = "0.21.1" dependencies = [ "js-sys", "pdfcraft-ocr", @@ -4590,7 +4590,7 @@ dependencies = [ [[package]] name = "pdfcraft-optimize" -version = "0.21.0" +version = "0.21.1" dependencies = [ "image", "pdfcraft-annot", @@ -4607,11 +4607,11 @@ dependencies = [ [[package]] name = "pdfcraft-oracle" -version = "0.21.0" +version = "0.21.1" [[package]] name = "pdfcraft-organize" -version = "0.21.0" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4622,7 +4622,7 @@ dependencies = [ [[package]] name = "pdfcraft-platform" -version = "0.21.0" +version = "0.21.1" dependencies = [ "async-channel", "futures-lite", @@ -4633,7 +4633,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc" -version = "0.21.0" +version = "0.21.1" dependencies = [ "flate2", "proptest", @@ -4642,7 +4642,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc-render" -version = "0.21.0" +version = "0.21.1" dependencies = [ "flate2", "pdfcraft-prc", @@ -4651,7 +4651,7 @@ dependencies = [ [[package]] name = "pdfcraft-preflight" -version = "0.21.0" +version = "0.21.1" dependencies = [ "flate2", "pdfcraft-content", @@ -4662,7 +4662,7 @@ dependencies = [ [[package]] name = "pdfcraft-print" -version = "0.21.0" +version = "0.21.1" dependencies = [ "craft-print", "log", @@ -4677,7 +4677,7 @@ dependencies = [ [[package]] name = "pdfcraft-redact" -version = "0.21.0" +version = "0.21.1" dependencies = [ "aho-corasick", "pdfcraft-annot", @@ -4701,7 +4701,7 @@ dependencies = [ [[package]] name = "pdfcraft-render" -version = "0.21.0" +version = "0.21.1" dependencies = [ "bytemuck", "flate2", @@ -4725,7 +4725,7 @@ dependencies = [ [[package]] name = "pdfcraft-sign" -version = "0.21.0" +version = "0.21.1" dependencies = [ "aes", "aws-lc-rs", @@ -4764,11 +4764,11 @@ dependencies = [ [[package]] name = "pdfcraft-testkit" -version = "0.21.0" +version = "0.21.1" [[package]] name = "pdfcraft-ui-egui" -version = "0.21.0" +version = "0.21.1" dependencies = [ "arboard", "base64 0.23.1", @@ -4827,7 +4827,7 @@ dependencies = [ [[package]] name = "pdfcraft-web" -version = "0.21.0" +version = "0.21.1" dependencies = [ "craft-web-api", "craft-web-host", @@ -4842,7 +4842,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfa" -version = "0.21.0" +version = "0.21.1" dependencies = [ "base64 0.23.1", "image", @@ -4856,7 +4856,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfdf" -version = "0.21.0" +version = "0.21.1" dependencies = [ "base64 0.23.1", "pdfcraft-annot", diff --git a/Cargo.toml b/Cargo.toml index 9cd5b3b7c3..752f580475 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor"] default-members = ["crates/*", "apps/pdfcraft", "apps/pdfcraft-cli", "xtask"] [workspace.package] -version = "0.21.0" +version = "0.21.1" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" diff --git a/crates/ui-egui/tests/flatpak_updates.rs b/crates/ui-egui/tests/flatpak_updates.rs new file mode 100644 index 0000000000..536613f08a --- /dev/null +++ b/crates/ui-egui/tests/flatpak_updates.rs @@ -0,0 +1,27 @@ +//! Release and source Flatpaks must allow release checks and host update helpers. +fn finish_args(manifest: &str) -> Vec<&str> { + manifest + .split("finish-args:") + .nth(1) + .unwrap_or_default() + .lines() + .take_while(|line| line.is_empty() || line.starts_with(' ') || line.starts_with('#')) + .filter_map(|line| line.trim().strip_prefix("- ")) + .map(|arg| arg.trim_matches(['\'', '"'])) + .collect() +} + +#[test] +fn flatpak_update_permissions() { + let manifests = [ + include_str!("../../../packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml"), + include_str!("../../../packaging/linux/flatpak/ai.storyteller.pdfcraft.yml"), + ]; + let args = manifests.map(finish_args); + assert_eq!(args[0], args[1], "source and release permissions must agree"); + for permissions in args { + for required in ["--share=network", "--talk-name=org.freedesktop.Flatpak"] { + assert!(permissions.contains(&required), "missing update permission: {required}"); + } + } +} diff --git a/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml b/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml index d781ad6c7c..8d2fb49192 100644 --- a/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml +++ b/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml @@ -17,6 +17,8 @@ runtime-version: '25.08' sdk: org.freedesktop.Sdk command: pdfcraft finish-args: + # The shared updater inspects and updates this installation via flatpak-spawn --host. + - --talk-name=org.freedesktop.Flatpak - --filesystem=xdg-data/ArtCraft/versions:ro # Display: Wayland first; X11 only when no Wayland compositor is running. - --socket=wayland diff --git a/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml b/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml index fba62fe382..2c5d6d88c6 100644 --- a/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml +++ b/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml @@ -18,6 +18,8 @@ sdk-extensions: - org.freedesktop.Sdk.Extension.rust-stable command: pdfcraft finish-args: + # The shared updater inspects and updates this installation via flatpak-spawn --host. + - --talk-name=org.freedesktop.Flatpak - --filesystem=xdg-data/ArtCraft/versions:ro # Display: Wayland first; X11 only when no Wayland compositor is running. - --socket=wayland From d749fd8062655c333c0733d8456bd8d4c9925a12 Mon Sep 17 00:00:00 2001 From: Ken Mantle Date: Thu, 12 Nov 2026 00:28:18 -0500 Subject: [PATCH 43/46] Print colour management: the app converts printed pictures into a printer profile (#6269) (#6853) * Print: app-managed colour converts printed pictures into a printer profile (#6269) doc_print takes color_management (printer, the default, or app). With app, color_profile (an ICC file or an installed profile's name), rendering_intent and black_point_compensation convert each rendered page picture from sRGB into the printer's RGB or grey profile before it is sent. app prints the pages as pictures; CMYK and non-destination profiles are refused. print_color_profiles lists the installed RGB and grey profiles. The conversion is craft-cms at craft-print's rev, in pdfcraft_print::colour, hooked into print_raster::rasterize. Raster carries the colour and is Clone rather than Copy. Co-Authored-By: Claude Opus 5.5 * Print colour: look for installed profiles inside the read root when confined With automation roots, print_color_profiles and doc_print's color_profile name lookup read the system's colour folders, outside the roots. They now look in the read root instead (as text_font_files does), check each file against it before reading it, and are refused without read authority. Under --root they look in the root. Co-Authored-By: Claude Opus 5.5 * automation: keep colour profile discovery inside --root, links followed Under --root alone, print_color_profiles and the color_profile name lookup listed the root's .icc files without checking each against the root, so a link inside it could read a profile outside. Each discovered file now goes through the same --root resolution as any other read (links followed) before it is opened. colour_profile_links_out_of_the_root_are_not_read makes such a link and checks it is neither listed nor found by name; it fails without the check (the link is listed). Co-Authored-By: Claude Opus 5.5 * print: as_chunks for the grey fill; area print keeps managed colour Clippy 1.99 (`chunks_exact_to_as_chunks`) flags the constant-width `chunks_exact_mut(4)` in the grey conversion; use `as_chunks_mut::<4>()`. main's print-area path built `Raster` without the new `colour` field; build it through `PrintDraft::raster()` so a printed area is colour managed like a printed page. Co-Authored-By: Claude Opus 5.5 * Lock craft-cms at the craft-libs commit craft-print now uses Co-Authored-By: Claude Opus 5.5 * fix(print): read only regular files as printer profiles, and no more than the cap A FIFO or device named `.icc`/`.icm` in a profile folder was listed and read, and opening a FIFO for reading waits for a writer. The listing now keeps regular files only (links followed), `PrinterProfile::read` refuses anything else before opening it, and the read goes through a limit one byte past `MAX_PROFILE_BYTES`, so a file that grew after it was measured is refused rather than read whole. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: KenM76 Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + Cargo.toml | 2 + crates/automation/Cargo.toml | 2 + crates/automation/src/dispatch.rs | 1 + crates/automation/src/lib.rs | 1 + crates/automation/src/print_colour.rs | 90 +++++ crates/automation/src/printing.rs | 33 +- crates/automation/src/tools.rs | 9 +- crates/automation/tests/it/automation.rs | 75 ++++ .../automation/tests/it/automation_roots.rs | 24 ++ crates/engine/Cargo.toml | 2 + crates/engine/src/print_raster.rs | 8 +- crates/engine/src/session/files.rs | 5 +- crates/engine/src/tests.rs | 59 ++- crates/print/Cargo.toml | 1 + crates/print/src/colour.rs | 338 ++++++++++++++++++ crates/print/src/lib.rs | 1 + crates/print/src/settings.rs | 9 +- .../ui-egui/src/workflows/print_ui/draft.rs | 6 +- parity/acrobat-features.toml | 12 + 20 files changed, 662 insertions(+), 27 deletions(-) create mode 100644 crates/automation/src/print_colour.rs create mode 100644 crates/print/src/colour.rs diff --git a/Cargo.lock b/Cargo.lock index 0e4c910427..e0968ff2c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1184,6 +1184,14 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-cms" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99" +dependencies = [ + "rayon", +] + [[package]] name = "craft-docpath" version = "0.1.0" @@ -4326,6 +4334,7 @@ name = "pdfcraft-automation" version = "0.21.1" dependencies = [ "base64 0.23.1", + "craft-cms", "flate2", "image", "pdfcraft-cos", @@ -4449,6 +4458,7 @@ name = "pdfcraft-engine" version = "0.21.1" dependencies = [ "chrono", + "craft-cms", "craft-docpath", "getrandom 0.4.3", "icu_properties", @@ -4664,6 +4674,7 @@ dependencies = [ name = "pdfcraft-print" version = "0.21.1" dependencies = [ + "craft-cms", "craft-print", "log", "pdfcraft-content", diff --git a/Cargo.toml b/Cargo.toml index 752f580475..18fa9ed37b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,8 @@ craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1 craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # Shared system spooler; document composition remains in pdfcraft-print. craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } +# ICC transforms for app-managed print colour; at craft-print's commit, so one craft-libs checkout. +craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } # Whether a path stored in a document may be opened (GoToR link targets). craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # craft-update: app-specific pretend versions apply to startup and manual checks. diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index a825b1fe52..60a5d0fa06 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -28,6 +28,8 @@ default = ["mcp"] mcp = ["dep:base64"] [dev-dependencies] +# Synthetic printer profiles for the app-managed print colour tests. +craft-cms = { workspace = true } pdfcraft-fonts = { workspace = true } png = "0.18" flate2.workspace = true diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index 4a91942fe4..8de3ff3c27 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -726,6 +726,7 @@ impl Automation { "redact_clear" => self.redact_clear(&a)?, "doc_hidden_info" => self.doc_hidden_info(&a)?, "printers" => self.printers()?, + "print_color_profiles" => self.print_color_profiles()?, "printer_options" => self.printer_options(&a)?, "printer_papers" => self.printer_papers(&a)?, "link_list" => self.link_list(&a)?, diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 7fa63d1a1d..5c49b25b23 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -30,6 +30,7 @@ pub mod mcp; mod measure; mod model3d; mod model_render; +mod print_colour; mod printing; mod reading; mod redact; diff --git a/crates/automation/src/print_colour.rs b/crates/automation/src/print_colour.rs new file mode 100644 index 0000000000..22279fbd8b --- /dev/null +++ b/crates/automation/src/print_colour.rs @@ -0,0 +1,90 @@ +//! App-managed print colour for `doc_print` (`color_management` app) and the installed printer +//! profiles it can name (`print_color_profiles`). + +use pdfcraft_engine::print::colour::{self, Intent, ManagedColour, PrinterProfile}; +use serde_json::{Value, json}; + +use crate::{Args, Automation, Result, ToolError, failed}; + +/// An invalid-arguments error saying `m`. +fn bad(m: impl Into) -> ToolError { + ToolError::InvalidArgs(m.into()) +} + +impl Automation { + /// `doc_print`'s colour handling: `None` for `printer` (the default), else the profile, intent + /// and black point compensation the pictures are converted with. + pub(crate) fn managed_colour(&self, a: &Args) -> Result> { + let app = match a.opt_str("color_management")?.unwrap_or("printer") { + "printer" => false, + "app" => true, + m => return Err(bad(format!("unknown color_management {m:?}: printer (the printer manages colour) or app"))), + }; + let asked = ["color_profile", "rendering_intent", "black_point_compensation"].into_iter().find(|k| a.get(k).is_some()); + if !app { + return match asked { + Some(k) => Err(bad(format!("{k} needs color_management app"))), + None => Ok(None), + }; + } + let name = a + .opt_str("color_profile")? + .ok_or_else(|| bad("color_management app needs color_profile: an ICC file or a name from print_color_profiles"))?; + let profile = self.printer_profile(name)?; + let intent = match a.opt_str("rendering_intent")? { + None => Intent::RelativeColorimetric, + Some(i) => { + Intent::parse(i).ok_or_else(|| bad(format!("unknown rendering_intent {i:?}: perceptual, relative, saturation or absolute")))? + } + }; + let black_point = a.opt_bool("black_point_compensation")?.unwrap_or(true); + Ok(Some(ManagedColour { profile, intent, black_point })) + } + + /// The profile `name` names: an ICC file, else an installed profile's name or file name. A + /// file is tried first so naming one doesn't read every installed profile. + fn printer_profile(&self, name: &str) -> Result { + let file = self.resolve_for(name, crate::file_access::Access::Read, true); + if let Ok(path) = &file + && path.is_file() + { + return PrinterProfile::read(path).map_err(bad); + } + let installed = match self.installed_profiles() { + Ok(list) => list, + // Confined without read authority: say why the file was refused, if it was. + Err(e) => return Err(file.err().unwrap_or(e)), + }; + if let Some(p) = colour::find_installed(&installed, name) { + return PrinterProfile::read(&p.path).map_err(failed); + } + file?; + Err(bad(format!("color_profile {name:?} is neither an installed profile (print_color_profiles) nor an ICC file"))) + } + + /// The profiles that can be named: the system's, or, when file access is confined, those in + /// the read root or `--root` (as `text_font_files` does), each file checked against it, links + /// followed, before it is read. + fn installed_profiles(&self) -> Result> { + let Some(roots) = &self.roots else { + let Some(root) = &self.root else { return Ok(colour::installed_profiles()) }; + let inside = |path: &std::path::Path| self.resolve(&path.to_string_lossy(), false).is_ok(); + return Ok(colour::installed_profiles_where(std::slice::from_ref(root), &inside)); + }; + let read = roots + .read_root() + .ok_or_else(|| failed("automation filesystem access is not granted: read authority is absent: installed colour profiles"))?; + let folders = vec![self.resolve(&read.to_string_lossy(), false)?]; + let inside = |path: &std::path::Path| roots.check(&path.to_string_lossy(), crate::file_access::Access::Read).is_ok(); + Ok(colour::installed_profiles_where(&folders, &inside)) + } + + pub(crate) fn print_color_profiles(&self) -> Result { + let list: Vec = self + .installed_profiles()? + .into_iter() + .map(|p| json!({ "name": p.name, "path": p.path.to_string_lossy(), "color_space": if p.gray { "gray" } else { "rgb" } })) + .collect(); + Ok(json!({ "count": list.len(), "profiles": list })) + } +} diff --git a/crates/automation/src/printing.rs b/crates/automation/src/printing.rs index 3d7a76865a..d099d4dc01 100644 --- a/crates/automation/src/printing.rs +++ b/crates/automation/src/printing.rs @@ -118,7 +118,7 @@ fn spool_job( title: String, duplex: Option, media: Option<&str>, - raster: Option, + raster: Option<&print::Raster>, sheets: &[print::Sheet], ) -> Result { let options = match a.get("options") { @@ -338,12 +338,19 @@ impl Automation { let (paper, media) = resolve_paper(a.opt_str("paper")?, resolved_printer.as_deref())?; // Print as image (Acrobat's Advanced Print Setup): pictures at `dpi`, grey when // `grayscale` is also asked for. - let raster = if a.opt_bool("as_image")?.unwrap_or(false) { + // App-managed colour converts those pictures into the printer's profile, so it prints + // as image whether or not as_image is given. + let colour = self.managed_colour(a)?; + let as_image = match a.opt_bool("as_image")? { + Some(false) if colour.is_some() => return Err(bad("color_management app prints the pages as images: leave out as_image false")), + asked => asked.unwrap_or(false) || colour.is_some(), + }; + let raster = if as_image { let dpi = a.opt_num("dpi")?.unwrap_or(300.0); if !(72.0..=600.0).contains(&dpi) { return Err(bad("dpi must be between 72 and 600")); } - Some(print::Raster { dpi, grayscale: a.opt_bool("grayscale")?.unwrap_or(false) }) + Some(print::Raster { dpi, grayscale: a.opt_bool("grayscale")?.unwrap_or(false), colour }) } else { None }; @@ -367,7 +374,15 @@ impl Automation { (Some(p), None) => Destination::File(self.resolve(p, true)?), (None, Some(printer)) => { let actual_printer = resolved_printer.as_deref().unwrap_or(printer); - Destination::Printer(spool_job(a, actual_printer, self.doc(a)?.name.clone(), duplex, media.as_deref(), settings.raster, &laid_out)?) + Destination::Printer(spool_job( + a, + actual_printer, + self.doc(a)?.name.clone(), + duplex, + media.as_deref(), + settings.raster.as_ref(), + &laid_out, + )?) } _ => return Err(bad("pass either path (save the print-ready PDF) or printer (a name from printers, or \"default\")")), }; @@ -379,6 +394,10 @@ impl Automation { }; let bytes = self.session.print_pdf(id, &settings).map_err(failed)?; let mut out = json!({ "sheets": sheets, "pages": settings.pages.len(), "bytes": bytes.len() }); + if let Some(c) = settings.raster.as_ref().and_then(|r| r.colour.as_ref()) { + out["color_management"] = + json!({ "profile": c.profile.name(), "rendering_intent": c.intent.id(), "black_point_compensation": c.black_point }); + } if !scripts.is_empty() { out["will_print"] = crate::a11y::form_tools::js_output_json(&scripts); } @@ -542,10 +561,10 @@ mod tests { /// grayscale, which Microsoft Print to PDF keeps no setting for and the job was refused. #[test] fn grey_pictures_dont_ask_the_driver_for_grayscale() { - let grey = print::Raster { dpi: 300.0, grayscale: true }; - let colour = print::Raster { dpi: 300.0, grayscale: false }; + let grey = print::Raster { dpi: 300.0, grayscale: true, colour: None }; + let colour = print::Raster { dpi: 300.0, grayscale: false, colour: None }; let job = |args: Value, raster: Option| { - spool_job(&Args(&args), "Office", String::new(), None, Some("Letter"), raster, &[]).unwrap().grayscale + spool_job(&Args(&args), "Office", String::new(), None, Some("Letter"), raster.as_ref(), &[]).unwrap().grayscale }; assert!(job(json!({ "grayscale": true }), None), "vector pages: the driver makes them grey"); assert!(!job(json!({ "grayscale": true }), Some(grey)), "grey pictures: nothing for the driver to do"); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index fa29d07634..9cecf1654c 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -751,6 +751,9 @@ pub fn tools() -> Vec { .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t("printers", "List printers", "The printers the system's print spooler knows (CUPS on macOS and Linux), with the default marked.").ro().with(schema(json!({}), &[])), + t("print_color_profiles", "List printer colour profiles", "The RGB and grey ICC profiles installed on this computer that doc_print can convert the pages into (color_management app): name, path and color_space. Windows: the system colour folder; macOS: the ColorSync folders; Linux: the shared and per-user icc folders. CMYK profiles are left out: the printed pictures are RGB.") + .ro() + .with(schema(json!({}), &[])), t("printer_options", "List printer options", "A printer driver's own job options (CUPS: from its PPD), such as the paper tray, paper type or finishing: key, label, group, default and choices. Pass chosen values to doc_print as options. Empty on Windows, where the driver's preferences window holds them.") .ro() .with(schema(json!({ "printer": { "type": "string" } }), &["printer"])), @@ -764,7 +767,7 @@ pub fn tools() -> Vec { t( "doc_print", "Print", - "Print with Acrobat's Print dialog options, or save the print-ready PDF. pages: a range such as \"1-3, 6, 9-\" (page labels allowed, \"(n)\" for the nth page whatever its label, unless a page is labelled exactly \"(n)\", since labels are looked up first; default all); subset odd/even (by page number, as Acrobat counts them: pages 2, 7-10 with even prints 2, 8 and 10, not every second page of the range); reverse. layout: fit (default), actual, shrink, custom (scale %), multiple (per_sheet 2/4/6/9/16, which turns the sheet to suit the grid, or a custom grid of cols across by rows down, each 1-99, on the sheet as oriented (auto prints portrait); order, border, auto_rotate; cut-stack order arranges single-sided sheets for cutting into piles and stacking left to right, top to bottom, keeping sheet order within each pile; duplex must be off), booklet (booklet_subset both/front/back, binding left/right; a booklet sets two-sided itself, as Acrobat does: both flips on the short edge, front or back prints one-sided), poster (scale %, overlap pt, cut_marks). orientation auto/portrait/landscape; comments_forms document / document-and-markups (default) / document-and-stamps / form-fields-only; paper Letter/Legal/Tabloid, ARCH A/B/C/D/E1/E, ANSI C/D/E, A0/A1/A2/A3/A4/A5/A6, US #10 Envelope, a driver paper name from printer_papers, or dimensions \"x\"; when printer is given and paper is omitted, uses the printer's default paper if available; use layout actual for 100% in the print-ready PDF; printer drivers may still substitute paper or scale. Then path (save) or printer (a name or \"default\") with copies, collate, duplex off/long-edge/short-edge, grayscale, and options (driver options from printer_options, CUPS).", + "Print with Acrobat's Print dialog options, or save the print-ready PDF. pages: a range such as \"1-3, 6, 9-\" (page labels allowed, \"(n)\" for the nth page whatever its label, unless a page is labelled exactly \"(n)\", since labels are looked up first; default all); subset odd/even (by page number, as Acrobat counts them: pages 2, 7-10 with even prints 2, 8 and 10, not every second page of the range); reverse. layout: fit (default), actual, shrink, custom (scale %), multiple (per_sheet 2/4/6/9/16, which turns the sheet to suit the grid, or a custom grid of cols across by rows down, each 1-99, on the sheet as oriented (auto prints portrait); order, border, auto_rotate; cut-stack order arranges single-sided sheets for cutting into piles and stacking left to right, top to bottom, keeping sheet order within each pile; duplex must be off), booklet (booklet_subset both/front/back, binding left/right; a booklet sets two-sided itself, as Acrobat does: both flips on the short edge, front or back prints one-sided), poster (scale %, overlap pt, cut_marks). orientation auto/portrait/landscape; comments_forms document / document-and-markups (default) / document-and-stamps / form-fields-only; paper Letter/Legal/Tabloid, ARCH A/B/C/D/E1/E, ANSI C/D/E, A0/A1/A2/A3/A4/A5/A6, US #10 Envelope, a driver paper name from printer_papers, or dimensions \"x\"; when printer is given and paper is omitted, uses the printer's default paper if available; use layout actual for 100% in the print-ready PDF; printer drivers may still substitute paper or scale. Then path (save) or printer (a name or \"default\") with copies, collate, duplex off/long-edge/short-edge, grayscale, and options (driver options from printer_options, CUPS). color_management app (Acrobat's Color Handling, the app managing colours) prints the pages as images converted into color_profile with rendering_intent and black_point_compensation; turn the driver's own colour management off for that job, or the printer converts them again.", ) .with(schema( json!({ @@ -798,6 +801,10 @@ pub fn tools() -> Vec { "as_image": { "type": "boolean", "description": "Print as image (Acrobat's Advanced Print Setup): every page rendered to a picture at dpi before it is placed on the paper, for printers that skip text, shading or transparency; the print is no longer searchable text, and with grayscale the pictures are grey, so the printer isn't asked for grayscale (a driver without that setting still prints them)." }, "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution for as_image (default 300)." }, "options": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Printer driver options from printer_options: key → choice value." }, + "color_management": { "type": "string", "enum": ["printer", "app"], "description": "Who converts the colours: printer (default, the driver does) or app, which prints the pages as images (dpi, grayscale apply) converted from sRGB into color_profile." }, + "color_profile": { "type": "string", "description": "For color_management app: a name or file name from print_color_profiles, or the path of an RGB or grey ICC printer profile." }, + "rendering_intent": { "type": "string", "enum": ["perceptual", "relative", "saturation", "absolute"], "description": "For color_management app: how colours the printer can't reach are mapped (default relative colorimetric)." }, + "black_point_compensation": { "type": "boolean", "description": "For color_management app: map sRGB black onto the printer's darkest black, so shadows stay apart instead of clipping (default true; no effect with absolute)." }, }), &["doc"], )), diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index db139844a7..f749ed2953 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -12485,6 +12485,81 @@ fn doc_print_as_image() { } } +/// #6269: color_management app prints converted pictures and says how; the profile must be an +/// RGB or grey one, and the colour arguments need app. +#[test] +fn doc_print_with_app_managed_colour() { + let dir = workdir("print-colour"); + std::fs::write(dir.join("wide.icc"), craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("press.icc"), craft_cms::Builtin::CoatedCmyk.profile().to_bytes().as_slice()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let args = |extra: Value| { + let mut v = json!({ "doc": doc, "pages": "1", "dpi": 72, "path": "managed.pdf" }); + v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + v + }; + let r = ok(&mut a, "doc_print", args(json!({ "color_management": "app", "color_profile": "wide.icc", "rendering_intent": "perceptual" }))); + assert_eq!(r["color_management"]["rendering_intent"], "perceptual", "{r}"); + assert_eq!(r["color_management"]["black_point_compensation"], true, "{r}"); + let out = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join("managed.pdf")).unwrap())).unwrap(); + let pictures = + out.scan_objects().filter(|(_, o)| matches!(&**o, pdfcraft_cos::Object::Stream(st) if st.dict.name(b"Subtype") == Some(b"Image"))).count(); + assert_eq!(pictures, 1, "app-managed colour prints the page as a picture"); + assert!(ok(&mut a, "doc_print", args(json!({})))["color_management"].is_null(), "the printer manages colour by default"); + for (extra, says) in [ + (json!({ "color_management": "app", "color_profile": "press.icc" }), "RGB or grey"), + (json!({ "color_management": "app" }), "color_profile"), + (json!({ "color_management": "app", "color_profile": "missing.icc" }), "print_color_profiles"), + (json!({ "color_profile": "wide.icc" }), "color_management app"), + (json!({ "color_management": "app", "color_profile": "wide.icc", "rendering_intent": "vivid" }), "rendering_intent"), + (json!({ "color_management": "app", "color_profile": "wide.icc", "as_image": false }), "as_image"), + (json!({ "color_management": "driver" }), "color_management"), + ] { + match a.call("doc_print", &args(extra.clone())) { + Err(ToolError::InvalidArgs(m)) => assert!(m.contains(says), "{extra}: {m}"), + other => panic!("{extra}: {other:?}"), + } + } + let listed = ok(&mut a, "print_color_profiles", json!({})); + assert_eq!(listed["count"].as_u64(), listed["profiles"].as_array().map(|p| p.len() as u64), "{listed}"); + // Under a root the profiles listed are the root's RGB and grey ones. + assert_eq!(listed["count"], 1, "{listed}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Under `--root`, a profile link inside the root that leads out of it is neither listed nor +/// found by name. +#[test] +fn colour_profile_links_out_of_the_root_are_not_read() { + let dir = workdir("print-colour-link"); + let outside = dir.with_extension("outside"); + let _ = std::fs::remove_dir_all(&outside); + std::fs::create_dir_all(&outside).unwrap(); + let profile = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + std::fs::write(dir.join("wide.icc"), profile.as_slice()).unwrap(); + std::fs::write(outside.join("other.icc"), profile.as_slice()).unwrap(); + #[cfg(unix)] + let linked = std::os::unix::fs::symlink(outside.join("other.icc"), dir.join("leak.icc")).is_ok(); + #[cfg(windows)] + let linked = std::os::windows::fs::symlink_file(outside.join("other.icc"), dir.join("leak.icc")).is_ok(); + #[cfg(not(any(unix, windows)))] + let linked = false; + // Windows needs Developer Mode or elevation to make links. + if !linked { + eprintln!("skipped: symbolic links can't be made here"); + return; + } + let mut a = auto(&dir); + let listed = ok(&mut a, "print_color_profiles", json!({})); + assert_eq!(listed["count"], 1, "only the profile inside the root: {listed}"); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let print = json!({ "doc": doc, "pages": "1", "dpi": 72, "path": "m.pdf", "color_management": "app", "color_profile": "leak" }); + assert!(a.call("doc_print", &print).is_err(), "a link out of the root isn't found by name"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); +} + /// Without a root, `write_output` still writes through a staging file (#2166): the folder is /// made, nothing is left beside the file, and a folder at the path is refused. #[test] diff --git a/crates/automation/tests/it/automation_roots.rs b/crates/automation/tests/it/automation_roots.rs index e65c582a73..ecc65c742c 100644 --- a/crates/automation/tests/it/automation_roots.rs +++ b/crates/automation/tests/it/automation_roots.rs @@ -200,6 +200,30 @@ fn tools_that_read_a_path_are_held_to_the_read_root() { ok(&mut server, "text_font_files", json!({"directories": [s(&base.join("in"))]})); } +/// The installed colour profiles are outside the read root: confined, `print_color_profiles` +/// lists and `doc_print` names only the profiles inside it, and without read authority none. +#[test] +fn colour_profiles_are_looked_for_inside_the_read_root() { + let (mut server, base) = confined("profiles", true, true); + let profile = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + std::fs::write(base.join("in/wide.icc"), profile.as_slice()).unwrap(); + std::fs::write(base.join("outside/other.icc"), profile.as_slice()).unwrap(); + let listed = ok(&mut server, "print_color_profiles", json!({})); + let paths: Vec<&str> = listed["profiles"].as_array().unwrap().iter().filter_map(|p| p["path"].as_str()).collect(); + assert_eq!(paths.len(), 1, "{listed}"); + assert!(paths[0].ends_with("wide.icc"), "{listed}"); + ok(&mut server, "doc_open", json!({"path": s(&base.join("in/a.pdf"))})); + let print = + |name: &str| json!({"doc": 1, "pages": "1", "dpi": 72, "path": s(&base.join("out/m.pdf")), "color_management": "app", "color_profile": name}); + ok(&mut server, "doc_print", print("wide")); + let (error, text) = call(&mut server, "doc_print", print("other")); + assert!(error, "a profile outside the read root isn't found by name: {text}"); + refused(&mut server, "doc_print", print(&s(&base.join("outside/other.icc"))), "outside the read root"); + + let (mut unread, _) = confined("profiles-unread", false, true); + refused(&mut unread, "print_color_profiles", json!({}), "read authority is absent"); +} + #[test] fn tools_that_write_a_path_are_held_to_the_write_root() { let (mut server, base) = confined("writes", true, true); diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index f5b924c921..4278dbb95f 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -58,6 +58,8 @@ getrandom = "0.4" getrandom = { version = "0.4", features = ["wasm_js"] } [dev-dependencies] +# Synthetic printer profiles for the app-managed print colour tests. +craft-cms = { workspace = true } pdfcraft-testkit = { workspace = true } # JBIG2 fixtures (`encode_jbig2_generic`). pdfcraft-filters = { workspace = true } diff --git a/crates/engine/src/print_raster.rs b/crates/engine/src/print_raster.rs index a6076d5b05..7a3bbc40d5 100644 --- a/crates/engine/src/print_raster.rs +++ b/crates/engine/src/print_raster.rs @@ -3,7 +3,8 @@ //! shading or transparency. The pages are rendered the way the viewer shows them (reading //! colours off, the print's own colours) with only what the print's content choice keeps, made //! into an image-only document, and that document is imposed on the paper like any other. The -//! print is no longer searchable text. +//! print is no longer searchable text. With app-managed colour the pictures are converted into +//! the printer's ICC profile ([`pdfcraft_print::colour`]). use pdfcraft_cos::{SaveOptions, write_incremental}; use pdfcraft_create::ImageResolution; @@ -51,6 +52,11 @@ pub fn rasterize(doc: &Document, pages: &[usize], raster: &Raster, content: Cont } let mut rgba = r.rgba.to_vec(); over_white(&mut rgba, raster.grayscale); + // App-managed colour: the picture's values become the printer's, so it is placed as + // device colour the driver passes on unconverted. + if let Some(colour) = &raster.colour { + colour.convert(&mut rgba).map_err(|e| format!("page {}: {e}", page + 1))?; + } images.push((format!("page-{}.png", page + 1), encode_png(r.width, r.height, &rgba)?)); } let dpi = used.unwrap_or(dpi); diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index 2f2774ed22..85a6d24532 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -236,8 +236,9 @@ impl Session { // Table 22, bit 12 clear: only a low-level representation may be printed, so the pages // always go as pictures, at no more than the low resolution. let low_res = doc.permissions().is_some_and(|p| !p.print_high_quality()).then(|| print::Raster { - dpi: settings.raster.map_or(LOW_RESOLUTION_PRINT_DPI, |r| r.dpi.min(LOW_RESOLUTION_PRINT_DPI)), - grayscale: settings.raster.is_some_and(|r| r.grayscale), + dpi: settings.raster.as_ref().map_or(LOW_RESOLUTION_PRINT_DPI, |r| r.dpi.min(LOW_RESOLUTION_PRINT_DPI)), + grayscale: settings.raster.as_ref().is_some_and(|r| r.grayscale), + colour: settings.raster.as_ref().and_then(|r| r.colour.clone()), }); // Print as image: the pages used become pictures first, then the picture document is // imposed in their place (its page k is the k-th distinct source page used). diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 91b00df4cc..bf1a4ebd01 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1935,8 +1935,8 @@ fn low_resolution_printing_prints_pictures_of_at_most_150_dpi() { let low = s.open("low.pdf", None, bytes.clone(), None).unwrap(); let width = f64::from(s.get(low).unwrap().info.pages[0].width); let actual = print::Settings { pages: vec![0], layout: print::Layout::Size(print::SizeMode::Actual), ..Default::default() }; - for raster in [None, Some(print::Raster { dpi: 600.0, grayscale: false })] { - let printed = s.print_pdf(low, &print::Settings { raster, ..actual.clone() }).unwrap(); + for raster in [None, Some(print::Raster { dpi: 600.0, grayscale: false, colour: None })] { + let printed = s.print_pdf(low, &print::Settings { raster: raster.clone(), ..actual.clone() }).unwrap(); let mut s2 = Session::new(); let id2 = s2.open("print.pdf", None, Arc::new(printed.clone()), None).unwrap(); assert!(page_texts(&s2, id2).iter().all(|t| t.trim().is_empty()), "{raster:?}: no text is left"); @@ -3934,8 +3934,8 @@ fn printing_applies_the_print_auto_state() { } // Hidden in the panel, View entry (no Print recommendation) leaves the preview too. assert!(printed_blue[1] * 4 < printed_blue[0] * 3, "the preview follows the panel: {printed_blue:?} blue"); - for raster in [None, Some(print::Raster { dpi: 72.0, grayscale: false })] { - let bytes = Arc::new(s.print_pdf(id, &print::Settings { pages: vec![0], raster, ..Default::default() }).unwrap()); + for raster in [None, Some(print::Raster { dpi: 72.0, grayscale: false, colour: None })] { + let bytes = Arc::new(s.print_pdf(id, &print::Settings { pages: vec![0], raster: raster.clone(), ..Default::default() }).unwrap()); let mut out = Session::new(); let printed = out.open("print.pdf", None, bytes.clone(), None).unwrap(); let (red, green, blue) = colours(&render_page(&out, printed)); @@ -5839,7 +5839,7 @@ fn assert_signature_artwork_survives_trust(mut cos: pdfcraft_cos::Document, expe } let mut settings = print::Settings { pages: vec![0], paper: (200.0, 200.0), ..Default::default() }; let original_vector_print = printed_pixels(session.print_pdf(id, &settings).unwrap()); - settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false, colour: None }); let original_raster_print = printed_pixels(session.print_pdf(id, &settings).unwrap()); session.set_trusted_certificates(vec![anchor.clone()]); let doc = session.get(id).unwrap(); @@ -6204,7 +6204,7 @@ fn named_layer_states_reach_exports_and_both_print_modes() { assert!(text.contains("Signature valid") && !text.contains("Not Verified"), "{text:?}"); let mut settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; let (_, _, vector) = printed_pixels(s.print_pdf(id, &settings).unwrap()); - settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false, colour: None }); let (_, _, raster) = printed_pixels(s.print_pdf(id, &settings).unwrap()); for (mode, pixels) in [("vector", &vector), ("raster", &raster)] { assert!(pixels_near(pixels, VALID_GREEN) > 500, "{mode} print draws the valid state"); @@ -11319,7 +11319,7 @@ fn printing_as_image_sends_pictures_instead_of_pages() { let settings = print::Settings { pages: vec![1, 2, 1], layout: print::Layout::Size(print::SizeMode::Actual), - raster: Some(print::Raster { dpi: 100.0, grayscale: false }), + raster: Some(print::Raster { dpi: 100.0, grayscale: false, colour: None }), ..Default::default() }; let bytes = s.print_pdf(id, &settings).unwrap(); @@ -11341,7 +11341,8 @@ fn printing_as_image_sends_pictures_instead_of_pages() { assert!((w as f64 - page_size.0 * 100.0 / 72.0).abs() <= 1.0, "{w} px for {} pt", page_size.0); assert!(images.iter().all(|d| d.name(b"ColorSpace") == Some(b"DeviceRGB")), "{images:?}"); // Grayscale renders the pictures grey (the printer is not merely asked for monochrome). - let grey = s.print_pdf(id, &print::Settings { raster: Some(print::Raster { dpi: 72.0, grayscale: true }), ..settings.clone() }).unwrap(); + let grey = + s.print_pdf(id, &print::Settings { raster: Some(print::Raster { dpi: 72.0, grayscale: true, colour: None }), ..settings.clone() }).unwrap(); let mut s3 = Session::new(); let id3 = s3.open("grey.pdf", None, Arc::new(grey), None).unwrap(); let mut ex = crate::export::Exporter::new(s3.get(id3).unwrap()); @@ -11350,6 +11351,44 @@ fn printing_as_image_sends_pictures_instead_of_pages() { assert!(img.pixels().all(|p| p[0] == p[1] && p[1] == p[2]), "every pixel grey"); } +/// #6269: with app-managed colour the printed pictures carry the printer profile's values, not +/// sRGB's: a pure green page converted into a wider-gamut RGB profile gains red; without it the +/// pictures stay sRGB. +#[test] +fn app_managed_colour_converts_the_printed_pictures() { + let pdf = pdfcraft_testkit::Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Contents 4 0 R >>") + .stream("", "0 1 0 rg 0 0 100 100 re f") + .build(); + let mut s = Session::new(); + let id = s.open("green.pdf", None, Arc::new(pdf), None).unwrap(); + let wide = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + let colour = print::colour::ManagedColour { + profile: print::colour::PrinterProfile::from_bytes(&wide).unwrap(), + intent: print::colour::Intent::RelativeColorimetric, + black_point: true, + }; + let middle = |colour: Option| { + let settings = print::Settings { + pages: vec![0], + layout: print::Layout::Size(print::SizeMode::Actual), + raster: Some(print::Raster { dpi: 72.0, grayscale: false, colour }), + ..Default::default() + }; + let mut out = Session::new(); + let printed = out.open("print.pdf", None, Arc::new(s.print_pdf(id, &settings).unwrap()), None).unwrap(); + let png = crate::export::Exporter::new(out.get(printed).unwrap()).png(0, 72.0).unwrap(); + let img = image::load_from_memory(&png).unwrap().to_rgba8(); + img.get_pixel(img.width() / 2, img.height() / 2).0 + }; + let srgb = middle(None); + assert!(srgb[0] < 10 && srgb[1] > 245, "sRGB green: {srgb:?}"); + let converted = middle(Some(colour)); + assert!(converted[0] > 100 && converted[1] > 240, "the profile's values for sRGB green: {converted:?}"); +} + /// A one-page PDF: a JPEG logo (DCTDecode) at the top and a big text title below it. fn jpeg_cover_pdf(logo_y: i32) -> Vec { let mut jpeg = Vec::new(); @@ -13160,7 +13199,7 @@ fn buttons_and_combs_left_to_the_viewer_are_displayed() { } assert_eq!(count(&shown, (95, 84, 97, 106), &blue), 0, "no line inside a cell"); // Print as Image: the page as a picture, at 72 dpi so it is the size of the page here. - let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false }; + let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false, colour: None }; let pictures = crate::print_raster::rasterize(s.get(id).unwrap(), &[0], &raster, pdfcraft_print::Content::Document).unwrap(); let printed = render(Arc::new(pictures)); assert!(count(&printed, (24, 154, 56, 186), &red) > 60, "the check box prints"); @@ -13330,7 +13369,7 @@ fn need_appearances_buttons_are_displayed_from_their_captions() { let mut s = Session::new(); let id = s.open("form.pdf", None, Arc::new(pdf(true)), None).unwrap(); let shown = render(s.get(id).unwrap().display.clone()); - let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false }; + let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false, colour: None }; let pictures = crate::print_raster::rasterize(s.get(id).unwrap(), &[0], &raster, pdfcraft_print::Content::Document).unwrap(); let printed = render(Arc::new(pictures)); for (page, how) in [(&shown, "shown"), (&printed, "printed")] { diff --git a/crates/print/Cargo.toml b/crates/print/Cargo.toml index 9d349b6568..bec4e283a7 100644 --- a/crates/print/Cargo.toml +++ b/crates/print/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +craft-cms = { workspace = true } craft-print = { workspace = true } log = { workspace = true } pdfcraft-cos = { workspace = true } diff --git a/crates/print/src/colour.rs b/crates/print/src/colour.rs new file mode 100644 index 0000000000..5f13de3352 --- /dev/null +++ b/crates/print/src/colour.rs @@ -0,0 +1,338 @@ +//! App-managed colour for printing (Acrobat's Color Handling ▸ "app manages colours"): printed +//! pages, rendered in sRGB, are converted into the printer's ICC profile with a rendering intent +//! and optional black point compensation before the driver sees them. The default leaves colour +//! to the printer, as before. +//! +//! The pages go as converted pictures, so the driver's own colour management should be off for +//! the job; otherwise the printer converts them a second time. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub use craft_cms::Intent; +use craft_cms::{Builtin, ColorSpace, Profile, TransformOptions}; + +/// The largest profile read from disk. Printer profiles with large tables are a few megabytes. +pub const MAX_PROFILE_BYTES: u64 = 32 << 20; + +/// How many files the installed-profile listing looks at, at most. +const MAX_LISTED: usize = 2000; + +/// An ICC profile pages can be printed into: an RGB or grey profile that can be a destination. +#[derive(Clone, Debug, PartialEq)] +pub struct PrinterProfile { + profile: Arc, + name: String, +} + +impl PrinterProfile { + /// `bytes` as a printer profile. Refused when they are not an ICC profile, when it has no + /// path from the PCS to the device, or when its device space is neither RGB nor grey (the + /// pictures a print carries are RGB). + pub fn from_bytes(bytes: &[u8]) -> Result { + let profile = Profile::parse(bytes).map_err(|e| format!("not a usable ICC profile: {e}"))?; + if !matches!(profile.color_space, ColorSpace::Rgb | ColorSpace::Gray) { + return Err(format!("a {:?} profile can't be printed into: choose an RGB or grey printer profile", profile.color_space)); + } + if !profile.can_be_destination() { + return Err("the profile only describes a source: it has no table to convert colours into it".into()); + } + let name = profile.description.trim().to_string(); + Ok(PrinterProfile { profile: Arc::new(profile), name }) + } + + /// The profile in the file at `path`, at most [`MAX_PROFILE_BYTES`] long. Anything but a + /// regular file (links followed) is refused before it is opened: opening a FIFO or a device + /// can wait forever. + pub fn read(path: &Path) -> Result { + let at = |e: String| format!("{}: {e}", path.display()); + let meta = std::fs::metadata(path).map_err(|e| at(e.to_string()))?; + if !meta.is_file() { + return Err(at("not a regular file".into())); + } + if meta.len() > MAX_PROFILE_BYTES { + return Err(at(too_large())); + } + let file = std::fs::File::open(path).map_err(|e| at(e.to_string()))?; + let bytes = read_limited(file, MAX_PROFILE_BYTES).map_err(at)?; + PrinterProfile::from_bytes(&bytes).map_err(at) + } + + /// The profile's description (its `desc` tag), possibly empty. + pub fn name(&self) -> &str { + &self.name + } + + /// The ICC bytes, as read. + pub fn bytes(&self) -> Arc> { + self.profile.to_bytes() + } + + /// Whether the printer's device space is grey. + pub fn is_gray(&self) -> bool { + self.profile.color_space == ColorSpace::Gray + } +} + +/// At most `limit` bytes of `source`: a file that grew after it was measured is refused, not read +/// whole. +fn read_limited(source: impl std::io::Read, limit: u64) -> Result, String> { + use std::io::Read as _; + let mut bytes = Vec::new(); + source.take(limit.saturating_add(1)).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() as u64 > limit { + return Err(too_large()); + } + Ok(bytes) +} + +fn too_large() -> String { + format!("larger than an ICC profile can be here ({} MB)", MAX_PROFILE_BYTES >> 20) +} + +/// App-managed colour: the profile the pages are converted into, how, and whether black point +/// compensation maps the darkest sRGB black onto the printer's darkest black. +#[derive(Clone, Debug, PartialEq)] +pub struct ManagedColour { + pub profile: PrinterProfile, + pub intent: Intent, + pub black_point: bool, +} + +impl ManagedColour { + /// Converts opaque sRGB pixels (RGBA, alpha ignored and kept) into the printer profile in + /// place. A grey profile's value is written to all three channels. + pub fn convert(&self, rgba: &mut [u8]) -> Result<(), String> { + let opts = TransformOptions { intent: self.intent, bpc: self.black_point, ..TransformOptions::default() }; + let t = craft_cms::cached(Builtin::Srgb.profile(), &self.profile.profile, opts) + .map_err(|e| format!("can't convert into the printer profile: {e}"))?; + let source = rgba.to_vec(); + if self.profile.is_gray() { + let mut grey = vec![0u8; source.len() / 4]; + t.convert_u8(&source, 4, &mut grey, 1, false).map_err(|e| format!("can't convert into the printer profile: {e}"))?; + for (px, g) in rgba.as_chunks_mut::<4>().0.iter_mut().zip(grey) { + px[..3].fill(g); + } + Ok(()) + } else { + t.convert_u8(&source, 4, rgba, 4, true).map_err(|e| format!("can't convert into the printer profile: {e}")) + } + } +} + +/// A profile installed on this computer that pages can be printed into. +#[derive(Clone, Debug, PartialEq)] +pub struct InstalledProfile { + pub path: PathBuf, + /// Its description, else its file name. + pub name: String, + pub gray: bool, +} + +/// The folders the system keeps colour profiles in, those that exist. +pub fn profile_folders() -> Vec { + let mut dirs: Vec = Vec::new(); + if cfg!(windows) { + let root = std::env::var_os("SystemRoot").map_or_else(|| PathBuf::from(r"C:\Windows"), PathBuf::from); + dirs.push(root.join("System32").join("spool").join("drivers").join("color")); + } else if cfg!(target_os = "macos") { + dirs.push("/System/Library/ColorSync/Profiles".into()); + dirs.push("/Library/ColorSync/Profiles".into()); + if let Some(home) = std::env::var_os("HOME") { + dirs.push(PathBuf::from(home).join("Library/ColorSync/Profiles")); + } + } else { + dirs.push("/usr/share/color/icc".into()); + dirs.push("/usr/local/share/color/icc".into()); + if let Some(home) = std::env::var_os("HOME") { + dirs.push(PathBuf::from(&home).join(".local/share/color/icc")); + dirs.push(PathBuf::from(home).join(".color/icc")); + } + } + dirs.retain(|d| d.is_dir()); + dirs +} + +/// The RGB and grey profiles in `folders` and their immediate subfolders that pages can be +/// printed into, sorted by name. Files that aren't such a profile are skipped. +pub fn installed_profiles_in(folders: &[PathBuf]) -> Vec { + installed_profiles_where(folders, &|_| true) +} + +/// [`installed_profiles_in`], reading only the files `allow` passes (a caller confined to some +/// folders checks each file, links followed, before it is opened). +pub fn installed_profiles_where(folders: &[PathBuf], allow: &dyn Fn(&Path) -> bool) -> Vec { + let mut files: Vec = Vec::new(); + for dir in folders { + collect(dir, 1, &mut files); + } + let mut out: Vec = files + .into_iter() + .filter(|path| allow(path)) + .filter_map(|path| { + let p = PrinterProfile::read(&path).ok()?; + let name = if p.name().is_empty() { path.file_name()?.to_string_lossy().into_owned() } else { p.name().to_string() }; + Some(InstalledProfile { gray: p.is_gray(), name, path }) + }) + .collect(); + out.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()).then_with(|| a.path.cmp(&b.path))); + out +} + +/// The profiles the system has installed (see [`profile_folders`]). +pub fn installed_profiles() -> Vec { + installed_profiles_in(&profile_folders()) +} + +/// The `.icc` and `.icm` regular files in `dir` (links followed), descending `depth` more levels, +/// up to [`MAX_LISTED`]. A FIFO or device with such a name is skipped: reading it can block. +fn collect(dir: &Path, depth: u32, files: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + if files.len() >= MAX_LISTED { + return; + } + let path = entry.path(); + let Ok(kind) = entry.file_type() else { continue }; + if kind.is_dir() { + if depth > 0 { + collect(&path, depth - 1, files); + } + } else if path.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("icc") || e.eq_ignore_ascii_case("icm")) + && std::fs::metadata(&path).is_ok_and(|m| m.is_file()) + { + files.push(path); + } + } +} + +/// The installed profile `name` names: its description or its file name, ignoring case. +pub fn find_installed<'a>(profiles: &'a [InstalledProfile], name: &str) -> Option<&'a InstalledProfile> { + let name = name.trim(); + profiles.iter().find(|p| p.name.eq_ignore_ascii_case(name)).or_else(|| { + profiles.iter().find(|p| { + let file = p.path.file_name().map(|f| f.to_string_lossy()).unwrap_or_default(); + let stem = p.path.file_stem().map(|f| f.to_string_lossy()).unwrap_or_default(); + file.eq_ignore_ascii_case(name) || stem.eq_ignore_ascii_case(name) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_cms::Curve; + + /// A source longer than the limit is refused after reading one byte past it, not read whole. + #[test] + fn a_profile_read_stops_one_byte_past_the_limit() { + let source = [7u8; 100]; + assert_eq!(read_limited(&source[..], 100).map(|b| b.len()), Ok(100)); + let mut rest = &source[..]; + assert!(read_limited(&mut rest, 10).is_err()); + assert_eq!(rest.len(), 89, "read the limit and one byte more"); + } + + /// A FIFO named like a profile is neither listed nor opened: opening it for reading would wait + /// for a writer that never comes. + #[cfg(unix)] + #[test] + fn a_fifo_named_like_a_profile_is_skipped() { + let dir = std::env::temp_dir().join(format!("pdfcraft-print-fifo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("printer.icc"); + let _ = std::fs::remove_file(&fifo); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()); + if !made { + eprintln!("mkfifo is unavailable; skipped"); + return; + } + // On a thread, so that a read which blocks fails the test instead of hanging it. + let (tx, rx) = std::sync::mpsc::channel(); + let (folders, path) = (vec![dir.clone()], fifo.clone()); + std::thread::spawn(move || { + let _ = tx.send((installed_profiles_in(&folders).len(), PrinterProfile::read(&path).err())); + }); + let outcome = rx.recv_timeout(std::time::Duration::from_secs(10)); + let _ = std::fs::remove_dir_all(&dir); + let (listed, refused) = outcome.expect("reading the FIFO blocked"); + assert_eq!(listed, 0, "the listing skipped the FIFO"); + assert!(refused.is_some_and(|e| e.contains("not a regular file")), "the FIFO was refused before it was opened"); + } + + /// A grey printer profile whose darkest black is a light charcoal (8% luminance, L* 34): + /// what a matte paper or a worn toner gives. Synthetic, from the built-in sGray. + fn lifted_black_grey() -> Vec { + let mut p = Builtin::SGray.profile().clone(); + p.gray_trc = Some(Curve::Table((0..=255).map(|i| 0.08 + 0.92 * (i as f32 / 255.0)).collect())); + p.black_point = None; + p.description = "Lifted black grey".into(); + p.with_encoded_bytes().to_bytes().to_vec() + } + + fn convert(profile: &[u8], bpc: bool, rgba: &[u8]) -> Vec { + let colour = ManagedColour { profile: PrinterProfile::from_bytes(profile).unwrap(), intent: Intent::RelativeColorimetric, black_point: bpc }; + let mut out = rgba.to_vec(); + colour.convert(&mut out).unwrap(); + out + } + + /// Dark sRGB greys below the printer's black: without compensation they all clip to its + /// black; with it they stay apart and in order (test the shadows, not only mid grey). + #[test] + fn black_point_compensation_keeps_dark_shadows_apart() { + let profile = lifted_black_grey(); + let darks = [20u8, 40, 60].iter().flat_map(|&v| [v, v, v, 255]).collect::>(); + let clipped = convert(&profile, false, &darks); + assert_eq!((clipped[0], clipped[4], clipped[8]), (0, 0, 0), "{clipped:?}"); + let kept = convert(&profile, true, &darks); + assert!(kept[0] < kept[4] && kept[4] < kept[8], "{kept:?}"); + assert!(kept.as_chunks::<4>().0.iter().all(|&[r, g, b, a]| r == g && g == b && a == 255), "grey written to R, G and B: {kept:?}"); + } + + /// An RGB printer profile changes the values (sRGB's green is not the profile's green) and + /// keeps alpha. + #[test] + fn rgb_profiles_convert_the_values_and_keep_alpha() { + let wide = Builtin::AdobeRgbCompat.profile().to_bytes().to_vec(); + let out = convert(&wide, true, &[0, 255, 0, 128, 255, 255, 255, 255]); + assert!(out[0] > 100 && out[1] > 240 && out[2] > 20, "sRGB green inside a wider gamut: {out:?}"); + assert_eq!(out[3], 128); + assert!(out[4..7].iter().all(|&v| v >= 254), "white stays white: {out:?}"); + let same = convert(&Builtin::Srgb.profile().to_bytes(), true, &[10, 120, 230, 255]); + assert!(same.iter().zip([10u8, 120, 230, 255]).all(|(a, b)| a.abs_diff(b) <= 1), "sRGB into sRGB: {same:?}"); + } + + #[test] + fn cmyk_and_other_bytes_are_refused() { + let cmyk = Builtin::CoatedCmyk.profile().to_bytes(); + assert!(PrinterProfile::from_bytes(&cmyk).is_err_and(|e| e.contains("RGB or grey"))); + assert!(PrinterProfile::from_bytes(b"not a profile").is_err()); + assert!(PrinterProfile::from_bytes(&[]).is_err()); + } + + /// The listing finds RGB and grey profiles one folder deep, by description, and skips CMYK + /// profiles, other files and broken ones; a name finds one by description or file name. + #[test] + fn installed_profiles_are_rgb_and_grey_ones() { + let dir = std::env::temp_dir().join(format!("pdfcraft-print-profiles-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("vendor").join("deeper")).unwrap(); + std::fs::write(dir.join("office.ICC"), Builtin::Srgb.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("vendor").join("matte.icm"), lifted_black_grey()).unwrap(); + std::fs::write(dir.join("vendor").join("deeper").join("hidden.icc"), Builtin::Srgb.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("press.icc"), Builtin::CoatedCmyk.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("broken.icc"), b"acsp?").unwrap(); + std::fs::write(dir.join("notes.txt"), b"hello").unwrap(); + let found = installed_profiles_in(std::slice::from_ref(&dir)); + let names: Vec<(&str, bool)> = found.iter().map(|p| (p.name.as_str(), p.gray)).collect(); + assert_eq!(names.len(), 2, "{found:?}"); + assert!(names.contains(&("Lifted black grey", true)), "{names:?}"); + assert!(found.iter().any(|p| !p.gray && p.path.ends_with("office.ICC")), "{found:?}"); + assert!(find_installed(&found, "lifted BLACK grey").is_some_and(|p| p.gray)); + assert!(find_installed(&found, "matte").is_some_and(|p| p.gray), "by file stem"); + assert!(find_installed(&found, "office.icc").is_some_and(|p| !p.gray), "by file name"); + assert!(find_installed(&found, "press").is_none()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/print/src/lib.rs b/crates/print/src/lib.rs index 5707a42012..815a8351d3 100644 --- a/crates/print/src/lib.rs +++ b/crates/print/src/lib.rs @@ -20,6 +20,7 @@ use pdfcraft_content::Matrix; use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; +pub mod colour; mod font_names; pub mod range; pub mod spool; diff --git a/crates/print/src/settings.rs b/crates/print/src/settings.rs index 50657b328a..49d90c0308 100644 --- a/crates/print/src/settings.rs +++ b/crates/print/src/settings.rs @@ -186,17 +186,20 @@ pub struct Settings { } /// How pages are rendered when printing as image. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Raster { /// Dots per inch, 72–600 (Acrobat offers 150, 300 and 600). pub dpi: f64, /// Render the pictures in grey, so the print-ready PDF itself is grayscale. pub grayscale: bool, + /// App-managed colour: the pictures are converted into the printer's profile. `None` leaves + /// colour to the printer. + pub colour: Option, } impl Default for Raster { fn default() -> Self { - Raster { dpi: 300.0, grayscale: false } + Raster { dpi: 300.0, grayscale: false, colour: None } } } @@ -204,7 +207,7 @@ impl Default for Raster { /// Not when the pages go as grey pictures (`raster` with [`Raster::grayscale`]): they are grey /// already, so a driver that keeps no grayscale setting (Microsoft Print to PDF, many inkjets) /// still prints them instead of the job being refused (#5806). -pub fn driver_needs_grayscale(grayscale: bool, raster: Option) -> bool { +pub fn driver_needs_grayscale(grayscale: bool, raster: Option<&Raster>) -> bool { grayscale && !raster.is_some_and(|r| r.grayscale) } diff --git a/crates/ui-egui/src/workflows/print_ui/draft.rs b/crates/ui-egui/src/workflows/print_ui/draft.rs index 2b661426a6..02dbb97856 100644 --- a/crates/ui-egui/src/workflows/print_ui/draft.rs +++ b/crates/ui-egui/src/workflows/print_ui/draft.rs @@ -441,7 +441,7 @@ impl PrintDraft { m => m, }); let paper = self.paper_size(); - let raster = self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale }); + let raster = self.raster(); // A selected area prints alone, its one page sized like a page of its own: Size is its // only handling (the dialog offers no other while it is chosen). if self.which == Which::Area { @@ -509,7 +509,7 @@ impl PrintDraft { /// Print as image: pictures at the chosen resolution, grey when grayscale is asked for. fn raster(&self) -> Option { - self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale }) + self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale, colour: None }) } /// Whether multiple pages per sheet should be cut and stacked into sorted piles. @@ -556,7 +556,7 @@ impl PrintDraft { self.chosen_duplex() }, scaling: spool::Scaling::Exact, - grayscale: print::driver_needs_grayscale(self.grayscale, self.raster()), + grayscale: print::driver_needs_grayscale(self.grayscale, self.raster().as_ref()), // Set when the copy sent is made grey (`spooler_copy`, #5806). grey_pages: false, title: title.to_string(), diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0cdbc3e856..c721ce18e9 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -8641,6 +8641,18 @@ tools = [] evidence = [] notes = "" +[[feature]] +id = "print.color-handling" +title = "Print colour handling (app-managed colours, printer profile)" +area = "L" +tier = "P1" +milestone = "M11" +status = "partial" +commands = [] +tools = ["doc_print", "print_color_profiles"] +evidence = ["crates/print/src/colour.rs::black_point_compensation_keeps_dark_shadows_apart", "crates/print/src/colour.rs::rgb_profiles_convert_the_values_and_keep_alpha", "crates/print/src/colour.rs::installed_profiles_are_rgb_and_grey_ones", "crates/engine/src/tests.rs::app_managed_colour_converts_the_printed_pictures", "crates/automation/tests/it/automation.rs::doc_print_with_app_managed_colour"] +notes = "doc_print color_management app (#6269): the pages print as images (dpi, grayscale apply), converted from sRGB into an RGB or grey ICC printer profile (a file, or a name from print_color_profiles: the system's colour folders on Windows, macOS and Linux) with a rendering intent and black point compensation (default on), through craft-cms; the default leaves colour to the printer as before. The driver's own colour management should be off for such a job. Still missing against Acrobat: the Print dialog's Color Handling choice, CMYK printer profiles, vector pages converted without rendering them, and the document's output intent as the source of DeviceCMYK colours." + [[feature]] id = "print.advanced-print-separations" title = "Print separations and in-RIP separations" From 72c607fc47a60fb44828be5db0f931a9800a1549 Mon Sep 17 00:00:00 2001 From: Ken Mantle Date: Thu, 12 Nov 2026 11:57:26 -0500 Subject: [PATCH 44/46] fix(render): an RGB colour in an isolated DeviceCMYK group blends in CMYK, as its ink (#7031) * fix(render): an RGB colour in an isolated DeviceCMYK group blends in CMYK, as its ink Part of #4887. An isolated DeviceCMYK group blended in CMYK only when every colour in it had process tints (DeviceCMYK, DeviceGray); one RGB fill sent the whole group back to RGB blending. Vendored hayro-interpret patch (104): Color::ink_tints gives DeviceRGB, CalRGB, Lab and three-component ICCBased colours the ink that draws back to them through the bundled CMYK profile (new cmyk_inverse module: black 0.4*k0^2 + 0.6*k0^8 with k0 = 1 - max(r, g, b), matched to Acrobat on synthetic pages; then cyan, magenta and yellow by at most 12 Newton steps against the profile; one-entry per-thread cache). ICCBased CMYK still has no tints. Test an_rgb_colour_in_an_isolated_cmyk_group_blends_as_its_ink: 50% grey under Difference with black within 10 levels of Acrobat's 137 112 106; in-gamut colours draw as themselves; red within 8 of Acrobat's 238 32 37. With the additive arms reverted the grey draws 93 97 96 and it fails. #5228's control now uses an RGB image, which still has no tints. Co-Authored-By: Claude Opus 5.5 * fix(render): an RGB colour with a non-finite component has no CMYK ink `to_rgba` turns a non-finite component into some finite colour, so the inverse could not see it and gave that colour's ink. `ink_from_srgb` now declines such a colour and the group draws it as before. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: KenM76 Co-authored-by: Claude Opus 5.5 --- .../src/raster/tests/isolated_cmyk_groups.rs | 49 +++++++- vendor/README.md | 2 +- vendor/hayro-interpret/src/cmyk_inverse.rs | 118 ++++++++++++++++++ vendor/hayro-interpret/src/color.rs | 31 ++++- vendor/hayro-interpret/src/lib.rs | 1 + vendor/hayro/src/renderer.rs | 4 +- 6 files changed, 191 insertions(+), 14 deletions(-) create mode 100644 vendor/hayro-interpret/src/cmyk_inverse.rs diff --git a/crates/render/src/raster/tests/isolated_cmyk_groups.rs b/crates/render/src/raster/tests/isolated_cmyk_groups.rs index aa15019277..65b44ec93e 100644 --- a/crates/render/src/raster/tests/isolated_cmyk_groups.rs +++ b/crates/render/src/raster/tests/isolated_cmyk_groups.rs @@ -88,11 +88,12 @@ endstream endobj assert!(distance(&group, &square) <= 3, "{mode}: group {group:?}, square {square:?}"); } - // A group whose contents have no ink tints (here the second element is RGB) is - // composited in RGB exactly as before: the same as a group blending in DeviceRGB. - let rgb = |_: [f32; 4]| "1 0 0 rg".to_string(); - let cmyk_group = render("/CS /DeviceCMYK /I true", &rgb); - let rgb_group = render("/CS /DeviceRGB /I true", &rgb); + // A group whose contents have no ink tints (here the second element draws an RGB image + // over the page) is composited in RGB exactly as before: the same as a group blending + // in DeviceRGB. + let image = |_: [f32; 4]| "q 370 0 0 60 0 0 cm BI /W 1 /H 1 /CS /DeviceRGB /BPC 8 /F /AHx ID ff0000> EI Q".to_string(); + let cmyk_group = render("/CS /DeviceCMYK /I true", &image); + let rgb_group = render("/CS /DeviceRGB /I true", &image); assert_eq!(cmyk_group.rgba, rgb_group.rgba, "contents without tints are drawn as before"); } @@ -247,6 +248,32 @@ trailer << /Root 1 0 R >> p.rgba[((y * p.width + x) * 4) as usize..][..3].to_vec() } +/// An RGB colour painted in an isolated `DeviceCMYK` group is blended as the process ink that +/// draws back to it through the CMYK profile (hayro-interpret patch (104)), instead of +/// sending the whole group back to RGB blending. +/// +/// Under `/BM /Difference` with black ink, a colour's cyan, magenta and yellow are +/// complemented and its black kept (ISO 32000-2 §11.3.4), so 50% grey, mostly cyan, magenta +/// and yellow ink, comes out a warm grey: neither the neutral grey of RGB blending nor the +/// near-black of a grey made from black ink alone. Acrobat draws the same blend on a CMYK page +/// as (137, 112, 106). Painted without a blend, a colour inside the profile's gamut draws as +/// itself, and pure red is clipped to the profile's red, as in Acrobat. +#[test] +fn an_rgb_colour_in_an_isolated_cmyk_group_blends_as_its_ink() { + let resources = "/ExtGState << /D << /BM /Difference >> >>"; + let pixel = |group: &str| cmyk_group_pixel([100, 100], "/G Do", [20, 20, 80, 80], group, resources, (50, 50)); + let near = |a: &[u8], b: [u8; 3], within: u8| a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= within); + + let blended = pixel("0.5 0.5 0.5 rg 20 20 60 60 re f /D gs 0 0 0 1 k 20 20 60 60 re f"); + assert!(near(&blended, [137, 112, 106], 10), "grey under Difference with black: {blended:?}"); + + // (colour, expected pixel, how far from it) + for (rgb, expected, within) in [("0.8 0.4 0.2", [204, 102, 51], 3), ("0.3 0.7 0.4", [77, 179, 102], 3), ("1 0 0", [238, 32, 37], 8)] { + let drawn = pixel(&format!("{rgb} rg 20 20 60 60 re f")); + assert!(near(&drawn, expected, within), "{rgb} drew as {drawn:?}, expected {expected:?}"); + } +} + /// From the review of #5228: a shading pattern painted before the group left its pattern /// transform on the context, and the finished group was drawn through it, so the cyan half /// of the group showed magenta. The group must look the same whether or not a pattern was @@ -325,3 +352,15 @@ trailer << /Root 1 0 R >> assert!(rgb[1] > 120 && rgb[0] < 60 && rgb[2] < 120, "blended in CMYK, green: {rgb:?}"); } } + +/// An RGB colour with a non-finite component has no tints: converting it to RGB first would +/// give some finite colour that is not its ink. +#[test] +fn an_rgb_colour_with_a_non_finite_component_has_no_tints() { + use hayro::hayro_interpret::color::AlphaColor; + use hayro::hayro_interpret::color::Color; + let rgb = |c: [f32; 3]| Color::from_rgba(AlphaColor::new([c[0], c[1], c[2], 1.0])).ink_tints(); + assert!(rgb([0.5, 0.5, 0.5]).is_some()); + assert_eq!(rgb([f32::NAN, 0.5, 0.5]), None); + assert_eq!(rgb([0.5, f32::INFINITY, 0.5]), None); +} diff --git a/vendor/README.md b/vendor/README.md index 558dbbbc12..cfe8c512ff 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -55,7 +55,7 @@ Other resource categories retain their existing parent-chain lookup. | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| -| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (61) stroke geometry by ISO 32000-2 §8.5.3.2 and §8.4.3: a degenerate subpath (a single-point closed path, or points that all coincide) is stroked as a dot of the line width with round caps and not at all with butt or projecting square caps (upstream nudged a zero-length `l` so square caps drew a square, and `x y m h` lost its `h` because kurbo's `is_empty` counts only segments); a segment after `h` begins a new subpath with a `MoveTo` to the point `h` reached (§8.5.2.1, Table 58), so `h` then an `l` back to that point is a degenerate subpath of its own; a dash array that is all zero or has a negative or non-finite entry is kept as `[]`, a solid line (`[0 0]` became hundredth-of-a-point dashes, a faint half-covered line); a negative or non-finite line width (`w`, `/LW`) is kept as 0, so no device sees one below zero; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (65) a fill or clip whose path has no width or no height (but not both) marks the device pixels the line touches (ISO 32000-2 §10.7.4: "A zero-width or zero-height rectangle paints a line 1 pixel wide", and a clip is "the set of pixels that would be included by a fill operation"): whole pixels when the line stays within one device row or column, a band one device pixel wide when a rotated or skewed CTM carries it across rows and columns, built per subpath so the gap between two subpaths on one line stays unpainted (`degenerate_fill_area`). A zero-height clip such as `10 10 100 0 re W n` hid everything painted after it, and the fill fell back to a thinnest-line stroke that half-covered two rows on a pixel boundary. A point paints as before, and a path with area, however thin, is filled as the shape it is (the thinnest-line fallback took a side under 1/256 unit as zero, so a thin rectangle that the CTM scales up to whole pixels was drawn as a hairline across its bounding box); (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; so is an image whose JBIG2 soft mask or `/Mask` one of the decoder's limits refuses, or PdfCraft's bound on the pixels of an image (`image_size_ok`), where a mask the file damaged still leaves its image whole (#6722), and an image drawn on its own, a stencil mask included, that a limit refuses is still a failed image (`ImageXObject::masks_picture`): `decode_raster` leaves the image out when its mask reported `ContentTruncated`, decoded or kept by the cache (91), and a size or count the file damaged can read as past a limit and is taken for one, the safe side; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888); (76) a CID of an Arial the PDF doesn't embed (`CIDFontType2`, not of an Adobe character collection; `ArialMT`, `Arial-BoldMT`, `Arial,Bold` and the other two faces, not Arial Narrow, Black or Unicode MS) that no Unicode route draws names, through its `/CIDToGIDMap`, the Arial glyph of the standard Macintosh order (read-fonts' `post::DEFAULT_GLYPH_NAMES`, its first 258 glyphs), and a CFF substitute draws its glyph of that name (.notdef when it has none) instead of the CID taken as its own glyph index (#5889); (77) a standard 14 font drawn with its bundled stand-in (`StandardFontBlob::Cff`) reads each glyph's own advance from its charstring (`CffFontBlob::advance_width`), so a stand-in glyph wider or narrower than the standard metrics is fitted to them as the OpenType stand-ins already were (ISO 32000-2 §9.6.2.2): with no advance the outline was drawn as it is, and the stand-ins' € (WinAnsiEncoding code 128, Annex D Table D.2), 667 to 722 units wide against the metrics' 556 (Helvetica) and 500 (Times), overlapped the next glyph. Symbol and ZapfDingbats (Annex D.5, D.6) are fitted the same way, the check mark `4` a check box draws for its `/MK /CA` caption among them; (78) `TilingPattern::instance` numbers each setting of a tiling pattern (`scn`/`SCN`), its copies keeping the number, and `TilingPattern::is_coloured` reports `/PaintType 1`, so a renderer can draw a coloured cell once for every fill of one setting (hayro patch (30)); (79) text extraction from a symbolic embedded TrueType font with no `/Encoding` (and no `/ToUnicode`) reads a code through the `post` table name of the glyph its `cmap` selects (`EmbeddedKind::symbolic_text`), when that name has a Unicode value by the glyph-name rules of `glyph_name_text`, which are the Adobe Glyph List Specification's (ISO 32000-2 §9.10.2) and the editor's in pdfcraft-fonts (a `.suffix` is dropped, `_` separates the characters of a ligature, `uni` with one or more groups of exactly four uppercase hex digits and `u` with four to six are code points; lowercase digits, other lengths and surrogate or out-of-range values name nothing, so `uni20ac` reads as WinAnsi), instead of reading the code as WinAnsi: FrameMaker files distilled with Distiller 3 number their codes in the Macintosh glyph order, 29 below ASCII, so `Solid` was extracted, edited and exported to Word as `6ROLG` (#3962). Codes with no glyph, or whose glyph has no such name, still read as WinAnsi; glyph selection is unchanged; (80) an `/Indexed` colour space that breaks ISO 32000-2 §8.6.6.3 is drawn as most readers draw it: a lookup table shorter than m × (hival + 1) bytes keeps the entries it holds in full and an index past them is drawn black after conversion, in any base space (zero bytes would be white in DeviceCMYK, Separation and DeviceN, and blue in Lab); a `hival` past 255 is taken as 255, and a real `hival` is truncated toward zero. An image drawn so reports `InterpreterWarning::MalformedIndexedImage`, surfaced as `RenderWarning::MalformedIndexedImage`, and a fill or stroke colour in such a space reports `ColorFallback`. An image whose lookup is missing, not a string or a stream, or a stream that does not decode (or decodes to nothing although it holds data) is not drawn, with the same warning; a colour set in such a space keeps the upstream DeviceGray fallback. Upstream treated all of these as an unreadable space and fell back to DeviceGray, painting the indices near-black, even ones whose entries were present. A `hival` below zero or not a number has no reading: the space falls back as upstream, and an image in it reports the warning. A lookup string given by an indirect reference is read (upstream consumed the reference trying it as a stream and found no table). The warning is taken from the space as built, so no table is read twice: inline images naming the space through a resource read its table once however many there are (`InterpreterCache::indexed_lookup_reads`); (82) a soft mask (or `/Mask` image) larger than its image and more than twice as large on a side as the device pixels its image is drawn at is averaged down to that size, rounded up to a power of two, in `ImageXObject::decoded_soft_mask`, and cached there (`DecodedImageKey::shrunk_to`, with the image's raster keyed by the same rounded size); every size is made from one cached intermediate mask at most 4096 on a side, so the full mask is decoded once however many sizes it is drawn at: it bypassed the decoded-image cache and was decoded, copied and staged at full size each time it was drawn. A 2 × 2 image with a 34,862 × 4,332 mask drawn 40 times on one slide (pdf.js `issue16263.pdf`) took about 3.6 s a draw and minutes a page; it now decodes the mask once and draws the page in about 2.5 s. A mask no larger than twice its drawn size keeps its resolution, which the renderer filters as before; `InterpreterCache::decoded_mask_decodes` counts the masks among the decodes; (83) an embedded TrueType program whose `head.indexToLocFormat` holds neither 0 nor 1 is read with the `loca` format its table's length gives (`with_loca_format_fixed`): the glyph reader took anything but 1 as short offsets, so a font marked 256 had its long offsets misread and drew boxes (pdf.js `issue2537r.pdf`); (84) a Type 0 font of an Adobe character collection (Japan1, GB1, CNS1, Korea1) reads a code its `/ToUnicode` has nothing for, or every code when it has none, through the collection's UCS2 CMap, code to CID to Unicode, embedded or not (ISO 32000-2 §9.10.2; `Type0Font::ucs2`, parsed the first time a code needs it): only fonts that are not embedded took that route, so Japanese and Chinese pages whose fonts are embedded without a `/ToUnicode`, or with one that maps nothing, had no text to select, find or copy; (85) a Type 0 font's `/ToUnicode` written as a CID CMap (`begincidchar` / `begincidrange`, destinations as numbers, as Word writes it) gives each number as the code's Unicode value for text, as glyph lookup (`map_via_unicode`) already read it and as pdftotext and PDFium extract it; ISO 32000-2 §9.10.3 has `beginbfchar` / `beginbfrange` there, and such text extracted nothing (pdf.js `issue9367`); (86) form fields' widgets are drawn after the page's other annotations, each group in `/Annots` order (`interpret_page`), as MuPDF and PDFium draw them; ISO 32000-2 doesn't order the two: a check box listed before a filled square comment that covers it was hidden under the comment (pdf.js `issue13003.pdf`); (87) a Type 3 font keeps each glyph procedure it decoded (`Type3::decoded_proc`), so a glyph shown again isn't inflated again: a font used as a fill pattern drew one glyph about 650,000 times on a page and decoded its procedure for each; a procedure over 64 KiB isn't kept, and the fonts of one `InterpreterCache` keep at most 32 MiB together (`MAX_CACHED_TYPE3_PROCS`), each kept procedure counting its bytes, its name and 64 bytes besides, as fonts live as long as the cache; a procedure holds its bytes within the page's content budget while the glyph is painted, kept or not (`context::hold_content`), so a page skips the same glyphs once the budget is spent, and one decoded with no more than 64 KiB of budget left, which the budget may have cut short, isn't kept; `InterpreterCache::type3_proc_decodes` counts the procedures decoded; (88) the rule of (76) also reads Times New Roman (`TimesNewRomanPSMT`, `TimesNewRomanPS-BoldMT`, `TimesNewRoman,Bold` and the other faces, not Times or Times New Roman MT Std), whose first 258 glyphs follow that order too: in a public form whose embedded Times New Roman subsets keep their glyph indexes, 941 of 942 `/ToUnicode` entries below 258 name that order's character, and its non-embedded `TimesNewRoman,Bold` drew "Salarié(e) du régime général" as "Salari²*g+f w't²i lo g'ì²p²tcn" (#6449); (89) a form XObject already being drawn is not drawn inside itself (`Context::enter_form`, reported as `ContentTruncated`, §8.10.1): a name in its content that reached it again, through its own resources or the page resources a form with its own still falls back on, drew it until the page's nested-paint budget was spent (PDFBOX-5782, 38 s); the forms being drawn are kept per page and thread, so the tiling patterns, Type 3 glyphs, soft masks and groups painted inside a form, which are interpreted in contexts of their own, see it too; (90) a Type 3 font without `/Widths` (or `/FirstChar`/`/LastChar`) advances each glyph by the width its glyph procedure sets with `d0` or `d1` (`Type3::proc_width`, ISO 32000-2 9.6.4) instead of 0, read once per code; a font with `/Widths` is unchanged, a code outside `/FirstChar`..`/LastChar` keeping width 0 (Table 110) (PDFBOX-2943: a misspelt `/Widths` key drew every glyph on one spot), and selecting such a font reports `InterpreterWarning::Type3WidthsMissing` (`Font::type3_widths_missing`, checked in `Context::resolve_font`); (91) the decoded-image cache keeps a failed decode for its key, as the warnings it reported, so a picture or stencil mask that doesn't decode (damaged data, a JPX or JBIG2 stream its decoder refuses) isn't decoded again at every draw, tile and render that shares the cache, and each draw still reports what the decode did (`DecodedImageCache::get_or_insert_with`, `ImageXObject::reporting_to`); a failure counts 128 bytes towards the cache's budget, and one the page's JBIG2 allowance may have caused (73) isn't kept (the hayro-syntax comment on `charge_page_reading` says so), nor one a JBIG2 mask's decoding work refused, while a JBIG2 mask refused by a limit its stream reaches by itself is kept, with the image it takes out (`note_stream_limited`), so a page drawing that image a hundred times decodes it once; (92) the text of a glyph whose name is not in the glyph list reads `uniXXXX` and `uXXXX` to `uXXXXXX` by the Adobe Glyph List Specification (`agl_unicode_from_name`: `uni` and exactly four uppercase hex digits, `u` and four to six, no surrogate, nothing past U+10FFFF), so `uni65`, `u65`, `uni20ac` and `uni+065` read as nothing instead of `e` and `€`; upstream's lenient `unicode_from_name` stays for finding a glyph in a font (TrueType `cmap` lookup, accented standard-font letters), so drawing is unchanged; (93) a CID-keyed CFF font keeps the font dicts it can read: a font dict without a Private DICT (one a subsetter emptied) leaves only the glyphs FDSelect gives it missing, where one such dict dropped the whole font and its text drew with a fallback (ISO 32000-2 §9.9 Table 124, Adobe TN #5176); (94) a font dictionary whose `/Subtype` is missing or the empty name (`/Subtype /`) is read as Type 1, so its `/BaseFont` picks the face (ISO 32000-2 Table 109 requires `/Subtype`; other readers fall back this way): a `/Courier-Bold` was drawn in the Helvetica fallback (PDFBOX-1988). Any other unknown subtype is still refused; (98) a non-embedded font named `SymbolMT`, or `Symbol`/`SymbolMT` with a `,` or `-` style suffix (`Symbol,Bold`, `Symbol-Bold`; `SYMBOL_STYLES`, so `Symbol-Helvetica` stays another font), is the standard Symbol font and draws through its built-in encoding, so a TrueType `SymbolMT` declaring `/WinAnsiEncoding` draws `mu` for code 0x6D, not `m` (ISO 32000-2 §9.6.2.2); (99) the mesh sampler walks each row of a triangle only across the columns where that row's sample points can lie in it, not its whole box (a row with none still costs a visit): thin triangles of the fixed 19 × 19 patch grid lying across the pixel grid (a narrow curved band of tensor patches) have boxes many times their area, and a mesh painted through a path the size of its own box spent the visit budget before its last patches and left them unpainted; (100) a Type 1 or Type 3 font glyph name that only numbers its glyph (`C39`, `MT72`, `G4E`: one to three letters, then decimal or hexadecimal digits) and names the very code it is shown with reads as that code's Latin-1 character (controls excepted; in a Type 3 font, which already reads codes 32-126 by the code, above 126), in a font where every such name names its own code (`numbered_names_name_their_codes`, worked out once per font); a font with any numbered name at another code (`/C1` at 233, review on #3239; `/Differences [1 /C70 …]`) or at a C0 control code (a subset counting its glyphs from code 1, `/Differences [1 /C1 /C2 …]`, where `/C65` is the 65th glyph used) keeps reading them as nothing (#3170); (101) a symbolic TrueType font with no encoding reads a code as Mac OS Roman in text extraction when its Macintosh Roman (1, 0) `cmap` subtable selects the glyph the code draws (ISO 32000-2 §9.6.5.4) and that glyph sits where the standard Macintosh glyph order (`post` version 1.0) puts the code's Mac OS Roman character, for a font whose `post` table names no glyphs, that names no base encoding (an unknown name included) and that isn't Symbol or ZapfDingbats (§9.6.6.1), after `/Differences` and before the WinAnsi last resort; Windows subsets in the same order keep their WinAnsi reading, and glyph selection is unchanged; (102) `OutlineGlyph::vertical_glyph_width`: a vertical-writing Type 0 glyph's width w0 from `/W` or `/DW`, its extent across the column (ISO 32000-2 §9.7.4.3), which `advance_width` (the horizontal advance, 0 in vertical writing) does not give; (81) a reply of type R (a markup annotation with an `/IRT` and an `/RT` other than `/Group`; R is the default, so a missing, unknown or malformed `/RT` reads as R; a widget, link or pop-up with `/IRT` still draws) is not drawn, whatever its appearance or flags: ISO 32000-2 Table 172 says such replies "shall not" be displayed individually but in their comment's thread (files from editors that give every reply a note-icon appearance showed a stack of icons, #5631); a `/RT /Group` member still draws; (103) `InterpreterSettings::view_auto_state` (on by default): the default configuration's auto state for the View event (`/AS` usage applications with `/Event /View` and `/View` in `/Category`) sets each managed group from its `/Usage /View /ViewState` before `ocg_overrides`, ON only if every recommendation is ON (ISO 32000-2 §8.11.4.4); `/Zoom`, `/User` and `/Language` give no recommendation; at most 256 applications, 4096 `/OCGs` entries and 16 categories each are read; off for print renders | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (61) `stroke_geometry::{degenerate_subpaths_draw_a_round_dot_or_nothing, degenerate_subpaths_leave_the_rest_of_the_path_alone, a_segment_after_h_begins_a_new_subpath, all_zero_dash_arrays_draw_a_solid_line, invalid_dash_arrays_reach_a_device_as_solid, valid_dash_patterns_still_dash, negative_line_widths_reach_a_device_as_zero, negative_line_widths_draw_the_thinnest_line_in_the_stroke_colour}` (pdfcraft-render integration tests); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (65) `it::zero_area_fill_clip` (`crates/render/tests/it/zero_area_fill_clip.rs`); (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn`, `a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `a_jbig2_image_drawn_on_its_own_that_a_limit_refuses_is_a_failed_image`, `a_limit_refusal_says_nothing_of_the_next_image`, `a_jpx_picture_whose_jbig2_mask_a_limit_refuses_stays_out_at_every_size` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render); (76) `raster::tests::non_embedded_arial::{an_arial_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (77) `euro_width::the_euro_fits_its_standard_advance`, `euro_width::two_euros_stay_apart`, `euro_width::a_slanted_euro_fits_its_standard_advance`, `symbol_dingbats_width::{symbol_glyphs_keep_their_standard_advances, zapf_dingbats_glyphs_keep_their_standard_advances}` (pdfcraft-render `tests/it`), `dingbat_check_box::a_check_box_caption_check_is_drawn_at_its_standard_advance` (pdfcraft-engine `tests/it`); (78) `raster::tests::{fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}` (pdfcraft-render); (79) `raster::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names`, `pdf::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names` (pdfcraft-fonts, the editor's reading by the same rules) ; (80) `crates/render/tests/it/indexed_lookup.rs` (`cargo test -p pdfcraft-render --test it indexed_lookup`), `raster::tests::indexed_lookup_cache`; (82) `raster::tests::a_soft_mask_far_larger_than_where_it_is_drawn_is_shrunk_once`, `raster::tests::an_averaged_down_soft_mask_follows_its_samples_and_its_drawn_size`, `raster::tests::decoded_images_serve_every_size_they_are_drawn_at` (pdfcraft-render); (83) `raster::tests::truetype_loca_format::a_truetype_font_with_an_out_of_range_loca_format_draws_its_glyphs` (pdfcraft-render); (84) `text::tests::an_embedded_adobe_collection_font_reads_through_its_ucs2_cmap`, `every_adobe_collection_reads_through_its_own_ucs2_cmap` (GB1, CNS1, Korea1, Identity-V, `/H` over CFF) and `a_font_outside_the_adobe_collections_gets_no_text_from_a_ucs2_cmap` (pdfcraft-render); (85) `text::tests::{a_tounicode_written_as_cid_ranges_gives_text, a_tounicode_bf_entry_wins_over_a_cid_entry_for_the_same_code}` (pdfcraft-render); (86) `widgets_above_annotations::a_widget_is_drawn_above_the_comments_on_its_page` (pdfcraft-render `tests/it`); (87) `raster::tests::type3_proc_cache::{a_type3_glyph_shown_again_decodes_its_procedure_once, a_procedure_over_64_kib_is_decoded_for_each_copy, the_procedures_one_cache_keeps_stop_at_32_mib, fonts_with_the_same_glyph_name_keep_their_own_procedures}` (pdfcraft-render); (88) `raster::tests::non_embedded_arial::{a_times_new_roman_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (89) `raster::tests::a_form_that_draws_itself_is_drawn_once`, `raster::tests::a_form_that_reaches_itself_through_a_pattern_or_a_glyph_is_drawn_once` (pdfcraft-render); (90) `raster::tests::type3_proc_widths::{a_type3_font_without_widths_advances_by_its_glyph_procedures, a_code_outside_first_char_to_last_char_still_has_width_zero, widths_the_font_gives_are_kept, a_code_without_a_usable_procedure_width_still_advances_nothing, a_type3_font_without_widths_is_reported}` (pdfcraft-render); (91) `raster::tests::failed_image_decodes::{a_picture_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_that_does_not_decode_is_decoded_once_at_each_size, a_stencil_mask_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_whose_soft_mask_does_not_decode_draws_unmasked_decoded_once, a_picture_that_decodes_is_still_decoded_once_and_drawn_every_time, a_picture_left_out_for_the_pages_allowance_draws_on_the_next_page, a_picture_whose_jbig2_mask_a_limit_refuses_is_decoded_once_however_often_it_is_drawn, a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw}` (pdfcraft-render); (92) `text::tests::agl_unicode_from_name_rule_reads_text` (pdfcraft-render); (93) `cid_cff_unusable_font_dict::{glyphs_of_a_usable_font_dict_draw_beside_an_unusable_one, a_program_with_no_usable_font_dict_draws_no_square}` (pdfcraft-render integration tests); (94) `raster::tests::a_font_without_a_subtype_is_drawn_by_its_base_font` (pdfcraft-render); (98) `symbol_mt_builtin_encoding::{symbol_mt_draws_symbols_through_the_built_in_encoding, a_name_that_only_starts_with_symbol_is_not_symbol, symbol_mt_applies_its_differences, an_embedded_symbol_mt_draws_its_own_program}` (pdfcraft-render integration tests); (99) `raster::tests::thin_patches_of_a_mesh_in_a_tight_path_are_all_painted` (pdfcraft-render); (100) `text::tests::numbered_glyph_names_read_as_their_own_code`, `text::tests::numbered_type3_glyph_names_read_as_their_own_code`, `text::tests::glyph_names_outside_the_glyph_list_still_read_as_text` (pdfcraft-render); (101) `raster::tests::symbolic_truetype_mac_roman_text::{a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman, winansi_codes_of_a_font_in_the_macintosh_order_read_as_before, a_symbolic_truetype_font_without_a_1_0_subtable_reads_as_before, a_1_0_subtable_selecting_other_glyphs_than_the_drawn_ones_is_not_read, a_font_that_names_its_glyphs_is_not_read_by_their_order, a_symbol_or_zapfdingbats_font_in_the_macintosh_order_gets_no_mac_roman_reading, differences_naming_notdef_clear_the_mac_roman_reading_for_its_code, a_base_encoding_name_this_reader_does_not_know_keeps_the_mac_roman_reading_off}` (pdfcraft-render), `pdf::tests::a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman` (pdfcraft-fonts); (102) `text::tests::tracked_vertical_latin_stays_one_line` (pdfcraft-render); (81) `raster::tests::replies_are_not_drawn_but_their_comment_and_group_members_are`, `raster::tests::an_unknown_or_malformed_rt_is_a_reply_and_a_widget_with_irt_still_draws` (hayro-interpret); (103) `raster::tests::the_view_auto_state_sets_a_groups_state_on_screen`, `raster::tests::layer_toggles_and_print_renders_are_not_the_view_auto_state` and `raster::tests::malformed_view_auto_state_is_ignored` (pdfcraft-render) | +| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (61) stroke geometry by ISO 32000-2 §8.5.3.2 and §8.4.3: a degenerate subpath (a single-point closed path, or points that all coincide) is stroked as a dot of the line width with round caps and not at all with butt or projecting square caps (upstream nudged a zero-length `l` so square caps drew a square, and `x y m h` lost its `h` because kurbo's `is_empty` counts only segments); a segment after `h` begins a new subpath with a `MoveTo` to the point `h` reached (§8.5.2.1, Table 58), so `h` then an `l` back to that point is a degenerate subpath of its own; a dash array that is all zero or has a negative or non-finite entry is kept as `[]`, a solid line (`[0 0]` became hundredth-of-a-point dashes, a faint half-covered line); a negative or non-finite line width (`w`, `/LW`) is kept as 0, so no device sees one below zero; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (65) a fill or clip whose path has no width or no height (but not both) marks the device pixels the line touches (ISO 32000-2 §10.7.4: "A zero-width or zero-height rectangle paints a line 1 pixel wide", and a clip is "the set of pixels that would be included by a fill operation"): whole pixels when the line stays within one device row or column, a band one device pixel wide when a rotated or skewed CTM carries it across rows and columns, built per subpath so the gap between two subpaths on one line stays unpainted (`degenerate_fill_area`). A zero-height clip such as `10 10 100 0 re W n` hid everything painted after it, and the fill fell back to a thinnest-line stroke that half-covered two rows on a pixel boundary. A point paints as before, and a path with area, however thin, is filled as the shape it is (the thinnest-line fallback took a side under 1/256 unit as zero, so a thin rectangle that the CTM scales up to whole pixels was drawn as a hairline across its bounding box); (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; so is an image whose JBIG2 soft mask or `/Mask` one of the decoder's limits refuses, or PdfCraft's bound on the pixels of an image (`image_size_ok`), where a mask the file damaged still leaves its image whole (#6722), and an image drawn on its own, a stencil mask included, that a limit refuses is still a failed image (`ImageXObject::masks_picture`): `decode_raster` leaves the image out when its mask reported `ContentTruncated`, decoded or kept by the cache (91), and a size or count the file damaged can read as past a limit and is taken for one, the safe side; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888); (76) a CID of an Arial the PDF doesn't embed (`CIDFontType2`, not of an Adobe character collection; `ArialMT`, `Arial-BoldMT`, `Arial,Bold` and the other two faces, not Arial Narrow, Black or Unicode MS) that no Unicode route draws names, through its `/CIDToGIDMap`, the Arial glyph of the standard Macintosh order (read-fonts' `post::DEFAULT_GLYPH_NAMES`, its first 258 glyphs), and a CFF substitute draws its glyph of that name (.notdef when it has none) instead of the CID taken as its own glyph index (#5889); (77) a standard 14 font drawn with its bundled stand-in (`StandardFontBlob::Cff`) reads each glyph's own advance from its charstring (`CffFontBlob::advance_width`), so a stand-in glyph wider or narrower than the standard metrics is fitted to them as the OpenType stand-ins already were (ISO 32000-2 §9.6.2.2): with no advance the outline was drawn as it is, and the stand-ins' € (WinAnsiEncoding code 128, Annex D Table D.2), 667 to 722 units wide against the metrics' 556 (Helvetica) and 500 (Times), overlapped the next glyph. Symbol and ZapfDingbats (Annex D.5, D.6) are fitted the same way, the check mark `4` a check box draws for its `/MK /CA` caption among them; (78) `TilingPattern::instance` numbers each setting of a tiling pattern (`scn`/`SCN`), its copies keeping the number, and `TilingPattern::is_coloured` reports `/PaintType 1`, so a renderer can draw a coloured cell once for every fill of one setting (hayro patch (30)); (79) text extraction from a symbolic embedded TrueType font with no `/Encoding` (and no `/ToUnicode`) reads a code through the `post` table name of the glyph its `cmap` selects (`EmbeddedKind::symbolic_text`), when that name has a Unicode value by the glyph-name rules of `glyph_name_text`, which are the Adobe Glyph List Specification's (ISO 32000-2 §9.10.2) and the editor's in pdfcraft-fonts (a `.suffix` is dropped, `_` separates the characters of a ligature, `uni` with one or more groups of exactly four uppercase hex digits and `u` with four to six are code points; lowercase digits, other lengths and surrogate or out-of-range values name nothing, so `uni20ac` reads as WinAnsi), instead of reading the code as WinAnsi: FrameMaker files distilled with Distiller 3 number their codes in the Macintosh glyph order, 29 below ASCII, so `Solid` was extracted, edited and exported to Word as `6ROLG` (#3962). Codes with no glyph, or whose glyph has no such name, still read as WinAnsi; glyph selection is unchanged; (80) an `/Indexed` colour space that breaks ISO 32000-2 §8.6.6.3 is drawn as most readers draw it: a lookup table shorter than m × (hival + 1) bytes keeps the entries it holds in full and an index past them is drawn black after conversion, in any base space (zero bytes would be white in DeviceCMYK, Separation and DeviceN, and blue in Lab); a `hival` past 255 is taken as 255, and a real `hival` is truncated toward zero. An image drawn so reports `InterpreterWarning::MalformedIndexedImage`, surfaced as `RenderWarning::MalformedIndexedImage`, and a fill or stroke colour in such a space reports `ColorFallback`. An image whose lookup is missing, not a string or a stream, or a stream that does not decode (or decodes to nothing although it holds data) is not drawn, with the same warning; a colour set in such a space keeps the upstream DeviceGray fallback. Upstream treated all of these as an unreadable space and fell back to DeviceGray, painting the indices near-black, even ones whose entries were present. A `hival` below zero or not a number has no reading: the space falls back as upstream, and an image in it reports the warning. A lookup string given by an indirect reference is read (upstream consumed the reference trying it as a stream and found no table). The warning is taken from the space as built, so no table is read twice: inline images naming the space through a resource read its table once however many there are (`InterpreterCache::indexed_lookup_reads`); (82) a soft mask (or `/Mask` image) larger than its image and more than twice as large on a side as the device pixels its image is drawn at is averaged down to that size, rounded up to a power of two, in `ImageXObject::decoded_soft_mask`, and cached there (`DecodedImageKey::shrunk_to`, with the image's raster keyed by the same rounded size); every size is made from one cached intermediate mask at most 4096 on a side, so the full mask is decoded once however many sizes it is drawn at: it bypassed the decoded-image cache and was decoded, copied and staged at full size each time it was drawn. A 2 × 2 image with a 34,862 × 4,332 mask drawn 40 times on one slide (pdf.js `issue16263.pdf`) took about 3.6 s a draw and minutes a page; it now decodes the mask once and draws the page in about 2.5 s. A mask no larger than twice its drawn size keeps its resolution, which the renderer filters as before; `InterpreterCache::decoded_mask_decodes` counts the masks among the decodes; (83) an embedded TrueType program whose `head.indexToLocFormat` holds neither 0 nor 1 is read with the `loca` format its table's length gives (`with_loca_format_fixed`): the glyph reader took anything but 1 as short offsets, so a font marked 256 had its long offsets misread and drew boxes (pdf.js `issue2537r.pdf`); (84) a Type 0 font of an Adobe character collection (Japan1, GB1, CNS1, Korea1) reads a code its `/ToUnicode` has nothing for, or every code when it has none, through the collection's UCS2 CMap, code to CID to Unicode, embedded or not (ISO 32000-2 §9.10.2; `Type0Font::ucs2`, parsed the first time a code needs it): only fonts that are not embedded took that route, so Japanese and Chinese pages whose fonts are embedded without a `/ToUnicode`, or with one that maps nothing, had no text to select, find or copy; (85) a Type 0 font's `/ToUnicode` written as a CID CMap (`begincidchar` / `begincidrange`, destinations as numbers, as Word writes it) gives each number as the code's Unicode value for text, as glyph lookup (`map_via_unicode`) already read it and as pdftotext and PDFium extract it; ISO 32000-2 §9.10.3 has `beginbfchar` / `beginbfrange` there, and such text extracted nothing (pdf.js `issue9367`); (86) form fields' widgets are drawn after the page's other annotations, each group in `/Annots` order (`interpret_page`), as MuPDF and PDFium draw them; ISO 32000-2 doesn't order the two: a check box listed before a filled square comment that covers it was hidden under the comment (pdf.js `issue13003.pdf`); (87) a Type 3 font keeps each glyph procedure it decoded (`Type3::decoded_proc`), so a glyph shown again isn't inflated again: a font used as a fill pattern drew one glyph about 650,000 times on a page and decoded its procedure for each; a procedure over 64 KiB isn't kept, and the fonts of one `InterpreterCache` keep at most 32 MiB together (`MAX_CACHED_TYPE3_PROCS`), each kept procedure counting its bytes, its name and 64 bytes besides, as fonts live as long as the cache; a procedure holds its bytes within the page's content budget while the glyph is painted, kept or not (`context::hold_content`), so a page skips the same glyphs once the budget is spent, and one decoded with no more than 64 KiB of budget left, which the budget may have cut short, isn't kept; `InterpreterCache::type3_proc_decodes` counts the procedures decoded; (88) the rule of (76) also reads Times New Roman (`TimesNewRomanPSMT`, `TimesNewRomanPS-BoldMT`, `TimesNewRoman,Bold` and the other faces, not Times or Times New Roman MT Std), whose first 258 glyphs follow that order too: in a public form whose embedded Times New Roman subsets keep their glyph indexes, 941 of 942 `/ToUnicode` entries below 258 name that order's character, and its non-embedded `TimesNewRoman,Bold` drew "Salarié(e) du régime général" as "Salari²*g+f w't²i lo g'ì²p²tcn" (#6449); (89) a form XObject already being drawn is not drawn inside itself (`Context::enter_form`, reported as `ContentTruncated`, §8.10.1): a name in its content that reached it again, through its own resources or the page resources a form with its own still falls back on, drew it until the page's nested-paint budget was spent (PDFBOX-5782, 38 s); the forms being drawn are kept per page and thread, so the tiling patterns, Type 3 glyphs, soft masks and groups painted inside a form, which are interpreted in contexts of their own, see it too; (90) a Type 3 font without `/Widths` (or `/FirstChar`/`/LastChar`) advances each glyph by the width its glyph procedure sets with `d0` or `d1` (`Type3::proc_width`, ISO 32000-2 9.6.4) instead of 0, read once per code; a font with `/Widths` is unchanged, a code outside `/FirstChar`..`/LastChar` keeping width 0 (Table 110) (PDFBOX-2943: a misspelt `/Widths` key drew every glyph on one spot), and selecting such a font reports `InterpreterWarning::Type3WidthsMissing` (`Font::type3_widths_missing`, checked in `Context::resolve_font`); (91) the decoded-image cache keeps a failed decode for its key, as the warnings it reported, so a picture or stencil mask that doesn't decode (damaged data, a JPX or JBIG2 stream its decoder refuses) isn't decoded again at every draw, tile and render that shares the cache, and each draw still reports what the decode did (`DecodedImageCache::get_or_insert_with`, `ImageXObject::reporting_to`); a failure counts 128 bytes towards the cache's budget, and one the page's JBIG2 allowance may have caused (73) isn't kept (the hayro-syntax comment on `charge_page_reading` says so), nor one a JBIG2 mask's decoding work refused, while a JBIG2 mask refused by a limit its stream reaches by itself is kept, with the image it takes out (`note_stream_limited`), so a page drawing that image a hundred times decodes it once; (92) the text of a glyph whose name is not in the glyph list reads `uniXXXX` and `uXXXX` to `uXXXXXX` by the Adobe Glyph List Specification (`agl_unicode_from_name`: `uni` and exactly four uppercase hex digits, `u` and four to six, no surrogate, nothing past U+10FFFF), so `uni65`, `u65`, `uni20ac` and `uni+065` read as nothing instead of `e` and `€`; upstream's lenient `unicode_from_name` stays for finding a glyph in a font (TrueType `cmap` lookup, accented standard-font letters), so drawing is unchanged; (93) a CID-keyed CFF font keeps the font dicts it can read: a font dict without a Private DICT (one a subsetter emptied) leaves only the glyphs FDSelect gives it missing, where one such dict dropped the whole font and its text drew with a fallback (ISO 32000-2 §9.9 Table 124, Adobe TN #5176); (94) a font dictionary whose `/Subtype` is missing or the empty name (`/Subtype /`) is read as Type 1, so its `/BaseFont` picks the face (ISO 32000-2 Table 109 requires `/Subtype`; other readers fall back this way): a `/Courier-Bold` was drawn in the Helvetica fallback (PDFBOX-1988). Any other unknown subtype is still refused; (98) a non-embedded font named `SymbolMT`, or `Symbol`/`SymbolMT` with a `,` or `-` style suffix (`Symbol,Bold`, `Symbol-Bold`; `SYMBOL_STYLES`, so `Symbol-Helvetica` stays another font), is the standard Symbol font and draws through its built-in encoding, so a TrueType `SymbolMT` declaring `/WinAnsiEncoding` draws `mu` for code 0x6D, not `m` (ISO 32000-2 §9.6.2.2); (99) the mesh sampler walks each row of a triangle only across the columns where that row's sample points can lie in it, not its whole box (a row with none still costs a visit): thin triangles of the fixed 19 × 19 patch grid lying across the pixel grid (a narrow curved band of tensor patches) have boxes many times their area, and a mesh painted through a path the size of its own box spent the visit budget before its last patches and left them unpainted; (100) a Type 1 or Type 3 font glyph name that only numbers its glyph (`C39`, `MT72`, `G4E`: one to three letters, then decimal or hexadecimal digits) and names the very code it is shown with reads as that code's Latin-1 character (controls excepted; in a Type 3 font, which already reads codes 32-126 by the code, above 126), in a font where every such name names its own code (`numbered_names_name_their_codes`, worked out once per font); a font with any numbered name at another code (`/C1` at 233, review on #3239; `/Differences [1 /C70 …]`) or at a C0 control code (a subset counting its glyphs from code 1, `/Differences [1 /C1 /C2 …]`, where `/C65` is the 65th glyph used) keeps reading them as nothing (#3170); (101) a symbolic TrueType font with no encoding reads a code as Mac OS Roman in text extraction when its Macintosh Roman (1, 0) `cmap` subtable selects the glyph the code draws (ISO 32000-2 §9.6.5.4) and that glyph sits where the standard Macintosh glyph order (`post` version 1.0) puts the code's Mac OS Roman character, for a font whose `post` table names no glyphs, that names no base encoding (an unknown name included) and that isn't Symbol or ZapfDingbats (§9.6.6.1), after `/Differences` and before the WinAnsi last resort; Windows subsets in the same order keep their WinAnsi reading, and glyph selection is unchanged; (102) `OutlineGlyph::vertical_glyph_width`: a vertical-writing Type 0 glyph's width w0 from `/W` or `/DW`, its extent across the column (ISO 32000-2 §9.7.4.3), which `advance_width` (the horizontal advance, 0 in vertical writing) does not give; (81) a reply of type R (a markup annotation with an `/IRT` and an `/RT` other than `/Group`; R is the default, so a missing, unknown or malformed `/RT` reads as R; a widget, link or pop-up with `/IRT` still draws) is not drawn, whatever its appearance or flags: ISO 32000-2 Table 172 says such replies "shall not" be displayed individually but in their comment's thread (files from editors that give every reply a note-icon appearance showed a stack of icons, #5631); a `/RT /Group` member still draws; (103) `InterpreterSettings::view_auto_state` (on by default): the default configuration's auto state for the View event (`/AS` usage applications with `/Event /View` and `/View` in `/Category`) sets each managed group from its `/Usage /View /ViewState` before `ocg_overrides`, ON only if every recommendation is ON (ISO 32000-2 §8.11.4.4); `/Zoom`, `/User` and `/Language` give no recommendation; at most 256 applications, 4096 `/OCGs` entries and 16 categories each are read; off for print renders; (104) an additive colour (`DeviceRGB`, `CalRGB`, `Lab`, three-component ICCBased) painted in an isolated `DeviceCMYK` group has ink tints, the ink that draws back to it through the CMYK profile (`cmyk_inverse::srgb_to_cmyk`: black `0.4 · k₀² + 0.6 · k₀⁸` with `k₀ = 1 − max(r, g, b)`, then cyan, magenta and yellow by at most 12 Newton steps; the last colour is cached per thread), so the group blends in CMYK instead of falling back to RGB | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (61) `stroke_geometry::{degenerate_subpaths_draw_a_round_dot_or_nothing, degenerate_subpaths_leave_the_rest_of_the_path_alone, a_segment_after_h_begins_a_new_subpath, all_zero_dash_arrays_draw_a_solid_line, invalid_dash_arrays_reach_a_device_as_solid, valid_dash_patterns_still_dash, negative_line_widths_reach_a_device_as_zero, negative_line_widths_draw_the_thinnest_line_in_the_stroke_colour}` (pdfcraft-render integration tests); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (65) `it::zero_area_fill_clip` (`crates/render/tests/it/zero_area_fill_clip.rs`); (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn`, `a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `a_jbig2_image_drawn_on_its_own_that_a_limit_refuses_is_a_failed_image`, `a_limit_refusal_says_nothing_of_the_next_image`, `a_jpx_picture_whose_jbig2_mask_a_limit_refuses_stays_out_at_every_size` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render); (76) `raster::tests::non_embedded_arial::{an_arial_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (77) `euro_width::the_euro_fits_its_standard_advance`, `euro_width::two_euros_stay_apart`, `euro_width::a_slanted_euro_fits_its_standard_advance`, `symbol_dingbats_width::{symbol_glyphs_keep_their_standard_advances, zapf_dingbats_glyphs_keep_their_standard_advances}` (pdfcraft-render `tests/it`), `dingbat_check_box::a_check_box_caption_check_is_drawn_at_its_standard_advance` (pdfcraft-engine `tests/it`); (78) `raster::tests::{fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}` (pdfcraft-render); (79) `raster::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names`, `pdf::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names` (pdfcraft-fonts, the editor's reading by the same rules) ; (80) `crates/render/tests/it/indexed_lookup.rs` (`cargo test -p pdfcraft-render --test it indexed_lookup`), `raster::tests::indexed_lookup_cache`; (82) `raster::tests::a_soft_mask_far_larger_than_where_it_is_drawn_is_shrunk_once`, `raster::tests::an_averaged_down_soft_mask_follows_its_samples_and_its_drawn_size`, `raster::tests::decoded_images_serve_every_size_they_are_drawn_at` (pdfcraft-render); (83) `raster::tests::truetype_loca_format::a_truetype_font_with_an_out_of_range_loca_format_draws_its_glyphs` (pdfcraft-render); (84) `text::tests::an_embedded_adobe_collection_font_reads_through_its_ucs2_cmap`, `every_adobe_collection_reads_through_its_own_ucs2_cmap` (GB1, CNS1, Korea1, Identity-V, `/H` over CFF) and `a_font_outside_the_adobe_collections_gets_no_text_from_a_ucs2_cmap` (pdfcraft-render); (85) `text::tests::{a_tounicode_written_as_cid_ranges_gives_text, a_tounicode_bf_entry_wins_over_a_cid_entry_for_the_same_code}` (pdfcraft-render); (86) `widgets_above_annotations::a_widget_is_drawn_above_the_comments_on_its_page` (pdfcraft-render `tests/it`); (87) `raster::tests::type3_proc_cache::{a_type3_glyph_shown_again_decodes_its_procedure_once, a_procedure_over_64_kib_is_decoded_for_each_copy, the_procedures_one_cache_keeps_stop_at_32_mib, fonts_with_the_same_glyph_name_keep_their_own_procedures}` (pdfcraft-render); (88) `raster::tests::non_embedded_arial::{a_times_new_roman_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (89) `raster::tests::a_form_that_draws_itself_is_drawn_once`, `raster::tests::a_form_that_reaches_itself_through_a_pattern_or_a_glyph_is_drawn_once` (pdfcraft-render); (90) `raster::tests::type3_proc_widths::{a_type3_font_without_widths_advances_by_its_glyph_procedures, a_code_outside_first_char_to_last_char_still_has_width_zero, widths_the_font_gives_are_kept, a_code_without_a_usable_procedure_width_still_advances_nothing, a_type3_font_without_widths_is_reported}` (pdfcraft-render); (91) `raster::tests::failed_image_decodes::{a_picture_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_that_does_not_decode_is_decoded_once_at_each_size, a_stencil_mask_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_whose_soft_mask_does_not_decode_draws_unmasked_decoded_once, a_picture_that_decodes_is_still_decoded_once_and_drawn_every_time, a_picture_left_out_for_the_pages_allowance_draws_on_the_next_page, a_picture_whose_jbig2_mask_a_limit_refuses_is_decoded_once_however_often_it_is_drawn, a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw}` (pdfcraft-render); (92) `text::tests::agl_unicode_from_name_rule_reads_text` (pdfcraft-render); (93) `cid_cff_unusable_font_dict::{glyphs_of_a_usable_font_dict_draw_beside_an_unusable_one, a_program_with_no_usable_font_dict_draws_no_square}` (pdfcraft-render integration tests); (94) `raster::tests::a_font_without_a_subtype_is_drawn_by_its_base_font` (pdfcraft-render); (98) `symbol_mt_builtin_encoding::{symbol_mt_draws_symbols_through_the_built_in_encoding, a_name_that_only_starts_with_symbol_is_not_symbol, symbol_mt_applies_its_differences, an_embedded_symbol_mt_draws_its_own_program}` (pdfcraft-render integration tests); (99) `raster::tests::thin_patches_of_a_mesh_in_a_tight_path_are_all_painted` (pdfcraft-render); (100) `text::tests::numbered_glyph_names_read_as_their_own_code`, `text::tests::numbered_type3_glyph_names_read_as_their_own_code`, `text::tests::glyph_names_outside_the_glyph_list_still_read_as_text` (pdfcraft-render); (101) `raster::tests::symbolic_truetype_mac_roman_text::{a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman, winansi_codes_of_a_font_in_the_macintosh_order_read_as_before, a_symbolic_truetype_font_without_a_1_0_subtable_reads_as_before, a_1_0_subtable_selecting_other_glyphs_than_the_drawn_ones_is_not_read, a_font_that_names_its_glyphs_is_not_read_by_their_order, a_symbol_or_zapfdingbats_font_in_the_macintosh_order_gets_no_mac_roman_reading, differences_naming_notdef_clear_the_mac_roman_reading_for_its_code, a_base_encoding_name_this_reader_does_not_know_keeps_the_mac_roman_reading_off}` (pdfcraft-render), `pdf::tests::a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman` (pdfcraft-fonts); (102) `text::tests::tracked_vertical_latin_stays_one_line` (pdfcraft-render); (81) `raster::tests::replies_are_not_drawn_but_their_comment_and_group_members_are`, `raster::tests::an_unknown_or_malformed_rt_is_a_reply_and_a_widget_with_irt_still_draws` (hayro-interpret); (103) `raster::tests::the_view_auto_state_sets_a_groups_state_on_screen`, `raster::tests::layer_toggles_and_print_renders_are_not_the_view_auto_state` and `raster::tests::malformed_view_auto_state_is_ignored` (pdfcraft-render); (104) `raster::tests::isolated_cmyk_groups::an_rgb_colour_in_an_isolated_cmyk_group_blends_as_its_ink` and `::an_rgb_colour_with_a_non_finite_component_has_no_tints` (pdfcraft-render) | | hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Each target row averages the whole source rows it covers (contiguous, cheap) and is then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms); working a row at a time (`area_shrink_rows`) keeps its work memory to a few rows. An image with a soft mask, a stencil's ink included, is premultiplied a source row at a time as it is averaged (`area_shrink_premultiplied`), never as a full-resolution RGBA copy: a 10718 × 13701 stencil shown at page size peaked at 1.16 GB with that copy and peaks at 1.03 GB without it, as with nearest-neighbour. A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged by more than 1% on either side (`SMOOTHED_IMAGE_SCALE`: a scan drawn at its resolution can come out a few ulps above 1, and blurred by half a pixel at a half-pixel offset), also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms; (30) a stencil mask painted with a pattern is drawn under a clip of its device rectangle, padded by two pixels: the pattern in an isolated layer, cut to the stencil by a layer holding it composited with `Compose::DestIn` (drawn without a Type 3 glyph's image rules, as the sub-render it replaces was), instead of through a canvas-sized luminance mask rendered for every draw; and a coloured tiling pattern's cell pixmap, repeated or shown once under patch (20), is drawn once per setting of the pattern and cell scale (`Renderer::tiling_cell`, `TilingCells`, eight entries per renderer, keyed by hayro-interpret patch (78)'s `TilingPattern::instance`) instead of once per fill, where every cell was kept until the page was composited. pdf.js issue 8078 paints 73,047 16 × 16 stencils with one pattern whose cell is 3.5 MB at 36 dpi: it ran out of memory after 5 minutes and now renders in about 5 s. For its tests, `offscreen_draws` counts on each thread the tiling cells drawn and the pixels of every pixmap drawn offscreen (all made by `offscreen_pixmap`), so a page's cost is checked by count, not by time; (31) `RenderSettings::max_coverage` keeps the rasterizer to that many coverage values for the page (vello_cpu patch 1), and a page whose paths it left out says so with `InterpreterWarning::ContentTruncated` | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::a_shrunk_stencil_mask_keeps_thin_strokes`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched`, `raster::tests::a_scan_drawn_a_rounding_error_above_its_resolution_stays_sharp`; (30) `raster::tests::{a_patterned_stencil_mask_paints_only_where_it_inks, a_patterned_stencil_mask_under_a_soft_mask_is_masked_once, a_patterned_stencil_mask_in_a_type3_glyph_keeps_thin_strokes, many_patterned_stencil_masks_draw_no_page_sized_masks, fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}`; (31) `raster::tests::coverage_budget::a_page_past_its_coverage_leaves_the_rest_out_and_says_so` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most as many symbol pixels per symbol dictionary as the largest page bitmap holds, 2^28 (a fuzzed dictionary decoded for minutes; each symbol now pays for its pixels out of the stream's decoding work and is held in the retained bitmaps, and at the earlier 2^26 the soft mask of a scanned drawing, one dictionary of 7,559 symbols, was refused: #6722), and a refusal by one of the decoder's limits is told apart from damaged data (`DecodeError::is_limit`, `OverflowError::BitmapLimit`); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change; (13) a bitmap filled with a default pixel of 1 (a page, or an intermediate text or halftone region) had 1s past its width in the last word of each row, and a refinement, which reads its reference a word at a time, read them as pixels where the standard has 0 (6.3.5.2): refining such a reference not a multiple of 32 wide decoded to other pixels than were coded, release builds too. `Bitmap::clear_padding` clears those bits when a bitmap is filled with 1s, as cutting symbols and patterns from a collective bitmap already did. Ghostscript reads the test's page and text streams as they decode now; (14) a region's location is unsigned (5.4.4), so one past `i32::MAX` is off the page, where the decoder took it for a signed one: a region there was drawn back onto the page from the left or the top (at 0xFFFFFFFC, four pixels left of it), and a refinement there, of the page or of a region at the same place (7.4.7.5), was refused as an overflow, and the page with it. `Bitmap::combine_region` keeps such a location past the page. A refinement of another region refines it where it is, as Table 35 has it (GRREFERENCEDX and GRREFERENCEDY 0), whatever the two locations say; a refinement of the page, see (16); (15) a refinement whose reference is offset right of it (GRREFERENCEDX above 0, which a refined text instance has from a wider bitmap, Table 12, and a refined symbol from an RDX above 0, Table 18) read the reference's first 32 columns as 0: `RefinementContextGatherer::maybe_reload_buffers` emptied its buffers left of the reference and didn't load them again at its first column. Such a refinement decoded to other pixels than were coded, or failed and lost the page. Left of the reference the buffers now keep the words of its first column each row starts with; with an offset of 0 or less nothing changes; (16) a refinement of the page refines "the current contents of the page buffer, restricted to the area of the page buffer specified by this segment's region segment information field" (7.4.7.4), at an offset of 0 (Table 35): `Bitmap::area` copies that area a word at a time, 0 off the page and past its width whatever the padding of its rows holds, and every refinement region reads its reference at offset 0. The page around the area was read too, so a refinement of the page decoded to other pixels than were coded, and one wider or taller than the page was refused. And `RefinementContextGatherer::start_row` loads the reference's words a column left of where a row starts in it, as the reload would, not from column 0: with the reference offset 32 or more left of the refinement (GRREFERENCEDX below −31, as a refined text instance has from a narrower bitmap or an RDX below −31, and a refined symbol from such an RDX), each row's first context read 0 for the reference column it starts at, and from 33 left for the one before it too, and the refinement decoded to other pixels or failed; (17) `Image::new_embedded` reads data that starts with a JBIG2 file header (the ID string `0x97 4A 42 32 0D 0A 1A 0A`, flags and an optional page count, D.4) past it, in the organization the header declares (sequential or random access), its segments counted together with the globals' as (8) has them: ISO 32000-1 7.4.7 has `JBIG2Decode` data in the embedded organization (D.3), but some producers write the file header too, and Acrobat and PDFBox draw such streams. The header's ID string was read as a segment header and the image failed (`unknown or reserved segment type`), so the page drew white | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding (about a second at the earlier 2^26 bound; not run at 2^28), `raster::tests::a_symbol_dictionary_holds_as_many_pixels_as_a_page`, and for `is_limit` and `BitmapLimit` `error::pdfcraft_limit_tests`, `bitmap::pdfcraft_limit_tests` and `gray_scale::pdfcraft_limit_tests` (hayro-jbig2); (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing`; (13) `jbig2::decode::tests::a_refinement_reads_nothing_past_its_references_width` (pdfcraft-filters), and in the crate `tests::a_bitmap_of_1s_has_nothing_past_its_width`; (14) `jbig2::decode::tests::a_region_located_past_i32_max_is_off_the_page` (pdfcraft-filters), and in the crate `tests::a_region_located_past_i32_max_is_off_the_page` and `tests::a_refinement_of_a_region_decodes_the_same_wherever_they_are`; (15) `jbig2::decode::tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column` (pdfcraft-filters), and in the crate `tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column`; (16) `jbig2::decode::tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` (pdfcraft-filters), and in the crate `tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` and `tests::a_page_area_is_the_page_inside_it_and_0_outside` ; (17) `jbig2::decode::tests::a_stream_with_a_jbig2_file_header_decodes_past_it`, `jbig2::decode::tests::a_truncated_jbig2_file_header_is_an_error`, `jbig2::decode::tests::segments_after_a_jbig2_file_header_count_with_the_globals` (pdfcraft-filters), `raster::tests::a_jbig2_image_with_a_file_header_draws` (pdfcraft-render) | | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365); (7) object parsing builds at most 4,194,304 objects (every array element and dictionary value at any depth, `parser::MAX_OBJECT_NODES`, the same cap as `pdfcraft_cos`) per top-level object, an object stream reads at most 1,000,000 index entries (cos's `/N` clamp), and one load builds at most 8,388,608 objects from all of its object streams together, shared across rayon workers (a decoded stream of `0 0 0 ...` inside the 256 MiB decompression limit built ~120M objects, 10+ GB, and aborted). A lookup parses only the member it wants, so it isn't skipped behind a large member, and a load parses only the members it would add: none its cross-reference table places in another stream (stale copies) and none already loaded as an ordinary object or from an earlier stream. Members the table doesn't list at all are still parsed (a rebuilt table lists no compressed objects) and share the budget, so they can spend it before listed members of the same stream; a load expands its object streams one at a time in object-number order, a stream's members in parallel only when the budget holds all of them and otherwise in index order, so a spent budget skips the same members on every load (#5083) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog`; (7) `inspect::tests::{object_stream_members_over_the_node_cap_are_skipped, object_stream_index_entries_past_the_clamp_are_ignored, object_streams_of_one_load_share_their_node_budget, object_streams_loaded_in_parallel_share_their_node_budget, a_malformed_object_stream_member_refunds_its_reservation, a_tight_budget_builds_object_stream_members_in_index_order, a_lookup_in_an_object_stream_parses_only_the_member_it_wants}` (pdfcraft-render; `NodeBudget` and `ObjectStream::new_within` are public so these can use a small budget) | diff --git a/vendor/hayro-interpret/src/cmyk_inverse.rs b/vendor/hayro-interpret/src/cmyk_inverse.rs new file mode 100644 index 0000000000..74804f92a7 --- /dev/null +++ b/vendor/hayro-interpret/src/cmyk_inverse.rs @@ -0,0 +1,118 @@ +//! PdfCraft patch (104): an sRGB colour as the process ink that draws back to it through the +//! CMYK profile pages are drawn with, for an additive colour painted into a `DeviceCMYK` +//! blending space (ISO 32000-2 §11.3.4). +//! +//! Black is `K = 0.4 · k₀² + 0.6 · k₀⁸` with `k₀ = 1 − max(r, g, b)`, light in the mid tones +//! and full for black; cyan, magenta and yellow are then solved for by Newton's method at that +//! black, clamped to the ink range. A colour inside the profile's gamut draws as itself; one +//! outside it lands where the clamped solve stops. The curve matches Acrobat's output for the +//! greys, black and red measured in a CMYK page group, plain and under a blend mode; +//! §10.4.2.4's black generation and undercolour removal is for a device without a profile. + +use moxcms::{ColorProfile, Layout, TransformF32Executor, TransformOptions}; +use std::cell::Cell; +use std::sync::{Arc, LazyLock}; + +use crate::color::CMYK_PROFILE; + +/// The weight of `k₀²` in the black ink; the rest is on `k₀⁸`. +const BLACK_GENERATION: f32 = 0.4; +/// Newton steps; each costs four profile evaluations. +const MAX_STEPS: usize = 12; +/// Close enough: well under one 8-bit level. +const TOLERANCE: f32 = 0.5 / 255.0; +/// The finite-difference step for the Jacobian. +const STEP: f32 = 0.02; + +static FORWARD: LazyLock>> = LazyLock::new(|| { + ColorProfile::new_from_slice(CMYK_PROFILE) + .ok()? + .create_transform_f32( + Layout::Rgba, + &ColorProfile::new_srgb(), + Layout::Rgb, + TransformOptions::default(), + ) + .ok() +}); + +/// The process ink `[C, M, Y, K]` (0 = no ink) that the CMYK profile draws as `rgb`; for a +/// colour outside its gamut, where the clamped solve stops. `None` when the +/// profile can't be evaluated or `rgb` isn't finite. +/// +/// The last colour converted on this thread is kept, as a run of fills (the glyphs of a +/// line of text) usually shares one colour. +pub(crate) fn srgb_to_cmyk(rgb: [f32; 3]) -> Option<[f32; 4]> { + if !rgb.iter().all(|v| v.is_finite()) { + return None; + } + let rgb = rgb.map(|v| v.clamp(0.0, 1.0)); + let key = rgb.map(f32::to_bits); + if let Some((last, ink)) = LAST.get() + && last == key + { + return Some(ink); + } + let ink = solve_ink(rgb)?; + LAST.set(Some((key, ink))); + Some(ink) +} + +thread_local! { + static LAST: Cell> = const { Cell::new(None) }; +} + +fn solve_ink(rgb: [f32; 3]) -> Option<[f32; 4]> { + let forward: &TransformF32Executor = FORWARD.as_ref()?.as_ref(); + let k0 = 1.0 - rgb[0].max(rgb[1]).max(rgb[2]); + let k = BLACK_GENERATION * k0 * k0 + (1.0 - BLACK_GENERATION) * k0.powi(8); + let eval = |cmy: [f32; 3]| draw(forward, [cmy[0], cmy[1], cmy[2], k]); + + let mut cmy = rgb.map(|v| 1.0 - v); + for _ in 0..MAX_STEPS { + let drawn = eval(cmy)?; + let residual: [f32; 3] = std::array::from_fn(|i| drawn[i] - rgb[i]); + if residual.iter().all(|r| r.abs() < TOLERANCE) { + break; + } + // Column `j` of the Jacobian: how the drawn colour moves with ink `j`. + let mut columns = [[0.0_f32; 3]; 3]; + for (j, column) in columns.iter_mut().enumerate() { + let step = if cmy[j] <= 1.0 - STEP { STEP } else { -STEP }; + let mut moved = cmy; + moved[j] += step; + let shifted = eval(moved)?; + *column = std::array::from_fn(|i| (shifted[i] - drawn[i]) / step); + } + let Some(delta) = solve(columns, residual.map(|r| -r)) else { + break; + }; + cmy = std::array::from_fn(|i| (cmy[i] + delta[i]).clamp(0.0, 1.0)); + } + Some([cmy[0], cmy[1], cmy[2], k]) +} + +fn draw(forward: &TransformF32Executor, cmyk: [f32; 4]) -> Option<[f32; 3]> { + let mut out = [0.0_f32; 3]; + forward.transform(&cmyk, &mut out).ok()?; + out.iter().all(|v| v.is_finite()).then_some(out) +} + +/// `x` with `A x = b`, `A` given by its columns (Cramer's rule). `None` when `A` is +/// singular or the result isn't finite. +fn solve(a: [[f32; 3]; 3], b: [f32; 3]) -> Option<[f32; 3]> { + let det = |[c0, c1, c2]: [[f32; 3]; 3]| { + c0[0] * (c1[1] * c2[2] - c2[1] * c1[2]) - c1[0] * (c0[1] * c2[2] - c2[1] * c0[2]) + + c2[0] * (c0[1] * c1[2] - c1[1] * c0[2]) + }; + let d = det(a); + if !d.is_finite() || d.abs() < 1e-9 { + return None; + } + let x: [f32; 3] = std::array::from_fn(|j| { + let mut m = a; + m[j] = b; + det(m) / d + }); + x.iter().all(|v| v.is_finite()).then_some(x) +} diff --git a/vendor/hayro-interpret/src/color.rs b/vendor/hayro-interpret/src/color.rs index 4021b31e04..4a7a05f1b6 100644 --- a/vendor/hayro-interpret/src/color.rs +++ b/vendor/hayro-interpret/src/color.rs @@ -1552,9 +1552,10 @@ impl Color { /// its opacity, for blending in a `DeviceCMYK` transparency group (ISO 32000-2 §11.3.4). /// /// `DeviceCMYK` gives its components. `DeviceGray` is black ink alone, `K = 1 − gray` - /// (§10.4.2.2). `None` for every other space, ICCBased CMYK included: such a colour has no - /// tints without a colour conversion this crate does not make (an ICCBased colour keeps - /// its own profile), and the caller draws it as before. + /// (§10.4.2.2). PdfCraft patch (104): an additive colour (`DeviceRGB`, `CalRGB`, `Lab`, a + /// three-component ICCBased colour) is the ink that draws back to it through the CMYK + /// profile. `None` for every other space, ICCBased CMYK included (it keeps its own + /// profile), and the caller draws it as before. pub fn ink_tints(&self) -> Option<([f32; 4], f32)> { let c = |i: usize| { self.components @@ -1566,11 +1567,27 @@ impl Color { let tints = match self.color_space.0.as_ref() { ColorSpaceType::DeviceCmyk => [c(0), c(1), c(2), c(3)], ColorSpaceType::DeviceGray => [0.0, 0.0, 0.0, 1.0 - c(0)], + ColorSpaceType::DeviceRgb | ColorSpaceType::CalRgb(_) | ColorSpaceType::Lab(_) => { + self.ink_from_srgb()? + } + ColorSpaceType::ICCBased(icc) if icc.0.number_components == 3 => { + self.ink_from_srgb()? + } _ => return None, }; Some((tints, self.opacity.clamp(0.0, 1.0))) } + fn ink_from_srgb(&self) -> Option<[f32; 4]> { + // `to_rgba` turns a non-finite component into some finite colour, which is not the + // colour's ink, so such a colour has no tints. + if self.components.iter().any(|c| !c.is_finite()) { + return None; + } + let [r, g, b, _] = self.to_rgba().components(); + crate::cmyk_inverse::srgb_to_cmyk([r, g, b]) + } + /// Create a color from RGBA. pub fn from_rgba(rgba: AlphaColor) -> Self { let c = rgba.components(); @@ -1592,9 +1609,11 @@ pub fn cmyk_to_srgb8(input: &[u8], output: &mut [u8]) -> Option<()> { CMYK_TRANSFORM.convert_u8(input, output) } -static CMYK_TRANSFORM: LazyLock = LazyLock::new(|| { - ICCProfile::new(include_bytes!("../assets/CGATS001Compat-v2-micro.icc"), 4).unwrap() -}); +/// The CMYK profile pages are drawn with. +pub(crate) const CMYK_PROFILE: &[u8] = include_bytes!("../assets/CGATS001Compat-v2-micro.icc"); + +static CMYK_TRANSFORM: LazyLock = + LazyLock::new(|| ICCProfile::new(CMYK_PROFILE, 4).unwrap()); pub(crate) trait ToRgb { fn convert_sample(&self, input: &[f32], output: &mut [u8], manual_scale: bool) -> Option<()> { diff --git a/vendor/hayro-interpret/src/lib.rs b/vendor/hayro-interpret/src/lib.rs index 01e717da93..03a0f44811 100644 --- a/vendor/hayro-interpret/src/lib.rs +++ b/vendor/hayro-interpret/src/lib.rs @@ -33,6 +33,7 @@ mod log; mod cache; mod cmyk_group; +mod cmyk_inverse; mod context; mod convert; mod device; diff --git a/vendor/hayro/src/renderer.rs b/vendor/hayro/src/renderer.rs index c8640adb14..372f7e28cc 100644 --- a/vendor/hayro/src/renderer.rs +++ b/vendor/hayro/src/renderer.rs @@ -922,8 +922,8 @@ impl TilingCells { #[derive(Clone, Copy, Default)] struct InkState { pass: Option, - /// Something in the group has no ink tints (an RGB colour, a pattern, an image): the - /// group is drawn as before instead. + /// Something in the group has no ink tints (a pattern, an image, an ICCBased CMYK colour): + /// the group is drawn as before instead. unsupported: bool, } From 490287fc9d04d1915c690a3d0f6954bde706e15e Mon Sep 17 00:00:00 2001 From: Ken Mantle Date: Thu, 12 Nov 2026 11:58:14 -0500 Subject: [PATCH 45/46] prc: rebuild compressed tessellations whose normals are recalculated (#1297) (#7034) * prc: rebuild compressed tessellations whose normals are recalculated A TESS_3D_Compressed (WD 7.8.9.7) was read to its end but never rebuilt, so prc-render drew nothing for it. Rebuild the form whose normals are recalculated, with three edge statuses per triangle, into a Mesh grouped by face and styled by each face's line attribute. The walk (tess/compressed.rs) takes a seed, then one apex per triangle, from a reference or the next point in the frame of the edge it continues across (WD 7.8.9.2), with the WD 12.3 Length, Unitize and MakeOrthoRep arithmetic bit for bit. Arrays that do not fit, or an edge shared by more than two triangles, leave the tessellation Compressed. Every triangle, style and face range is charged to the element allowance first. Stored-normals meshes, the one-status-per-triangle form, the planar bend check, the best-fit search and partial salvage are later PRs. Adapted from pdfcer (MIT, same author) at 0a249a73. Co-Authored-By: Claude Opus 5.5 * prc: charge rebuilt compressed meshes after they fit; model_render test The rebuilt triangles, their styles and the face ranges are now charged to the element allowance once the arrays fit and before the face ranges are made. Arrays the walk cannot fit stay Compressed and spend nothing, so they no longer fail the file over a face count they never use. Tests: - a_rebuilt_compressed_mesh_is_charged_its_triangles_and_faces pins the exact charge (4 triangles x 3 elements + 2 faces); - arrays_that_do_not_fit_spend_no_triangles_or_faces; - rebuilt_triangles_are_grouped_by_face_with_their_styles, a direct by_face test with out-of-order faces, empty faces and styles; - model_render's a_compressed_prc_model_is_rebuilt_and_drawn, end to end from a PDF, cited in misc.3d-view; its hex fixture is the prc-render builder's output, kept so by the_automation_fixture_is_the_compressed_triangle. The walk's working tables are documented as bounded by the triangle count rather than charged. The module docs now say the arithmetic matches the WD pseudocode as read and that the traversal rules are inferred, not WD text. Co-Authored-By: Claude Opus 5.5 * prc: charge compressed styles and face ranges before making them A rebuilt compressed mesh's per-triangle styles were built before the budget was charged, and also when the normals were stored and the mesh was read past. The face attributes are now kept as read, and the styles are built only for a rebuilt mesh, after the charge. Each face range is a Range, two budget elements on 64-bit targets, so it is charged at its size, not as one element. Co-Authored-By: Claude Opus 5.5 * prc: bound the compressed rebuild walk by the allowance left The walk's vertices, triangles, edge counts, edge log and stack were allocated before anything was charged, and could hold several times the decoded arrays. The walk now gets the elements the reader has left and stops once its arrays, by length, would hold more; the tessellation then stays compressed, as when its arrays do not fit the walk. A face whose line attribute is 0 but whose behaviour is not now styles its triangles, so the behaviour bits are kept. Co-Authored-By: Claude Opus 5.5 * prc: say the rebuild walk is bounded by the allowance Co-Authored-By: Claude Opus 5.5 * prc: cap the rebuild walk's first reservation by its limit Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: KenM76 Co-authored-by: Claude Opus 5.5 --- crates/automation/tests/it/model_render.rs | 23 ++ crates/prc-render/tests/model.rs | 48 +++ crates/prc/src/bits.rs | 1 - crates/prc/src/tess.rs | 181 ++++++++--- crates/prc/src/tess/compressed.rs | 353 +++++++++++++++++++++ crates/prc/src/tess/compressed/tests.rs | 299 +++++++++++++++++ crates/prc/src/tess/tests.rs | 271 +++++++++++++++- parity/acrobat-features.toml | 4 +- 8 files changed, 1136 insertions(+), 44 deletions(-) create mode 100644 crates/prc/src/tess/compressed.rs create mode 100644 crates/prc/src/tess/compressed/tests.rs diff --git a/crates/automation/tests/it/model_render.rs b/crates/automation/tests/it/model_render.rs index c7dcca1af6..4323ca50e7 100644 --- a/crates/automation/tests/it/model_render.rs +++ b/crates/automation/tests/it/model_render.rs @@ -9,6 +9,11 @@ use serde_json::{Value, json}; /// file structure with no product tree, in hexadecimal for the stream's `/ASCIIHexDecode`. const TRIANGLE_PRC_HEX: &str = "505243c91f0000c91f00000100000002000000030000000400000000000000000000000000000000000000010000000500000006000000070000000800000000000000060000006b0000009a000000a3000000ab000000cd000000d5000000dd000000e500000000000000505243c91f0000c91f0000050000000600000007000000080000000000000000000000000000000000000000000000789c63000000010001789c030000000001789c9b71c087758da2422b43084fe3140e038713464cd759381a162800006ed70828789c030000000001789c030000000001789c030000000001"; +/// The same triangle stored as a `TESS_3D_Compressed` whose normals are recalculated, so it is +/// drawn only once its triangles are rebuilt. `prc-render`'s tests build it +/// (`the_automation_fixture_is_the_compressed_triangle` keeps this copy their output). +const COMPRESSED_TRIANGLE_PRC_HEX: &str = "505243c91f0000c91f00000100000002000000030000000400000000000000000000000000000000000000010000000500000006000000070000000800000000000000060000006b0000009a000000a3000000ab000000d2000000da000000e2000000ea00000000000000505243c91f0000c91f0000050000000600000007000000080000000000000000000000000000000000000000000000789c63000000010001789c030000000001789c9b71c08775aded1d0624d0d2c808055a8d0c8c8ccc0c0c0c0a0a223ce2e50c2c00bed0061f789c030000000001789c030000000001789c030000000001"; + /// A one-page PDF whose page holds a 3D annotation over a stream of `/Subtype /{subtype}`. fn pdf(subtype: &str, hex: &str) -> Vec { let stream = format!("<< /Type /3D /Subtype /{subtype} /Filter /ASCIIHexDecode /Length {} >>\nstream\n{hex}>\nendstream", hex.len() + 1); @@ -82,6 +87,24 @@ fn a_prc_model_is_drawn_over_the_background() { assert!(a.call("model_render", &json!({ "doc": doc, "page": 1, "model": 1 })).is_ok()); } +#[test] +fn a_compressed_prc_model_is_rebuilt_and_drawn() { + let (dir, mut a) = setup("prc-compressed"); + std::fs::write(dir.join("in/compressed.pdf"), pdf("PRC", COMPRESSED_TRIANGLE_PRC_HEX)).unwrap(); + let doc = open(&mut a, "compressed.pdf"); + let args = json!({ "doc": doc, "page": 1, "width": 64, "height": 48, "direction": [0, 0, -1], "perspective": false }); + let out = a.call("model_render", &args).unwrap(); + let Content::Png { data, .. } = &out[0] else { panic!("expected a PNG") }; + assert_eq!(rgb(data, 0, 0), [255, 255, 255], "the corner is background"); + assert_eq!(rgb(data, 32, 24), [190, 192, 200], "the centre is the triangle, lit head on"); + // Its apex is up: the drawn span narrows from the base to the top, and the centre column + // is covered over many rows. + let (top, bottom) = rows(data, 32); + assert!(bottom - top > 20, "rows {top}..{bottom}"); + let (near_top, near_bottom) = (extent(data, top + 4), extent(data, bottom - 2)); + assert!(near_bottom.1 - near_bottom.0 > 2 * (near_top.1 - near_top.0), "{near_top:?} {near_bottom:?}"); +} + #[test] fn a_u3d_model_or_a_missing_one_is_a_clear_error() { let (_dir, mut a) = setup("u3d"); diff --git a/crates/prc-render/tests/model.rs b/crates/prc-render/tests/model.rs index 80e84252c6..ce94ceb4ce 100644 --- a/crates/prc-render/tests/model.rs +++ b/crates/prc-render/tests/model.rs @@ -78,6 +78,30 @@ fn tess_section(points: &[[f64; 3]], normals: &[[f64; 3]], corners: &[(u32, u32) w.bytes() } +/// A tessellation section of one `TESS_3D_Compressed` (authoring version 8137, origin 0, +/// tolerance 0.5) whose normals are recalculated: the seed triangle `seed`, in tolerances +/// [WD 7.8.9.7]. +fn compressed_section(seed: [i64; 9]) -> Vec { + let mut w = W::default(); + w.uint(305).uint(0).bit(true).uint(1); + w.uint(173).bit(false).bit(true).double(0.5).put(0, 32).put(0, 32).put(0, 32); + // point_array: 4-bit sign-magnitude integers. + w.bit(false).uint(9); + seed.iter().for_each(|_| _ = w.put(4, 8)); + seed.iter().for_each(|&p| _ = w.bit(p < 0).put(p.unsigned_abs(), 3)); + // Three edge statuses, none set; one triangle on face 0; three slots, none a reference. + w.bit(false).uint(3).put(0, 24); + w.bit(false).uint(1).put(1, 8).bit(false); + w.uint(3).put(0, 3).uint(0); + // Normals recalculated; no colours or multiple line attributes; no style; no texture or + // behaviours. + w.bit(true).bit(false).double(0.5).put(0, 8).bit(false).bit(false); + w.bit(false).uint(0).bit(true).bit(false); + // The section's user data. + w.uint(0); + w.bytes() +} + /// A one-file-structure PRC stream whose tessellation section is `tess`, with no product tree. fn prc_file(tess: &[u8]) -> Vec { const HEADER_LEN: usize = 107; @@ -133,6 +157,30 @@ fn a_synthetic_prc_renders_its_nearer_triangle_over_the_farther() { assert_eq!(pixel(&image, 20, 34), [57, 58, 60, 255], "the farther triangle, lit edge on"); } +/// The seed (-1, -1, 0), (1, -1, 0), then the midpoint (0, -1, 0) raised by 2 to (0, 1, 0). +const COMPRESSED_TRIANGLE: [i64; 9] = [-2, -2, 0, 4, 0, 0, 0, 4, 0]; + +#[test] +fn a_compressed_tessellation_is_rebuilt_and_rendered() { + let prc = PrcFile::parse(&prc_file(&compressed_section(COMPRESSED_TRIANGLE))).unwrap(); + let meshes = assemble(&prc).unwrap(); + assert_eq!(meshes.len(), 1); + assert_eq!(meshes[0].mesh.triangles.len(), 1); + let camera = Camera::fit_meshes(&meshes, [0.0, 0.0, -1.0], [0.0, 1.0, 0.0], false, 1.0).unwrap(); + let image = render(&meshes, &camera, &RenderOptions { width: 40, height: 40, ..RenderOptions::default() }).unwrap(); + assert_eq!(pixel(&image, 1, 1), [255, 255, 255, 255], "background corner"); + assert_ne!(pixel(&image, 20, 24), [255, 255, 255, 255], "the triangle"); +} + +/// `model_render`'s end-to-end test embeds this builder's compressed triangle as hexadecimal; +/// this keeps that fixture the builder's output. +#[test] +fn the_automation_fixture_is_the_compressed_triangle() { + let hex: String = prc_file(&compressed_section(COMPRESSED_TRIANGLE)).iter().map(|b| format!("{b:02x}")).collect(); + let fixture = include_str!("../../automation/tests/it/model_render.rs"); + assert!(fixture.contains(&format!("const COMPRESSED_TRIANGLE_PRC_HEX: &str = \"{hex}\";")), "{hex}"); +} + #[test] fn damaged_streams_are_errors_not_panics() { let points = [[0., 0., 0.], [1., 0., 0.], [0., 1., 0.]]; diff --git a/crates/prc/src/bits.rs b/crates/prc/src/bits.rs index 837d281f3b..b6897e56f6 100644 --- a/crates/prc/src/bits.rs +++ b/crates/prc/src/bits.rs @@ -69,7 +69,6 @@ impl<'a> BitReader<'a> { } /// Array elements this reader may still decode. - #[cfg(test)] pub(crate) fn elements_left(&self) -> usize { self.budget.elements } diff --git a/crates/prc/src/tess.rs b/crates/prc/src/tess.rs index c9bffb30f4..a7614b0a5c 100644 --- a/crates/prc/src/tess.rs +++ b/crates/prc/src/tess.rs @@ -2,12 +2,14 @@ //! (172) with its `TESS_Face`s (174), `TESS_3D_Wire` (175), `TESS_Markup` (176) and //! `TESS_3D_Compressed` (173) [WD 7.3.7, 7.8]. //! -//! `TESS_3D_Compressed` is read field by field to its end [WD 7.8.9.7] so the entities after -//! it decode; its triangles are not rebuilt. +//! `TESS_3D_Compressed` is read field by field to its end [WD 7.8.9.7]. One whose normals are +//! recalculated is rebuilt into triangles ([`compressed`]); one that stores its normals, or +//! whose arrays do not fit the traversal, is not. //! //! The schema runs after the fields of each concrete type read here (201, 172, 173, 174, 175, //! 176, 305), not for the abstract levels (`PRCBase`, `TESS`). +mod compressed; mod triangulate; use crate::arrays; @@ -49,7 +51,8 @@ pub enum Tessellation { Wire(Vec>), /// A `TESS_Markup` (PMI drawing), read past but not decoded. Markup, - /// A `TESS_3D_Compressed` mesh, read past but not rebuilt. + /// A `TESS_3D_Compressed` mesh, read past but not rebuilt: it stores its normals, or its + /// arrays do not fit the traversal. One that is rebuilt is a [`Tessellation::Mesh`]. #[non_exhaustive] Compressed { /// The entity's triangle count. @@ -65,7 +68,8 @@ pub struct TriangleMesh { /// The coordinate array as points. pub positions: Vec<[f64; 3]>, /// Counter-clockwise triangles seen from the outside, as indices into - /// [`Self::positions`] [WD 7.8.5.2]. + /// [`Self::positions`] [WD 7.8.5.2]. A rebuilt `TESS_3D_Compressed` keeps its traversal's + /// winding, which is not oriented. pub triangles: Vec<[u32; 3]>, /// `triangles` split by `TESS_Face`: face `i` is `triangles[faces[i].clone()]`. pub faces: Vec>, @@ -136,7 +140,7 @@ impl Ctx<'_, '_> { self.markup()?; Tessellation::Markup } - TESS_3D_COMPRESSED => Tessellation::Compressed { triangles: self.tess_3d_compressed()? }, + TESS_3D_COMPRESSED => self.tess_3d_compressed()?, t => return Err(malformed(format!("entity type {t} in the tessellation array"))), }); } @@ -295,23 +299,25 @@ impl Ctx<'_, '_> { self.schema.skip_added_fields(TESS_MARKUP, &mut self.r) } - /// `TESS_3D_Compressed` (173) after its type code [WD 7.8.9.7], read past; returns its - /// triangle count. prc-rs: the field order and version gates follow its schema. + /// `TESS_3D_Compressed` (173) after its type code [WD 7.8.9.7]: rebuilt into a mesh when + /// its normals are recalculated and its arrays fit, else read past to its end. prc-rs: the + /// field order and version gates follow its schema. /// /// Sizes the WD leaves unstated: the face count is the largest face index + 1, /// `normal_is_reversed` is one bit per triangle, and the point-reference array's /// compressed flag is implicit (more than three references). - fn tess_3d_compressed(&mut self) -> Result { + fn tess_3d_compressed(&mut self) -> Result { let r = &mut self.r; r.bit()?; // is_calculated r.bit()?; // has_faces - r.double()?; // tolerance + let tolerance = r.double()?; + let mut origin = [0.0f64; 3]; if self.version >= ORIGIN_FROM { - for _ in 0..3 { - r.float_as_bytes()?; + for o in &mut origin { + *o = f64::from(r.float_as_bytes()?); } } - arrays::compressed_integer_array(r)?; // point_array + let points = arrays::compressed_integer_array(r)?; // point_array let edge_status = arrays::character_array(r, 2, None, false)?; let face_of = arrays::compressed_indice_array(r, None)?; let t = face_of.len(); @@ -320,42 +326,50 @@ impl Ctx<'_, '_> { } let faces = face_of.iter().max().map_or(0, |&m| (m as usize).saturating_add(1)); let n = r.unsigned_integer()? as usize; - let references = arrays::bool_array(r, n)?.iter().filter(|&&b| b).count(); - let refs = arrays::compressed_indice_array(r, Some(references > 3))?; - if refs.len() != references { - return Err(malformed(format!("{} point references for {references} flagged points", refs.len()))); + let is_reference = arrays::bool_array(r, n)?; + let flagged = is_reference.iter().filter(|&&b| b).count(); + let references = arrays::compressed_indice_array(r, Some(flagged > 3))?; + if references.len() != flagged { + return Err(malformed(format!("{} point references for {flagged} flagged points", references.len()))); } - self.compressed_normals(t, faces)?; - let r = &mut self.r; - if r.bit()? { - arrays::bool_array(r, faces)?; // is_point_color_on_face - arrays::character_array(r, 8, None, false)?; // point_color_array + let recalc = self.compressed_normals(t, faces)?; + let attributes = self.compressed_attributes(faces)?; + if !recalc { + return Ok(Tessellation::Compressed { triangles: t }); } - if r.bit()? { - arrays::bool_array(r, faces)?; // is_multiple_line_attribute_on_face - } - arrays::short_array(r, 16)?; // line_attributes - if !r.bit()? { - self.compressed_texture_parameter()?; - if !self.r.bit()? { - arrays::bool_array(&mut self.r, faces)?; // face_has_texture - } - } - if self.r.bit()? { - arrays::character_array(&mut self.r, 8, None, false)?; // behaviours - } - self.schema.skip_added_fields(TESS_3D_COMPRESSED, &mut self.r)?; - Ok(t) + let arrays = compressed::Arrays { + tolerance, + origin, + points: &points, + edge_status: &edge_status, + triangles: t, + is_reference: &is_reference, + references: &references, + limit: self.r.elements_left(), + }; + // A walk that would outgrow the allowance left stops there, and the arrays stay + // compressed, as do arrays the walk does not fit. + let Ok(m) = compressed::rebuild(&arrays) else { + return Ok(Tessellation::Compressed { triangles: t }); + }; + // The rebuilt triangles, their styles and every face range are charged once the arrays + // fit and before any of them is made, so arrays that do not fit spend nothing. + let styled = attributes.styled(faces); + let kept = m.triangles.len().saturating_mul(triangle_elements(false, 0, styled)); + self.r.take_elements(kept.saturating_add(faces.saturating_mul(RANGE_ELEMENTS)))?; + let styles = if styled { attributes.styles(&face_of, faces) } else { Vec::new() }; + Ok(Tessellation::Mesh(by_face(m, &face_of, faces, &styles))) } - /// A compressed tessellation's normals, from `must_recalculate_normals` on, read past. - fn compressed_normals(&mut self, triangles: usize, faces: usize) -> Result<(), PrcError> { + /// A compressed tessellation's normals, from `must_recalculate_normals` on, read past; + /// returns whether they are recalculated. + fn compressed_normals(&mut self, triangles: usize, faces: usize) -> Result { let r = &mut self.r; if r.bit()? { arrays::bool_array(r, triangles)?; // normal_is_reversed r.double()?; // crease_angle r.character()?; // normal_recalculation_flags - return Ok(()); + return Ok(true); } let angle_bits = u32::from(r.character()?); if angle_bits > MAX_ANGLE_BITS { @@ -365,7 +379,29 @@ impl Ctx<'_, '_> { arrays::bool_array(r, n)?; // is_binary_normal arrays::short_array(r, angle_bits)?; // normal angles arrays::bool_array(r, faces)?; // is_face_planar - Ok(()) + Ok(false) + } + + /// A compressed tessellation's colours, line attributes, texture and behaviours, through + /// its added schema fields; returns what its faces' styles are made from. + fn compressed_attributes(&mut self, faces: usize) -> Result { + let r = &mut self.r; + if r.bit()? { + arrays::bool_array(r, faces)?; // is_point_color_on_face + arrays::character_array(r, 8, None, false)?; // point_color_array + } + // is_multiple_line_attribute_on_face + let multi = if r.bit()? { arrays::bool_array(r, faces)? } else { Vec::new() }; + let line_attributes = arrays::short_array(r, 16)?; + if !r.bit()? { + self.compressed_texture_parameter()?; + if !self.r.bit()? { + arrays::bool_array(&mut self.r, faces)?; // face_has_texture + } + } + let behaviours = if self.r.bit()? { arrays::character_array(&mut self.r, 8, None, false)? } else { Vec::new() }; + self.schema.skip_added_fields(TESS_3D_COMPRESSED, &mut self.r)?; + Ok(FaceAttributes { multi, line_attributes, behaviours }) } /// `CompressedTextureParameter` [WD 7.8.9.8-7.8.9.9; pdf-issues #729, #749], read past. @@ -387,6 +423,71 @@ impl Ctx<'_, '_> { } } +/// Budget elements one face range (`Range`) takes. +const RANGE_ELEMENTS: usize = std::mem::size_of::>().div_ceil(8); + +/// What a compressed tessellation's faces are styled from [WD 7.8.9.5]: with no face flagged +/// `is_multiple_line_attribute_on_face` (`multi`), `line_attributes` holds one style index + 1 +/// per face (0 keeps the owner's graphics) and `behaviours` one behaviour byte per face. +struct FaceAttributes { + multi: Vec, + line_attributes: Vec, + behaviours: Vec, +} + +impl FaceAttributes { + /// Whether the faces give their triangles styles. The one-per-triangle layout of a flagged + /// face is not established, so such a tessellation keeps its owner's graphics, as does one + /// whose attributes and behaviours are all 0. + fn styled(&self, faces: usize) -> bool { + let behaviours = self.behaviours.len() == faces && self.behaviours.iter().any(|&b| b != 0); + !self.multi.iter().any(|&b| b) && self.line_attributes.len() == faces && (behaviours || self.line_attributes.iter().any(|&a| a != 0)) + } + + /// Per triangle, the style its face gives it; call only when [`Self::styled`]. + fn styles(&self, face_of: &[u32], faces: usize) -> Vec { + let behaviour = |f: usize| { + if self.behaviours.len() == faces { self.behaviours.get(f).and_then(|&b| u16::try_from(b).ok()).unwrap_or(0) } else { 0 } + }; + face_of + .iter() + .map(|&f| { + let f = f as usize; + TriangleStyle { style: self.line_attributes.get(f).and_then(|&a| u32::try_from(a).ok()).unwrap_or(0), behaviour: behaviour(f) } + }) + .collect() + } +} + +/// A rebuilt compressed mesh with its triangles grouped by face, in walk order within each, +/// so face `i` is `triangles[faces[i].clone()]` as for a `TESS_3D`. `faces` is charged to the +/// element budget by the caller. +fn by_face(m: compressed::Rebuilt, face_of: &[u32], faces: usize, styles: &[TriangleStyle]) -> TriangleMesh { + let mut order: Vec = (0..m.triangles.len()).collect(); + order.sort_by_key(|&i| face_of.get(i).copied().unwrap_or(0)); + let mut mesh = TriangleMesh { positions: m.positions, normals_recalculated: true, ..TriangleMesh::default() }; + mesh.faces = Vec::with_capacity(faces); + for &i in &order { + let f = face_of.get(i).map_or(0, |&f| f as usize); + while mesh.faces.len() <= f { + let at = mesh.triangles.len(); + mesh.faces.push(at..at); + } + if let (Some(&t), Some(r)) = (m.triangles.get(i), mesh.faces.last_mut()) { + mesh.triangles.push(t); + r.end = mesh.triangles.len(); + } + if let Some(&s) = styles.get(i) { + mesh.triangle_styles.push(s); + } + } + while mesh.faces.len() < faces { + let at = mesh.triangles.len(); + mesh.faces.push(at..at); + } + mesh +} + /// The normal array and each triangle's corner normals, or both empty when any slot fails to /// name a normal: a mesh shades from its stored normals wholly or not at all. fn stored_normals(coords: &[f64], slots: &[[u32; 3]], triangles: usize) -> (Vec<[f64; 3]>, Vec<[u32; 3]>) { diff --git a/crates/prc/src/tess/compressed.rs b/crates/prc/src/tess/compressed.rs new file mode 100644 index 0000000000..53e082b4d2 --- /dev/null +++ b/crates/prc/src/tess/compressed.rs @@ -0,0 +1,353 @@ +//! Triangle reconstruction for a `TESS_3D_Compressed` (173) whose normals are recalculated +//! [WD 7.8.9, 7.8.9.2, 7.8.9.7]. +//! +//! The WD names the arrays but not the traversal that wrote them; the rules below are the ones +//! the arrays fit when every slot, reference and point is consumed exactly. They are inferred +//! from compressed tessellations and are not WD text. +//! +//! Traversal. A component starts with a seed triangle of three slots; every later triangle +//! takes one slot, its apex. A slot flagged in `point_is_reference` takes the next +//! `point_reference_array` value, a vertex number; any other slot makes the next vertex from +//! the next `point_array` triple. Triangle `[A B C]`, entered across `[A B]`, reads +//! `edge_status & 3`: bit 1 continues across `[C B]`, bit 0 across `[A C]`; with both, +//! `[C B]` is taken and `[A C]` pushed. With neither, pushed edges are popped, skipping any +//! already shared by two triangles; an empty stack starts the next component. A triangle +//! whose continuation edge is already shared by two triangles, while the other edge is open, +//! continues across the open one. +//! +//! Geometry, in units of `tolerance`: +//! - seed: `V0 = origin + d`, `V1 = V0 + d`, `V2 = mid(V0, V1) + d`; +//! - apex across `[P Q]` with the triangle's third vertex `W` [WD 7.8.9.2]: with +//! `O = (P + Q) * 0.5`, `L` and `H` the lower- and higher-numbered of `P`, `Q`, +//! `X = unitize(L - H)`, `Z = unitize((W - O) x X)` and `Y = unitize(Z x X)`, the apex is +//! `((O + d.x X) - d.y Y) - d.z Z`. +//! +//! Arithmetic. The encoder frames each apex on the points it already approximated [WD 7.8.9], +//! so the decoder repeats the encoder's floating point: a last-bit difference compounds from +//! apex to apex. `unitize` therefore matches the WD's `PrcPt::Unitize` pseudocode [WD 12.3] as +//! read, whose length is not exact, and the operations keep the order above. +//! +//! Fold. A triangle folded against its parent swaps bits 1 and 0 for its own continuation. +//! Without stored normals to orient it, a new apex lying in its parent's plane on the parent's +//! side (`d.z == 0`, `d.y > 0`) is folded. A component that does not fit is rewound and walked +//! again with the fold inverted at its second triangle, where an apex that is a reference +//! leaves no other trace of its fold. +//! +//! Only the three-statuses-per-triangle `edge_status` form is reconstructed (the first `T` +//! are read); the one-per-triangle form walks differently. + +use std::collections::HashMap; + +type V = [f64; 3]; + +/// The arrays of one `TESS_3D_Compressed` the walk reads. +pub(super) struct Arrays<'a> { + pub(super) tolerance: f64, + pub(super) origin: V, + pub(super) points: &'a [i64], + pub(super) edge_status: &'a [i32], + pub(super) triangles: usize, + pub(super) is_reference: &'a [bool], + pub(super) references: &'a [u32], + /// Budget elements the walk may hold at once (see [`Walk::held`]). + pub(super) limit: usize, +} + +/// The rebuilt vertices and triangles, in walk order. +#[derive(Debug)] +pub(super) struct Rebuilt { + pub(super) positions: Vec, + pub(super) triangles: Vec<[u32; 3]>, +} + +/// Rebuilds the mesh, or says why the arrays do not fit the traversal. +pub(super) fn rebuild(a: &Arrays<'_>) -> Result { + if a.edge_status.len() != a.triangles.saturating_mul(3) { + return Err("the one-status-per-triangle edge form is not reconstructed"); + } + let mut w = Walk::new(a.triangles, a.limit); + w.walk(a)?; + if w.slot != a.is_reference.len() || w.ri != a.references.len() || w.pi != a.points.len() { + return Err("the decoded triangles do not use up the stored arrays"); + } + Ok(Rebuilt { positions: w.pos, triangles: w.tris }) +} + +fn add(a: V, b: V) -> V { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +fn sub(a: V, b: V) -> V { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +fn scale(a: V, s: f64) -> V { + a.map(|c| c * s) +} +fn dot(a: V, b: V) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +fn cross(a: V, b: V) -> V { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +fn key(a: u32, b: u32) -> (u32, u32) { + if a < b { (a, b) } else { (b, a) } +} + +/// `PrcPt::Length` [WD 12.3]: the squared length summed in `f64` and rounded to `f32`, then +/// Newton's square root in `f64`, seeded with that `f32` with its exponent halved. The printed +/// loop never advances its iterate; it is read as iterating until two successive values are +/// equal, `None` (the WD's -1) when 100 steps do not settle. +fn wd_length(a: V) -> Option { + let squared = (a[0] * a[0] + a[1] * a[1] + a[2] * a[2]) as f32; + let bits = squared.to_bits(); + let exponent = (bits >> 23) & 0xff; + let seed = if exponent > 127 { f32::from_bits((bits & !(0xff << 23)) | (((exponent - 127) / 2 + 127) << 23)) } else { squared }; + let squared = f64::from(squared); + let mut x = f64::from(seed); + for _ in 0..100 { + let next = 0.5 * (x + squared / x); + if next == x { + return Some(next); + } + x = next; + } + None +} + +/// `PrcPt::Unitize` [WD 12.3]: each component divided by [`wd_length`]; `None` below +/// `FLT_EPSILON`. +fn unitize(a: V) -> Option { + let l = wd_length(a).filter(|&l| l >= f64::from(f32::EPSILON))?; + Some([a[0] / l, a[1] / l, a[2] / l]) +} + +/// The `Z` of `PrcPt::MakeOrthoRep` [WD 12.3] on `X`: `unitize(X x (0, 1, 0))`, or +/// `X x (1, 0, 0)` when that is null, from `X` unitized again as the pseudocode does. +fn make_ortho_rep_z(x: V) -> Option { + let x = unitize(x)?; + unitize(cross(x, [0.0, 1.0, 0.0])).or_else(|| unitize(cross(x, [1.0, 0.0, 0.0]))) +} + +/// The apex across `[lo hi]` (`lo` the lower-numbered vertex) opposite `w`, `d` already scaled +/// by the tolerance [WD 7.8.9.2]. A null `Z` or `Y` takes `MakeOrthoRep(X)`'s `Z` as the WD +/// prescribes, with `Y = unitize(Z x X)` from the frame's own `X` (not the copy `MakeOrthoRep` +/// unitized again, whose last bits differ), turned a half-turn about `X` when the residue of +/// `(W - O) x X` points against that `Z`. A null `X`, an edge the WD's non-degeneracy rule +/// excludes, is used as it is. +fn apex(lo: V, hi: V, w: V, d: V) -> V { + let o = scale(add(lo, hi), 0.5); + let x = sub(lo, hi); + let x = unitize(x).unwrap_or(x); + let residue = cross(sub(w, o), x); + let (y, z) = unitize(residue) + .and_then(|z| Some((unitize(cross(z, x))?, z))) + .or_else(|| { + let z = make_ortho_rep_z(x)?; + let y = unitize(cross(z, x))?; + Some(if dot(residue, z) < 0.0 { (scale(y, -1.0), scale(z, -1.0)) } else { (y, z) }) + }) + .unwrap_or_default(); + sub(sub(add(o, scale(x, d[0])), scale(y, d[1])), scale(z, d[2])) +} + +/// Traversal state; `log` records every edge-count increment so a component can be rewound. +/// Everything it holds is checked against [`Arrays::limit`] after each triangle. +#[derive(Default)] +struct Walk { + pos: Vec, + tris: Vec<[u32; 3]>, + edges: HashMap<(u32, u32), u8>, + log: Vec<(u32, u32)>, + stack: Vec<(u32, u32, u32)>, + next: Option<(u32, u32, u32)>, + slot: usize, + ri: usize, + pi: usize, +} + +/// Where a component starts, to rewind to. +#[derive(Clone, Copy)] +struct Mark { + tri: usize, + pos: usize, + log: usize, + slot: usize, + ri: usize, + pi: usize, +} + +impl Walk { + /// Reserves room for `t` triangles, no more than `limit` elements' worth (two each). + fn new(t: usize, limit: usize) -> Self { + Walk { tris: Vec::with_capacity(t.min(limit / 2).min(1 << 20)), ..Walk::default() } + } + + fn mark(&self, tri: usize) -> Mark { + Mark { tri, pos: self.pos.len(), log: self.log.len(), slot: self.slot, ri: self.ri, pi: self.pi } + } + + fn rewind(&mut self, m: Mark) { + for k in self.log.drain(m.log..) { + if let Some(n) = self.edges.get_mut(&k) { + *n = n.saturating_sub(1); + } + } + self.pos.truncate(m.pos); + self.tris.truncate(m.tri); + self.stack.clear(); + self.next = None; + (self.slot, self.ri, self.pi) = (m.slot, m.ri, m.pi); + } + + /// One slot: `Some(v)` a reference to vertex `v`, `None` a new point. + fn take(&mut self, a: &Arrays<'_>) -> Result, &'static str> { + let flagged = *a.is_reference.get(self.slot).ok_or("slots run out")?; + self.slot = self.slot.saturating_add(1); + if !flagged { + return Ok(None); + } + let v = *a.references.get(self.ri).ok_or("references run out")?; + self.ri = self.ri.saturating_add(1); + self.get(v)?; + Ok(Some(v)) + } + + fn point(&mut self, a: &Arrays<'_>) -> Result { + let Some(&[x, y, z]) = a.points.get(self.pi..self.pi.saturating_add(3)) else { + return Err("points run out"); + }; + self.pi = self.pi.saturating_add(3); + let tol = a.tolerance; + Ok([x as f64 * tol, y as f64 * tol, z as f64 * tol]) + } + + fn get(&self, v: u32) -> Result { + self.pos.get(v as usize).copied().ok_or("a triangle refers to a vertex not yet decoded") + } + + fn push(&mut self, p: V) -> Result { + let id = u32::try_from(self.pos.len()).map_err(|_| "too many vertices")?; + self.pos.push(p); + Ok(id) + } + + /// The next triangle and its default fold: the seed, or the apex across the edge to + /// continue from. + fn triangle(&mut self, a: &Arrays<'_>) -> Result<([u32; 3], bool), &'static str> { + if let Some((p, q, w)) = self.next.take() { + if let Some(r) = self.take(a)? { + return Ok(([p, q, r], false)); + } + let (pp, qq, ww) = (self.get(p)?, self.get(q)?, self.get(w)?); + let (lo, hi) = if p < q { (pp, qq) } else { (qq, pp) }; + let fold = matches!(a.points.get(self.pi..self.pi.saturating_add(3)), Some(&[_, dy, 0]) if dy > 0); + let d = self.point(a)?; + let r = self.push(apex(lo, hi, ww, d))?; + return Ok(([p, q, r], fold)); + } + let mut v = [0u32; 3]; + for k in 0..3 { + let id = match self.take(a)? { + Some(r) => r, + None => { + let d = self.point(a)?; + let p = match k { + 0 => add(a.origin, d), + 1 => add(self.get(v[0])?, d), + _ => add(scale(add(self.get(v[0])?, self.get(v[1])?), 0.5), d), + }; + self.push(p)? + } + }; + if let Some(s) = v.get_mut(k) { + *s = id; + } + } + Ok((v, false)) + } + + /// Budget elements (8 bytes each) the walk's arrays hold, by length: growth can add up to + /// as much again in spare capacity. + fn held(&self) -> usize { + let pos = self.pos.len().saturating_mul(3); + let tris = self.tris.len().saturating_mul(2); + let edges = self.edges.len().saturating_mul(2); + let stack = self.stack.len().saturating_mul(2); + pos.saturating_add(tris).saturating_add(edges).saturating_add(self.log.len()).saturating_add(stack) + } + + fn closed(&self, e: (u32, u32, u32)) -> bool { + self.edges.get(&key(e.0, e.1)).copied().unwrap_or(0) >= 2 + } + + /// Adds one triangle; `flip` inverts its fold. + fn step(&mut self, a: &Arrays<'_>, status: i32, flip: bool) -> Result<(), &'static str> { + let ([ta, tb, tc], fold) = self.triangle(a)?; + for (x, y) in [(ta, tb), (tb, tc), (tc, ta)] { + let n = self.edges.entry(key(x, y)).or_insert(0); + *n = n.saturating_add(1); + let shared = *n; + self.log.push(key(x, y)); + if shared > 2 { + return Err("an edge is shared by more than two triangles"); + } + } + self.tris.push([ta, tb, tc]); + if self.held() > a.limit { + return Err("the rebuild needs more than the decoded-element allowance left"); + } + let (mut left, mut right) = ((tc, tb, ta), (ta, tc, tb)); + if fold != flip { + std::mem::swap(&mut left, &mut right); + } + let (taken, other) = if status & 2 != 0 { (left, right) } else { (right, left) }; + if status & 3 != 0 && self.closed(taken) && !self.closed(other) { + std::mem::swap(&mut left, &mut right); + } + match (status & 2 != 0, status & 1 != 0) { + (true, true) => { + self.next = Some(left); + self.stack.push(right); + } + (true, false) => self.next = Some(left), + (false, true) => self.next = Some(right), + (false, false) => {} + } + while self.next.is_none() { + let Some(e) = self.stack.pop() else { break }; + if !self.closed(e) { + self.next = Some(e); + } + } + Ok(()) + } + + /// Walks every component. One that fails is rewound and walked once more with the fold + /// inverted at its second triangle, so the walk is at most twice the triangle count. + fn walk(&mut self, a: &Arrays<'_>) -> Result<(), &'static str> { + let mut start = self.mark(0); + let mut flip_at = None; + let mut i = 0; + while i < a.triangles { + if self.next.is_none() && !(flip_at.is_some() && i == start.tri) { + start = self.mark(i); + flip_at = None; + } + let status = a.edge_status.get(i).copied().unwrap_or(0); + match self.step(a, status, flip_at == Some(i)) { + Ok(()) => i = i.saturating_add(1), + Err(e) => { + if flip_at.is_some() || i == start.tri { + return Err(e); + } + self.rewind(start); + flip_at = Some(start.tri.saturating_add(1)); + i = start.tri; + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/prc/src/tess/compressed/tests.rs b/crates/prc/src/tess/compressed/tests.rs new file mode 100644 index 0000000000..d174ff657e --- /dev/null +++ b/crates/prc/src/tess/compressed/tests.rs @@ -0,0 +1,299 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::*; + +const S: f64 = std::f64::consts::SQRT_2; + +fn run(points: &[i64], status: &[i32], t: usize, is_ref: &[bool], refs: &[u32]) -> Result { + rebuild(&Arrays { + tolerance: 0.5, + origin: [10.0, 0.0, 0.0], + points, + edge_status: status, + triangles: t, + is_reference: is_ref, + references: refs, + limit: usize::MAX, + }) +} + +fn close(a: V, b: V) -> bool { + (0..3).all(|i| (a[i] - b[i]).abs() < 1e-9) +} + +/// Seed `[0 1 2]`, then the apex across `[2 1]`: `X` runs from vertex 2 to vertex 1, and +/// each of `d`'s three components lands on its own axis with its own sign. Tolerance 0.5 +/// halves every step. +#[test] +fn a_seed_and_one_apex_rebuild() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 2, -6, 2]; + let m = run(&pts, &[2, 0, 0, 0, 0, 0], 2, &[false; 4], &[]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3]]); + let p: Vec = m.positions.iter().map(|v| [v[0] - 10.0, v[1], v[2]]).collect(); + assert!(close(p[0], [0.0, 0.0, 0.0]) && close(p[1], [2.0, 0.0, 0.0])); + assert!(close(p[2], [0.0, 2.0, 0.0]), "{:?}", p[2]); + assert!(close(p[3], [1.0 + 2.0 * S, 1.0 + S, -1.0]), "{:?}", p[3]); +} + +/// The walk stops once its arrays would hold more than `limit` elements: the seed-and-apex +/// pair ends holding four vertices (12), two triangles (4), five edges (10) and six logged +/// edges (6), 32 in all. +#[test] +fn a_walk_past_its_limit_stops() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 2, -6, 2]; + let with = |limit| { + rebuild(&Arrays { + tolerance: 0.5, + origin: [0.0; 3], + points: &pts, + edge_status: &[2, 0, 0, 0, 0, 0], + triangles: 2, + is_reference: &[false; 4], + references: &[], + limit, + }) + }; + assert_eq!(with(32).unwrap().triangles.len(), 2); + assert_eq!(with(31).unwrap_err(), "the rebuild needs more than the decoded-element allowance left"); +} + +/// The triangles reserved up front fit the limit too. +#[test] +fn the_walk_reserves_within_its_limit() { + assert!(Walk::new(1 << 20, 10).tris.capacity() < 1 << 20); + assert!(Walk::new(1 << 20, 10).tris.capacity() <= 5); + assert!(Walk::new(4, usize::MAX).tris.capacity() >= 4); +} + +/// Bit 0 alone continues across `[A C]`, keeping `W = B`. +#[test] +fn the_right_edge_is_entered_from_a_to_c() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, -2, 0]; + let m = run(&pts, &[1, 0, 0, 0, 0, 0], 2, &[false; 4], &[]).unwrap(); + assert_eq!(m.triangles[1], [0, 2, 3]); +} + +/// A reference slot reuses a vertex; a closed pushed edge is skipped and the next component +/// starts from a fresh seed. +#[test] +fn references_and_components() { + let pts = [0; 9 + 9]; + let is_ref = [false, false, false, true, false, false, false]; + let m = run(&pts, &[3, 0, 0, 0, 0, 0, 0, 0, 0], 3, &is_ref, &[0]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 0], [3, 4, 5]]); +} + +/// A double-sided panel: four coplanar vertices, four triangles. The second triangle's apex +/// folds back onto the seed's side, so its own continuation takes `[A C]` first; the last two +/// apexes are the seed's first two vertices. +#[test] +fn a_folded_apex_swaps_its_continuation() { + let is_ref = [false, false, false, false, true, true]; + let status = [3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let m = run(&[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], &status, 4, &is_ref, &[0, 1]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [2, 3, 0], [0, 3, 1]]); + // The same panel unfolded is rescued by the component retry. + let m = run(&[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, -3, 0], &status, 4, &is_ref, &[0, 1]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [2, 3, 0], [0, 3, 1]]); +} + +/// The fourth apex `(1, 3, 0)` lies in its parent's plane on the parent's side, so its +/// triangle folds and continues across `[A C]` to the reference. +#[test] +fn a_fold_past_the_retry_point_is_detected() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, -3, 1, 1, 3, 0]; + let is_ref = [false, false, false, false, false, true]; + let m = run(&pts, &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], 4, &is_ref, &[0]).unwrap(); + assert_eq!(m.triangles[3], [3, 4, 0]); +} + +/// A panel whose fold apex is a reference carries no fold signal: the unfolded walk closes an +/// edge it still needs, so the component is rewound and walked again folded at its second +/// triangle. +#[test] +fn a_reference_fold_is_found_by_retrying_the_component() { + let pts: Vec = (1..=30).collect(); + let mut is_ref = vec![false; 9]; + is_ref.extend([true, true, false, true, true, true]); + let mut status = vec![0; 21]; + status[3..7].copy_from_slice(&[3, 3, 2, 0]); + let m = run(&pts, &status, 7, &is_ref, &[0, 3, 6, 0, 3]).unwrap(); + assert_eq!(m.triangles[3..], [[0, 3, 9], [9, 3, 6], [9, 6, 0], [0, 6, 3]]); +} + +/// A tetrahedron closed by two references: after `[3 1 0]` the continuation edge `[0 1]` is +/// already shared twice, so the walk takes the open `[3 0]` instead. +#[test] +fn a_closed_continuation_edge_gives_way_to_the_open_one() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, 0, 4]; + let mut status = [0; 12]; + status[..3].copy_from_slice(&[2, 2, 2]); + let mut is_ref = [false; 6]; + is_ref[4..].copy_from_slice(&[true, true]); + let m = run(&pts, &status, 4, &is_ref, &[0, 2]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [3, 1, 0], [3, 0, 2]]); +} + +#[test] +fn arrays_that_do_not_fit_are_refused() { + let pts = [0; 9]; + let ok = [false; 3]; + assert!(run(&pts, &[0, 0, 0], 1, &ok, &[]).is_ok()); + assert!(run(&pts, &[0], 1, &ok, &[]).is_err(), "T form"); + assert!(run(&[0; 12], &[0, 0, 0], 1, &ok, &[]).is_err(), "points left"); + assert!(run(&[0; 6], &[0, 0, 0], 1, &ok, &[]).is_err(), "points short"); + assert!(run(&pts, &[0, 0, 0], 1, &[false; 4], &[]).is_err(), "slots left"); + assert!(run(&[0; 6], &[0, 0, 0], 1, &[false, false, true], &[7]).is_err(), "forward reference"); + assert!(run(&[0; 9], &[0, 0, 0], 1, &[false, false, false, true], &[0]).is_err(), "references left"); + // A third triangle on the seed's edge [0 1], each apex a reference. + let three = [false, false, false, true, true]; + let shared = run(&[0; 9], &[2, 2, 0, 0, 0, 0, 0, 0, 0], 3, &three, &[0, 0]).map(|_| ()); + assert_eq!(shared, Err("an edge is shared by more than two triangles")); +} + +/// `PrcPt::Length` rounds the squared length to `f32` and takes Newton's root from a halved +/// exponent, which can settle an ulp off the correctly rounded root; `Unitize` refuses +/// lengths below `FLT_EPSILON` [WD 12.3]. +#[test] +fn length_and_unitize_follow_the_wd_pseudocode() { + assert_eq!(wd_length([3.0, 4.0, 0.0]), Some(5.0)); + let newton = f64::from_bits(S.to_bits() - 1); + assert_eq!(wd_length([1.0, 1.0, 0.0]), Some(newton)); + let long = 1.0 + 2f64.powi(-30); + assert_eq!(wd_length([long, 0.0, 0.0]), Some(1.0)); + assert_eq!(unitize([0.0, long, 0.0]), Some([0.0, long, 0.0])); + assert_eq!(unitize([1e-8, 0.0, 0.0]), None); + assert_eq!(unitize([0.0; 3]), None); +} + +/// A `W` on the edge's line leaves `Z` null, so the frame comes from `MakeOrthoRep(X)`: `Z` +/// from `X x (0, 1, 0)`, or from `X x (1, 0, 0)` when `X` is along `Y` [WD 7.8.9.2, 12.3]. +#[test] +fn a_null_z_takes_the_ortho_rep_of_x() { + let d = [1.0, 2.0, 3.0]; + assert_eq!(apex([0.0; 3], [-2.0, 0.0, 0.0], [5.0, 0.0, 0.0], d), [0.0, -2.0, -3.0]); + assert_eq!(apex([0.0; 3], [0.0, -2.0, 0.0], [0.0, 5.0, 0.0], d), [-2.0, 0.0, 3.0]); +} + +/// A degenerate frame is turned a half-turn about `X` when the sub-epsilon residue of +/// `(W - O) x X` points against `MakeOrthoRep`'s `Z`, and kept when it points along it. +#[test] +fn a_degenerate_frame_follows_its_residue() { + let (lo, hi, d) = ([0.0; 3], [-2.0, 0.0, 0.0], [1.0, 2.0, 3.0]); + assert_eq!(apex(lo, hi, [5.0, 1e-9, 0.0], d), [0.0, 2.0, 3.0]); + assert_eq!(apex(lo, hi, [5.0, -1e-9, 0.0], d), [0.0, -2.0, -3.0]); +} + +/// A degenerate frame's `Y` comes from the frame's own `X`, whose length `unitize` left off by +/// a few 1e-8, not from the copy `MakeOrthoRep` unitized again: the two differ by that much, +/// and the walk would carry the difference into later apexes. +#[test] +fn a_degenerate_frame_takes_y_from_its_own_x() { + let (lo, hi) = ([0.1, 0.2, 0.3], [0.7, -0.4, 0.9]); + let o = scale(add(lo, hi), 0.5); + let x = unitize(sub(lo, hi)).unwrap(); + let w = add(o, scale(x, 0.37)); + let d = [1.0, 2.0, 3.0]; + let z = make_ortho_rep_z(x).unwrap(); + let p = apex(lo, hi, w, d); + let own = unitize(cross(z, x)).unwrap(); + let again = unitize(cross(z, unitize(x).unwrap())).unwrap(); + assert_ne!(own, again, "the probe must tell the two Ys apart"); + let sign = if dot(cross(sub(w, o), x), z) < 0.0 { -1.0 } else { 1.0 }; + let expect = |y: V| sub(sub(add(o, scale(x, d[0])), scale(y, sign * d[1])), scale(z, sign * d[2])); + assert_eq!(p, expect(own)); + assert_ne!(p, expect(again)); +} + +/// A strip of thin, non-planar triangles compressed by an encoder written out from the WD: +/// each point is expressed in the frame of the working mesh, rounded to the tolerance and +/// reinjected [WD 7.8.9, 7.8.9.2], with `PrcPt::Length` and `Unitize` transcribed from the +/// pseudocode [WD 12.3] as read. The decoder must land exactly on that working mesh, which +/// keeps every vertex within the rounding bound of its source. +#[test] +fn a_strip_decodes_onto_the_encoders_working_mesh() { + fn length(v: V) -> f64 { + let f = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) as f32; + let mut bits = f.to_bits(); + let exponent = (bits >> 23) & 0xff; + if exponent > 127 { + bits = (bits & 0x807f_ffff) | (((exponent - 127) / 2 + 127) << 23); + } + let (squared, mut x0) = (f64::from(f), f64::from(f32::from_bits(bits))); + for _ in 0..100 { + let xi = 0.5 * (x0 + squared / x0); + if xi == x0 { + return xi; + } + x0 = xi; + } + -1.0 + } + fn unitized(v: V) -> V { + let l = length(v); + assert!(l >= f64::from(f32::EPSILON), "null axis"); + [v[0] / l, v[1] / l, v[2] / l] + } + let (tol, n) = (1e-4, 48); + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + let mut jitter = |scale: f64| { + state = state.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + ((state >> 11) as f64 / (1u64 << 53) as f64 - 0.5) * scale + }; + // Two rails 4 apart, 0.05 between a rail's points, 0.02 of relief. + let source: Vec = (0..n) + .map(|i| { + let (along, rail) = (i as f64 * 0.025, (i % 2) as f64 * 4.0); + [along + jitter(0.01), rail + jitter(0.01), jitter(0.02)] + }) + .collect(); + let round = |v: f64| (v / tol).round() as i64; + let (mut points, mut work) = (Vec::new(), Vec::::new()); + for (i, target) in source.iter().take(3).enumerate() { + let base = match i { + 0 => [0.0; 3], + 1 => work[0], + _ => scale(add(work[0], work[1]), 0.5), + }; + let d = [0, 1, 2].map(|c| round(target[c] - base[c])); + points.extend(d); + work.push(add(base, d.map(|c| c as f64 * tol))); + } + // Triangle k enters across vertices k and k + 1, opposite k - 1, and continues right on + // odd k, left on even k, unless it folds. + let mut status = vec![0; 3 * (n - 2)]; + status[0] = 2; + for k in 1..n - 2 { + let (lo, hi, w) = (work[k], work[k + 1], work[k - 1]); + let o = scale(add(hi, lo), 0.5); + let x = unitized(sub(lo, hi)); + let z = unitized(cross(sub(w, o), x)); + let y = unitized(cross(z, x)); + let r = sub(source[k + 2], o); + let d = [round(dot(r, x)), round(-dot(r, y)), round(-dot(r, z))]; + points.extend(d); + let [dx, dy, dz] = d.map(|c| c as f64 * tol); + work.push(sub(sub(add(o, scale(x, dx)), scale(y, dy)), scale(z, dz))); + if k < n - 3 { + let fold = d[2] == 0 && d[1] > 0; + status[k] = if (k % 2 == 1) != fold { 1 } else { 2 }; + } + } + let m = rebuild(&Arrays { + tolerance: tol, + origin: [0.0; 3], + points: &points, + edge_status: &status, + triangles: n - 2, + is_reference: &vec![false; n], + references: &[], + limit: usize::MAX, + }) + .unwrap(); + assert_eq!(m.triangles[1..4], [[2, 1, 3], [2, 3, 4], [4, 3, 5]]); + assert_eq!(m.positions, work); + for (p, s) in m.positions.iter().zip(&source) { + let off = dot(sub(*p, *s), sub(*p, *s)).sqrt() / tol; + assert!(off < 0.87, "{off} tolerances off"); + } +} diff --git a/crates/prc/src/tess/tests.rs b/crates/prc/src/tess/tests.rs index 26219f2865..253e1c9d5e 100644 --- a/crates/prc/src/tess/tests.rs +++ b/crates/prc/src/tess/tests.rs @@ -529,12 +529,281 @@ fn a_compressed_mesh_is_read_to_its_end() { let schema = Schema::default(); let mut ctx = Ctx::new(BitReader::new(&bytes), &schema, v); let t = ctx.file_structure_tessellation().unwrap(); - assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }, Tessellation::Mesh(_)]), "full={full} v={v}: {t:?}"); assert_eq!(ctx.r.position(), w.len(), "full={full} v={v}"); + assert!(matches!(t[1], Tessellation::Mesh(_)), "full={full} v={v}: {t:?}"); + // Stored normals and the one-status-per-triangle form are not rebuilt; the + // recalculated 3T seed is. + if full || !three_t { + assert!(matches!(t[0], Tessellation::Compressed { triangles: 1, .. }), "full={full} v={v}: {t:?}"); + } else { + let m = mesh(&t[0]); + assert_eq!((m.triangles.as_slice(), m.faces.as_slice()), ([[0, 1, 2]].as_slice(), std::slice::from_ref(&(0..1))), "v={v}"); + assert!(m.normals_recalculated && m.normals.is_empty()); + assert_eq!(m.triangle_styles, [TriangleStyle { style: 1, behaviour: 0 }]); + // Each point component is 1, in units of the 0.001 tolerance, from the origin 0. + let want = [0.001, 0.002, 0.0025]; + assert!(m.positions.iter().zip(want).all(|(p, c)| p.iter().all(|x| (x - c).abs() < 1e-12)), "{:?}", m.positions); + } } } } +/// A `TESS_3D_Compressed` (authoring version 8137, origin 0) whose normals are recalculated, +/// with `points` in units of `tolerance`, the 3T `status`, each triangle's face, the slots' +/// reference flags, the references and one line attribute per face; every other optional +/// branch absent. +struct Packed<'a> { + tolerance: f64, + points: &'a [i64], + status: &'a [u8], + face_of: &'a [u32], + /// The width of each face index; 8 unless an index needs more. + face_bits: u32, + is_ref: &'a [bool], + refs: &'a [u32], + line_attributes: &'a [u16], +} + +/// A `CompressedIndiceArray` [WD 11.9] of `bits`-wide deltas, `implicit` with no compressed +/// flag. +fn indices(w: &mut W, v: &[u32], implicit: bool, bits: u32) { + if !implicit { + w.bit(false); + } + w.uint(u32::try_from(v.len()).unwrap()); + for i in 0..v.len() { + w.put(if i == 0 { u64::from(bits) } else { 0 }, 8); + } + let mut prev = 0i64; + for &x in v { + let d = i64::from(x) - prev; + w.bit(d < 0).put(d.unsigned_abs(), bits - 1); + prev = i64::from(x); + } +} + +fn packed(w: &mut W, c: &Packed<'_>) { + w.uint(173).bit(false).bit(true).double(c.tolerance); + w.put(0, 32).put(0, 32).put(0, 32); + // point_array: 8-bit sign-magnitude integers. + w.bit(false).uint(u32::try_from(c.points.len()).unwrap()); + for _ in c.points { + w.put(8, 8); + } + for &p in c.points { + w.bit(p < 0).put(p.unsigned_abs(), 7); + } + w.bit(false).uint(u32::try_from(c.status.len()).unwrap()); + for &e in c.status { + w.put(u64::from(e), 8); + } + indices(w, c.face_of, false, c.face_bits); + w.uint(u32::try_from(c.is_ref.len()).unwrap()); + for &b in c.is_ref { + w.bit(b); + } + assert!(c.refs.len() <= 3, "more than three references are Huffman-coded"); + indices(w, c.refs, true, 8); + // must_recalculate_normals, normal_is_reversed, crease angle, flags; no colours, no + // multiple line attributes. + w.bit(true); + for _ in c.face_of { + w.bit(false); + } + w.double(0.5).put(0, 8).bit(false).bit(false); + w.bit(false).uint(u32::try_from(c.line_attributes.len()).unwrap()); + for &a in c.line_attributes { + w.put(u64::from(a & 0xff), 8).put(u64::from(a >> 8), 8); + } + // No texture, no behaviours. + w.bit(true).bit(false); +} + +/// A double-sided panel of two faces packed in walk order, its fold apex and its last two +/// apexes references: rebuilt into a mesh grouped by face, each triangle styled by its face. +#[test] +fn a_recalculated_compressed_mesh_is_rebuilt_by_face() { + let mut b = W::default(); + packed( + &mut b, + &Packed { + tolerance: 0.5, + points: &[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], + status: &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], + face_of: &[1, 0, 1, 0], + face_bits: 8, + is_ref: &[false, false, false, false, true, true], + refs: &[0, 1], + line_attributes: &[3, 0], + }, + ); + let w = section(1, &b); + let bytes = w.bytes(); + let schema = Schema::default(); + let mut ctx = Ctx::new(BitReader::new(&bytes), &schema, 8137); + let t = ctx.file_structure_tessellation().unwrap(); + assert_eq!(ctx.r.position(), w.len()); + let m = mesh(&t[0]); + // Walk order [0 1 2] [2 1 3] [2 3 0] [0 3 1]; faces 1, 0, 1, 0. + assert_eq!(m.triangles, [[2, 1, 3], [0, 3, 1], [0, 1, 2], [2, 3, 0]]); + assert_eq!(m.faces, [0..2, 2..4]); + assert_eq!(m.positions[..3], [[0.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0]]); + let style = |style| TriangleStyle { style, behaviour: 0 }; + assert_eq!(m.triangle_styles, [style(3), style(3), style(0), style(0)]); + assert!(m.normals_recalculated); +} + +/// Arrays the walk cannot fit stay a [`Tessellation::Compressed`], read to their end. +#[test] +fn a_compressed_mesh_that_does_not_fit_is_left_compressed() { + let mut b = W::default(); + // One point too many for a lone seed. + let c = Packed { + tolerance: 0.5, + points: &[0; 12], + status: &[0, 0, 0], + face_of: &[0], + face_bits: 8, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[0], + }; + packed(&mut b, &c); + let w = section(1, &b); + let t = decode(&w, &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }]), "{t:?}"); +} + +/// A face index near `u32::MAX` claims four billion faces: once the triangles rebuild, they are +/// charged before any face range is made, so the decode stops at the element allowance instead +/// of allocating them. +#[test] +fn a_huge_face_index_is_charged_not_allocated() { + let mut b = W::default(); + let c = Packed { + tolerance: 0.5, + points: &[0; 9], + status: &[0, 0, 0], + face_of: &[u32::MAX - 1], + face_bits: 33, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[], + }; + packed(&mut b, &c); + let r = decode(§ion(1, &b), &Schema::default(), 8137); + assert!(matches!(r, Err(PrcError::TooManyElements { .. })), "{r:?}"); +} + +/// The same huge face index on arrays the walk cannot fit charges nothing for triangles or +/// faces: the tessellation stays compressed and the file reads on. +#[test] +fn arrays_that_do_not_fit_spend_no_triangles_or_faces() { + let mut b = W::default(); + let c = Packed { + tolerance: 0.5, + points: &[0; 12], + status: &[0, 0, 0], + face_of: &[u32::MAX - 1], + face_bits: 33, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[], + }; + packed(&mut b, &c); + let t = decode(§ion(1, &b), &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }]), "{t:?}"); +} + +/// The two-face panel of [`a_recalculated_compressed_mesh_is_rebuilt_by_face`], its second +/// reference `last_ref`. +fn panel(last_ref: u32) -> W { + let mut b = W::default(); + packed( + &mut b, + &Packed { + tolerance: 0.5, + points: &[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], + status: &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], + face_of: &[1, 0, 1, 0], + face_bits: 8, + is_ref: &[false, false, false, false, true, true], + refs: &[0, last_ref], + line_attributes: &[3, 0], + }, + ); + section(1, &b) +} + +/// A rebuilt mesh is charged its triangles, each with its style, and each face range at its +/// size; the same arrays left compressed (a reference to a vertex not yet made) are charged +/// neither. +#[test] +fn a_rebuilt_compressed_mesh_is_charged_its_triangles_and_faces() { + let fits = panel(1); + let t = decode(&fits, &Schema::default(), 8137).unwrap(); + assert_eq!(mesh(&t[0]).triangles.len(), 4); + let left = panel(9); + let t = decode(&left, &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 4, .. }]), "{t:?}"); + assert_eq!(spent(&fits) - spent(&left), 4 * triangle_elements(false, 0, true) + 2 * RANGE_ELEMENTS); + assert_eq!(triangle_elements(false, 0, true), 3); + #[cfg(target_pointer_width = "64")] + assert_eq!(RANGE_ELEMENTS, 2); +} + +/// A rebuild whose walk would outgrow the allowance left keeps the arrays compressed, even +/// when the rebuilt mesh alone would have fit, and the stream reads on. +#[test] +fn a_compressed_rebuild_is_bounded_by_the_allowance_left() { + let fits = panel(1); + let bytes = fits.bytes(); + let schema = Schema::default(); + let read = |n| Ctx::new(BitReader::with_elements(&bytes, n), &schema, 8137).file_structure_tessellation(); + let arrays = spent(&panel(9)); + let mesh_alone = spent(&fits) - arrays; + let t = read(arrays + mesh_alone).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 4, .. }]), "{t:?}"); + assert!(matches!(read(crate::MAX_DECODED_ELEMENTS).unwrap()[..], [Tessellation::Mesh(_)])); +} + +/// A face's nonzero behaviour styles its triangles even when its line attribute is 0 (the +/// owner's graphics); all zero, the faces give no styles. +#[test] +fn a_compressed_face_behaviour_alone_styles_its_triangles() { + let attributes = |behaviours: Vec| FaceAttributes { multi: Vec::new(), line_attributes: vec![0], behaviours }; + let a = attributes(vec![0x30]); + assert!(a.styled(1)); + assert_eq!(a.styles(&[0, 0], 1), [TriangleStyle { style: 0, behaviour: 0x30 }; 2]); + assert!(!attributes(vec![0]).styled(1)); + assert!(!attributes(Vec::new()).styled(1)); +} + +fn rebuilt(triangles: usize) -> compressed::Rebuilt { + compressed::Rebuilt { positions: vec![[0.0; 3]; 3], triangles: (0..triangles).map(|i| [0, 1, u32::try_from(i).unwrap()]).collect() } +} + +/// Triangles are grouped by face in walk order within each, faces no triangle names (between +/// and after) get empty ranges, and each style moves with its triangle. +#[test] +fn rebuilt_triangles_are_grouped_by_face_with_their_styles() { + let face_of = [2, 0, 2, 1, 0]; + let m = by_face(rebuilt(5), &face_of, 5, &[]); + let apex: Vec = m.triangles.iter().map(|t| t[2]).collect(); + assert_eq!(apex, [1, 4, 3, 0, 2]); + assert_eq!(m.faces, [0..2, 2..3, 3..5, 5..5, 5..5]); + assert!(m.triangle_styles.is_empty()); + assert!(m.normals_recalculated); + + let style = |style| TriangleStyle { style, behaviour: u16::try_from(style).unwrap() }; + let styles: Vec = face_of.iter().map(|&f| style(f + 1)).collect(); + let m = by_face(rebuilt(5), &[2, 0, 2, 3, 0], 4, &styles); + let apex: Vec = m.triangles.iter().map(|t| t[2]).collect(); + assert_eq!(apex, [1, 4, 0, 2, 3]); + assert_eq!(m.faces, [0..2, 2..2, 2..4, 4..5]); + assert_eq!(m.triangle_styles, [style(1), style(1), style(3), style(3), style(2)]); +} + #[test] fn refusals_and_damage_are_errors() { // Edge statuses neither T nor 3T long. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index c721ce18e9..e533ecf6c7 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -10062,8 +10062,8 @@ milestone = "M12" status = "partial" commands = [] tools = ["model_render"] -evidence = ["crates/automation/tests/it/model_render.rs::a_prc_model_is_drawn_over_the_background", "crates/automation/tests/it/model_render.rs::an_orbit_frames_the_model_from_a_named_view_and_its_moves", "crates/prc-render/src/nav/orbit_tests.rs::a_pressed_point_off_centre_keeps_its_place_as_the_view_turns", "crates/annot/src/model3d/tests.rs::a_3dref_leads_to_the_stream_it_names", "crates/prc-render/tests/model.rs::a_synthetic_prc_renders_its_nearer_triangle_over_the_farther"] -notes = "headless only: model_render draws a page's PRC model (through a /3DRef too) to a PNG, framed from a chosen direction, or by orbit navigation (named views about a chosen up axis, turn, zoom about a picture point, pan) from prc-render's GUI-free nav module, with headlight shading. Missing: the 3D view in the app (no activation, no on-screen navigation; nav is not wired to a viewer yet), walk and fly navigation, U3D (an error), the annotation's /3DV views, lighting schemes and /RM render modes, materials and textures, compressed tessellation, picking, cross sections, the model tree, and measurements" +evidence = ["crates/automation/tests/it/model_render.rs::a_prc_model_is_drawn_over_the_background", "crates/automation/tests/it/model_render.rs::an_orbit_frames_the_model_from_a_named_view_and_its_moves", "crates/prc-render/src/nav/orbit_tests.rs::a_pressed_point_off_centre_keeps_its_place_as_the_view_turns", "crates/annot/src/model3d/tests.rs::a_3dref_leads_to_the_stream_it_names", "crates/prc-render/tests/model.rs::a_synthetic_prc_renders_its_nearer_triangle_over_the_farther", "crates/prc-render/tests/model.rs::a_compressed_tessellation_is_rebuilt_and_rendered", "crates/automation/tests/it/model_render.rs::a_compressed_prc_model_is_rebuilt_and_drawn"] +notes = "headless only: model_render draws a page's PRC model (through a /3DRef too) to a PNG, framed from a chosen direction, or by orbit navigation (named views about a chosen up axis, turn, zoom about a picture point, pan) from prc-render's GUI-free nav module, with headlight shading. Missing: the 3D view in the app (no activation, no on-screen navigation; nav is not wired to a viewer yet), walk and fly navigation, U3D (an error), the annotation's /3DV views, lighting schemes and /RM render modes, materials and textures, compressed tessellations that store their normals (those whose normals are recalculated are rebuilt), picking, cross sections, the model tree, and measurements" [[feature]] id = "misc.3d-add" From 120c62fb82f41998ecdf6dbbfe39f27d4c9a410e Mon Sep 17 00:00:00 2001 From: Ken Mantle Date: Thu, 12 Nov 2026 11:58:31 -0500 Subject: [PATCH 46/46] Auto-tag Apply tags headings by size and weight (#7062) (#7069) * feat(a11y): auto-tag Apply tags headings by size and weight Apply now chooses H1-H6 or P per text object instead of P for all. The body style is the half-point size showing the most characters (bold when most of those are bold). A short object (at most 3 lines, 200 characters, with a letter) is a heading when it is 1.15x the body, or bold at 0.9x the body over a regular body in at most 2 lines. Heading styles rank across the document, larger and bolder first, capped at H6, and a level never skips past the one before it so heading-nesting passes. Size is Tf x Tm x CTM with q/Q tracked; bold is Metrics.bold. Text in render mode 3 (an OCR layer) is not counted. Font metrics are measured once per font object per document, at most 256 fonts. AutoTagResult gains a headings count. Parity, gaps, roadmap and the apply tool's note say headings are tagged. Fixes #7062. Co-Authored-By: Claude Opus 5.5 * fix(a11y): rank heading styles by map and skip clip-only text Look up each heading style's rank in a map built once after sorting, so ranking is linear in the number of text objects. Text in render mode 7 (clip only) paints nothing, so it is excluded from heading evidence like mode 3. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: KenM76 Co-authored-by: Claude Opus 5.5 --- crates/a11y/src/autotag.rs | 66 +++- crates/a11y/src/autotag_roles.rs | 382 +++++++++++++++++++++++ crates/a11y/src/lib.rs | 1 + crates/a11y/src/tests.rs | 14 +- crates/automation/src/a11y.rs | 2 +- crates/automation/tests/it/automation.rs | 25 ++ docs/gaps.md | 2 +- docs/roadmap.md | 2 +- parity/acrobat-features.toml | 4 +- 9 files changed, 472 insertions(+), 26 deletions(-) create mode 100644 crates/a11y/src/autotag_roles.rs diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 59d30930ff..bf3df8d0a4 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -1,7 +1,8 @@ //! Create a valid, reviewable structure tree for untagged PDFs. //! -//! Each text object becomes a paragraph, which can then be retagged with the existing -//! tag-property tool after human review. Painted paths, shadings and images outside text objects +//! Each text object becomes a paragraph, or a heading when its size or weight sets it apart from +//! the body text ([`crate::autotag_roles`]); either can be retagged with the existing tag-property +//! tool after human review. Painted paths, shadings and images outside text objects //! are wrapped in `/Artifact BMC … EMC` so the resulting page passes the tagged-content check. use std::collections::{HashMap, HashSet}; @@ -9,6 +10,8 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::{Op, Pieces, serialize_ops}; use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; +use crate::autotag_roles::{self, Evidence}; + const MAX_PAGES: usize = 1_000; const MAX_PAGE_BYTES: usize = 16 * 1024 * 1024; const MAX_OPERATORS: usize = 100_000; @@ -22,6 +25,7 @@ const MAX_TEXT_OBJECTS: usize = 4_096; pub struct AutoTagResult { pub pages: usize, pub paragraphs: usize, + pub headings: usize, } #[derive(Debug, thiserror::Error, PartialEq)] @@ -44,6 +48,7 @@ struct Plan { page: ObjRef, ops: Vec, spans: Vec<(usize, usize)>, + evidence: Vec, artifacts: HashMap, } @@ -98,6 +103,14 @@ fn page_xobjects(doc: &Document, page: &Dict) -> Dict { .unwrap_or_default() } +fn page_fonts(doc: &Document, page: &Dict) -> Dict { + page.get(b"Resources") + .map(|r| doc.resolve(r)) + .and_then(|r| r.as_dict().and_then(|r| r.get(b"Font").map(|x| doc.resolve(x)))) + .and_then(|x| x.as_dict().cloned()) + .unwrap_or_default() +} + fn harmless_outside(op: &[u8]) -> bool { matches!( op, @@ -183,7 +196,13 @@ fn harmless_inside(op: &[u8]) -> bool { /// `totals` is the decoded bytes and operators of the pages planned so far, bounded by /// `MAX_TOTAL_BYTES` and `MAX_TOTAL_OPERATORS`. -fn plan_page(doc: &Document, page: &pdfcraft_model::Page, index: usize, totals: &mut (usize, usize)) -> Result { +fn plan_page( + doc: &Document, + page: &pdfcraft_model::Page, + index: usize, + totals: &mut (usize, usize), + fonts: &mut autotag_roles::Fonts, +) -> Result { if page.dict.contains(b"StructParents") { return Err(unsupported(index, "page already has a structure parent key")); } @@ -248,7 +267,9 @@ fn plan_page(doc: &Document, page: &pdfcraft_model::Page, index: usize, totals: if path.is_some() { return Err(unsupported(index, "unterminated path object")); } - Ok(Plan { page: page.obj, ops: parsed.ops, spans, artifacts }) + let resources = page_fonts(doc, &page.dict); + let evidence = autotag_roles::evidence(doc, fonts, &resources, &parsed.ops, &spans); + Ok(Plan { page: page.obj, ops: parsed.ops, spans, evidence, artifacts }) } fn plan(doc: &Document) -> Result, AutoTagError> { @@ -268,8 +289,9 @@ fn plan(doc: &Document) -> Result, AutoTagError> { let mut out = Vec::with_capacity(pages.len()); let mut total = 0usize; let mut totals = (0usize, 0usize); + let mut fonts = autotag_roles::Fonts::default(); for (index, page) in pages.iter().enumerate() { - let planned = plan_page(doc, page, index, &mut totals)?; + let planned = plan_page(doc, page, index, &mut totals, &mut fonts)?; total = total.saturating_add(planned.spans.len()); out.push(planned); } @@ -279,16 +301,21 @@ fn plan(doc: &Document) -> Result, AutoTagError> { Ok(out) } -fn mark_ops(ops: Vec, starts: &HashMap, ends: &HashSet, artifacts: &HashMap) -> Result, AutoTagError> { +fn mark_ops( + ops: Vec, + starts: &HashMap, + ends: &HashSet, + artifacts: &HashMap, +) -> Result, AutoTagError> { let artifact_ends: HashSet = artifacts.values().copied().collect(); let extra = starts.len().saturating_add(artifacts.len()).saturating_mul(2); let mut marked = Vec::with_capacity(ops.len().saturating_add(extra)); for (at, op) in ops.into_iter().enumerate() { let is_artifact = artifacts.contains_key(&at); - if let Some(mcid) = starts.get(&at) { + if let Some(&(mcid, role)) = starts.get(&at) { let mut properties = Dict::new(); - properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(*mcid).map_err(|_| AutoTagError::TooManyPages)?)); - marked.push(Op::new("BDC", vec![Object::name("P"), Object::Dict(properties)])); + properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); + marked.push(Op::new("BDC", vec![Object::name(role), Object::Dict(properties)])); } else if is_artifact { marked.push(Op::new("BMC", vec![Object::name("Artifact")])); } @@ -334,16 +361,18 @@ fn write_structure( Ok(()) } -/// Tag every text object in an untagged PDF as a paragraph and mark painted paths, shadings and +/// Tag every text object in an untagged PDF as a paragraph or a heading and mark painted paths, shadings and /// images outside text objects as artifacts. The full structure tree, parent tree and /// marked-content IDs are written together. Unsupported operators are refused before any -/// mutation. On success the caller can use tag-property edits to assign reviewed heading roles. +/// mutation. On success the caller can use tag-property edits to correct the roles after review. pub fn autotag_text(doc: &mut Document) -> Result { let plans = plan(doc)?; + let all: Vec = plans.iter().flat_map(|p| p.evidence.iter().copied()).collect(); + let mut roles = autotag_roles::roles(&all).into_iter(); let mut next = doc.clone(); let catalog = next.root().ok_or(AutoTagError::NoCatalog)?; let (root, document) = (next.add(Object::Null), next.add(Object::Null)); - let (mut children, mut numbers, mut tagged_pages, mut paragraphs) = (Vec::new(), Vec::new(), 0usize, 0usize); + let (mut children, mut numbers, mut tagged_pages, mut paragraphs, mut headings) = (Vec::new(), Vec::new(), 0usize, 0usize, 0usize); for page in plans { if page.spans.is_empty() && page.artifacts.is_empty() { continue; @@ -353,18 +382,23 @@ pub fn autotag_text(doc: &mut Document) -> Result { let key = i64::try_from(tagged_pages).map_err(|_| AutoTagError::TooManyPages)?; let mut parents = Vec::with_capacity(page.spans.len()); for (mcid, (first, last)) in page.spans.iter().enumerate() { + let role = roles.next().unwrap_or("P"); let mut element = Dict::new(); element.set(b"Type".to_vec(), Object::name("StructElem")); - element.set(b"S".to_vec(), Object::name("P")); + element.set(b"S".to_vec(), Object::name(role)); element.set(b"P".to_vec(), Object::Ref(document)); element.set(b"Pg".to_vec(), Object::Ref(page.page)); element.set(b"K".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); let reference = next.add(element); parents.push(Object::Ref(reference)); children.push(Object::Ref(reference)); - starts.insert(*first, mcid); + starts.insert(*first, (mcid, role)); ends.insert(*last); - paragraphs = paragraphs.saturating_add(1); + if role == "P" { + paragraphs = paragraphs.saturating_add(1); + } else { + headings = headings.saturating_add(1); + } } next.update_dict(page.page, |d| d.set(b"StructParents".to_vec(), Object::Int(key)))?; numbers.extend([Object::Int(key), Object::Array(parents)]); @@ -376,5 +410,5 @@ pub fn autotag_text(doc: &mut Document) -> Result { } write_structure(&mut next, catalog, root, document, children, numbers, tagged_pages)?; *doc = next; - Ok(AutoTagResult { pages: tagged_pages, paragraphs }) + Ok(AutoTagResult { pages: tagged_pages, paragraphs, headings }) } diff --git a/crates/a11y/src/autotag_roles.rs b/crates/a11y/src/autotag_roles.rs new file mode 100644 index 0000000000..a72e14a417 --- /dev/null +++ b/crates/a11y/src/autotag_roles.rs @@ -0,0 +1,382 @@ +//! Heading roles for auto-tagged text objects, from the size and weight of the text each shows. +//! +//! The body style is the size (to the half point) carrying the most characters in the document, +//! and whether most of those characters are bold. A text object is a heading when it shows at most +//! three lines and 200 characters, including a letter, and is either 15% larger than the body or, +//! where the body is not bold, bold at no less than 90% of the body size on at most two lines. +//! Levels rank the heading styles (size, then bold) over the whole document, largest first, capped +//! at 6, and never skip a level after the heading before (the `heading-nesting` check). + +use std::collections::HashMap; +use std::rc::Rc; + +use pdfcraft_content::{Matrix, Op}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object}; +use pdfcraft_fonts::pdf::Metrics; + +/// A line this much larger than body text reads as a heading. +const HEADING_SIZE_RATIO: f64 = 1.15; +/// Bold text this much of the body size or more can be a heading by weight. +const WEIGHT_SIZE_RATIO: f64 = 0.9; +const MAX_LINES: usize = 3; +const MAX_WEIGHT_LINES: usize = 2; +const MAX_CHARS: usize = 200; +/// Saved graphics states followed; deeper `q` nesting keeps the current state. +const MAX_SAVED: usize = 64; +/// Distinct fonts measured per document; text in any further font counts as unstyled. +const MAX_FONTS: usize = 256; + +/// What one text object shows. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub(crate) struct Evidence { + /// The size, in user space, carrying the most of the object's characters. + pub size: f64, + /// Most of the object's characters are in a bold font. + pub bold: bool, + pub lines: usize, + /// Characters shown, whitespace excluded. + pub chars: usize, + pub letters: bool, +} + +/// The structure type for each text object, in document order: `H1`–`H6` or `P`. +pub(crate) fn roles(objects: &[Evidence]) -> Vec<&'static str> { + const HEADINGS: [&str; 6] = ["H1", "H2", "H3", "H4", "H5", "H6"]; + let Some((body, body_bold)) = body_style(objects) else { return vec!["P"; objects.len()] }; + let candidate = |e: &Evidence| { + let fits = e.lines >= 1 && e.lines <= MAX_LINES && e.chars <= MAX_CHARS && e.letters; + let by_size = e.size >= body * HEADING_SIZE_RATIO; + let by_weight = e.bold && !body_bold && e.lines <= MAX_WEIGHT_LINES && e.size >= body * WEIGHT_SIZE_RATIO; + fits && (by_size || by_weight) + }; + let mut styles: Vec<(i64, bool)> = objects.iter().filter(|e| candidate(e)).map(|e| (half_points(e.size), e.bold)).collect(); + styles.sort_by(|a, b| b.cmp(a)); + styles.dedup(); + let ranks: HashMap<(i64, bool), usize> = styles.into_iter().enumerate().map(|(rank, style)| (style, rank)).collect(); + let mut previous = 0usize; + objects + .iter() + .map(|e| { + if !candidate(e) { + return "P"; + } + let rank = ranks.get(&(half_points(e.size), e.bold)).copied().unwrap_or(0); + let level = rank.saturating_add(1).min(previous.saturating_add(1)).min(HEADINGS.len()); + previous = level; + HEADINGS.get(level.saturating_sub(1)).copied().unwrap_or("P") + }) + .collect() +} + +fn half_points(size: f64) -> i64 { + // Sizes are finite and bounded by `evidence`; `as` saturates in any case. + (size * 2.0).round() as i64 +} + +/// The size carrying the most characters, and whether most of them are bold. +fn body_style(objects: &[Evidence]) -> Option<(f64, bool)> { + let mut by_size: HashMap = HashMap::new(); + for e in objects.iter().filter(|e| e.chars > 0) { + let entry = by_size.entry(half_points(e.size)).or_default(); + entry.0 = entry.0.saturating_add(e.chars); + if e.bold { + entry.1 = entry.1.saturating_add(e.chars); + } + } + let (size, (chars, bold)) = by_size.into_iter().filter(|&(s, _)| s > 0).max_by_key(|&(s, (n, _))| (n, s))?; + Some((size as f64 / 2.0, bold.saturating_mul(2) > chars)) +} + +#[derive(Clone)] +struct State { + ctm: Matrix, + tm: Matrix, + tlm: Matrix, + size: f64, + leading: f64, + /// Text render mode 3 or 7: shown text that paints nothing, such as an OCR layer or a + /// clip-only object. + invisible: bool, + metrics: Option>, +} + +/// Font metrics for a whole document, measured once per font object and capped at +/// [`MAX_FONTS`]. +#[derive(Default)] +pub(crate) struct Fonts { + by_ref: HashMap>>, + measured: usize, +} + +impl Fonts { + fn get(&mut self, doc: &Document, font: &Object) -> Option> { + let key = match font { + Object::Ref(r) => Some(*r), + _ => None, + }; + if let Some(found) = key.and_then(|r| self.by_ref.get(&r)) { + return found.clone(); + } + if self.measured >= MAX_FONTS { + return None; + } + self.measured = self.measured.saturating_add(1); + let metrics = doc.resolve(font).as_dict().map(|d| Rc::new(Metrics::from_dict(doc, d))); + if let Some(r) = key { + self.by_ref.insert(r, metrics.clone()); + } + metrics + } +} + +#[derive(Default)] +struct Shown { + by_size: HashMap, + lines: usize, + shown: bool, + moved: bool, + letters: bool, +} + +impl Shown { + fn show(&mut self, state: &State, bytes: &[u8]) { + let Some(metrics) = state.metrics.as_ref().filter(|_| !state.invisible) else { return }; + let text = metrics.decode(bytes); + let chars = text.chars().filter(|c| !c.is_whitespace()).count(); + if chars == 0 { + return; + } + self.letters |= text.chars().any(char::is_alphabetic); + if !self.shown { + (self.shown, self.lines) = (true, 1); + } else if self.moved { + self.lines = self.lines.saturating_add(1); + } + self.moved = false; + let [_, _, c, d, _, _] = state.tm.then(&state.ctm).0; + let size = (state.size * c.hypot(d)).abs(); + if !size.is_finite() || size <= 0.0 || size > 1e6 { + return; + } + let entry = self.by_size.entry(half_points(size)).or_default(); + entry.0 = entry.0.saturating_add(chars); + if metrics.bold { + entry.1 = entry.1.saturating_add(chars); + } + } + + /// A line move counts once text is shown after it, so blank lines and trailing moves do not. + fn new_line(&mut self) { + self.moved = self.shown; + } + + fn finish(self) -> Evidence { + let chars = self.by_size.values().fold(0usize, |n, v| n.saturating_add(v.0)); + let Some((size, (count, bold))) = self.by_size.into_iter().max_by_key(|&(s, (n, _))| (n, s)) else { + return Evidence::default(); + }; + Evidence { size: size as f64 / 2.0, bold: bold.saturating_mul(2) > count, lines: self.lines, chars, letters: self.letters } + } +} + +/// The evidence for each `(BT, ET)` operator span of a page, in order. `resources` is the page's +/// `/Font` resource dictionary; `fonts` is shared by the document's pages. +pub(crate) fn evidence(doc: &Document, fonts: &mut Fonts, resources: &Dict, ops: &[Op], spans: &[(usize, usize)]) -> Vec { + let mut by_name: HashMap, Option>> = HashMap::new(); + let mut state = + State { ctm: Matrix::IDENTITY, tm: Matrix::IDENTITY, tlm: Matrix::IDENTITY, size: 0.0, leading: 0.0, invisible: false, metrics: None }; + let (mut saved, mut unsaved) = (Vec::new(), 0usize); + let mut out = Vec::with_capacity(spans.len()); + let mut spans = spans.iter().peekable(); + let mut current: Option<(usize, Shown)> = None; + for (at, op) in ops.iter().enumerate() { + if current.is_none() && spans.peek().is_some_and(|s| s.0 == at) { + current = spans.next().map(|s| (s.1, Shown::default())); + } + let object = current.as_mut().map(|c| &mut c.1); + match op.op.as_slice() { + b"q" if saved.len() < MAX_SAVED => saved.push(state.clone()), + b"q" => unsaved = unsaved.saturating_add(1), + b"Q" if unsaved > 0 => unsaved -= 1, + b"Q" => { + if let Some(s) = saved.pop() { + state = s; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + state.ctm = Matrix(m).then(&state.ctm); + } + } + b"BT" => (state.tm, state.tlm) = (Matrix::IDENTITY, Matrix::IDENTITY), + b"Tf" => { + state.size = op.num(1).unwrap_or(state.size); + state.metrics = op + .name(0) + .and_then(|name| by_name.entry(name.to_vec()).or_insert_with(|| resources.get(name).and_then(|f| fonts.get(doc, f))).clone()); + } + b"TL" => state.leading = op.num(0).unwrap_or(state.leading), + b"Tr" => state.invisible = matches!(op.num(0), Some(m) if m == 3.0 || m == 7.0), + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if op.op == b"TD" { + state.leading = -y; + } + state.tlm = Matrix::translate(x, y).then(&state.tlm); + state.tm = state.tlm; + if y != 0.0 + && let Some(o) = object + { + o.new_line(); + } + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + let moved = m[5] != state.tlm.0[5]; + (state.tm, state.tlm) = (Matrix(m), Matrix(m)); + if moved && let Some(o) = object { + o.new_line(); + } + } + } + b"T*" | b"'" | b"\"" => { + state.tlm = Matrix::translate(0.0, -state.leading).then(&state.tlm); + state.tm = state.tlm; + if let Some(o) = object { + o.new_line(); + if op.op != b"T*" + && let Some(Object::String(s)) = op.operands.last() + { + o.show(&state, &s.bytes); + } + } + } + b"Tj" => { + if let (Some(o), Some(Object::String(s))) = (object, op.operands.first()) { + o.show(&state, &s.bytes); + } + } + b"TJ" => { + if let (Some(o), Some(Object::Array(items))) = (object, op.operands.first()) { + for item in items { + if let Object::String(s) = item { + o.show(&state, &s.bytes); + } + } + } + } + _ => {} + } + if current.as_ref().is_some_and(|c| c.0 == at) + && let Some((_, o)) = current.take() + { + out.push(o.finish()); + } + } + out.resize(spans.len().saturating_add(out.len()), Evidence::default()); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(size: f64, chars: usize) -> Evidence { + Evidence { size, bold: false, lines: 1, chars, letters: true } + } + + fn bold(e: Evidence) -> Evidence { + Evidence { bold: true, ..e } + } + + #[test] + fn a_line_larger_than_the_body_is_a_heading_and_body_text_is_not() { + let objects = [text(20.0, 10), text(12.0, 400), text(12.0, 300)]; + assert_eq!(roles(&objects), ["H1", "P", "P"]); + } + + #[test] + fn the_body_is_the_size_with_the_most_characters_not_the_most_objects() { + // Ten short 9 pt captions and three 12 pt body lines: the body lines hold more characters, + // so 12 pt is the body. Counting objects would make 9 pt the body and the body lines headings. + let mut objects = vec![text(12.0, 150); 3]; + objects.extend(std::iter::repeat_n(text(9.0, 5), 10)); + assert!(roles(&objects).iter().all(|r| *r == "P")); + } + + #[test] + fn bold_at_body_size_is_a_heading_only_where_the_body_is_not_bold() { + let plain_body = [bold(text(12.0, 15)), text(12.0, 500)]; + assert_eq!(roles(&plain_body), ["H1", "P"]); + let bold_body = [bold(text(12.0, 15)), bold(text(12.0, 500))]; + assert_eq!(roles(&bold_body), ["P", "P"]); + } + + #[test] + fn long_text_numbers_alone_and_four_lines_stay_paragraphs() { + let long = text(20.0, MAX_CHARS + 1); + let number = Evidence { letters: false, ..text(20.0, 2) }; + let four_lines = Evidence { lines: 4, ..text(20.0, 40) }; + let two_bold_lines = Evidence { lines: 3, ..bold(text(12.0, 40)) }; + assert_eq!(roles(&[long, number, four_lines, two_bold_lines, text(12.0, 2_000)]), ["P"; 5]); + } + + #[test] + fn levels_rank_styles_largest_first_and_never_skip_a_level() { + // Styles: 24 pt, 18 pt, bold 12 pt. The first heading met is 18 pt, ranked second, which + // would skip H1: it becomes H1, and the bold heading after it H2, not H3. + let objects = [text(18.0, 10), bold(text(12.0, 10)), text(24.0, 10), text(18.0, 10), bold(text(12.0, 10)), text(12.0, 3_000)]; + assert_eq!(roles(&objects), ["H1", "H2", "H1", "H2", "H3", "P"]); + } + + #[test] + fn levels_stop_at_six() { + let mut objects: Vec = (0..8).map(|i| text(40.0 - f64::from(i) * 2.0, 10)).collect(); + objects.push(text(12.0, 5_000)); + assert_eq!(roles(&objects), ["H1", "H2", "H3", "H4", "H5", "H6", "H6", "H6", "P"]); + } + + #[test] + fn a_document_without_text_has_only_paragraphs() { + assert_eq!(roles(&[Evidence::default(), Evidence::default()]), ["P", "P"]); + assert!(roles(&[]).is_empty()); + } + + /// The evidence for each `BT`..`ET` of `content`, drawn in Helvetica-Bold as `/F1`. + fn walk(content: &[u8]) -> Vec { + let doc = Document::new_empty(); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type1")); + font.set(b"BaseFont".to_vec(), Object::name("Helvetica-Bold")); + let mut resources = Dict::new(); + resources.set(b"F1".to_vec(), Object::Dict(font)); + let ops = pdfcraft_content::parse(content).ops; + let at = |name: &[u8]| -> Vec { ops.iter().enumerate().filter(|(_, o)| o.op == name).map(|(i, _)| i).collect() }; + let spans: Vec<(usize, usize)> = at(b"BT").into_iter().zip(at(b"ET")).collect(); + evidence(&doc, &mut Fonts::default(), &resources, &ops, &spans) + } + + #[test] + fn size_comes_from_the_font_size_the_text_matrix_and_the_ctm() { + let found = walk( + b"q 2 0 0 2 0 0 cm BT /F1 1 Tf 9 0 0 9 10 10 Tm (Title) Tj ET Q BT /F1 12 Tf 10 50 Td (one) Tj T* (two) ' 0 -14 Td (three) Tj ET BT ET", + ); + assert_eq!(found[0], Evidence { size: 18.0, bold: true, lines: 1, chars: 5, letters: true }); + assert_eq!(found[1], Evidence { size: 12.0, bold: true, lines: 3, chars: 11, letters: true }); + assert_eq!(found[2], Evidence::default()); + assert_eq!(found.len(), 3); + } + + #[test] + fn line_moves_leading_invisible_text_and_absurd_sizes() { + let found = walk( + b"BT /F1 10 Tf 1 0 0 1 0 90 Tm (a) Tj 1 0 0 1 40 90 Tm (b) Tj 1 0 0 1 0 70 Tm (c) Tj ET BT /F1 10 Tf 14 TL 0 0 Td (a) Tj 0 0 (b) \" 0 -12 TD (c) Tj T* (d) Tj ET BT /F1 10 Tf 3 Tr (hidden) Tj 7 Tr (clip) Tj 0 Tr (shown) Tj ET BT /F1 2000000 Tf (huge) Tj ET", + ); + assert_eq!(found[0].lines, 2, "a Tm on the same baseline is not a new line"); + assert_eq!(found[1].lines, 4, "\" and TD each start a line, and T* uses TD's leading"); + assert_eq!(found[2].chars, 5, "render mode 3 and 7 text is not counted"); + assert_eq!(found[3], Evidence::default(), "a size past 1e6 is dropped"); + } +} diff --git a/crates/a11y/src/lib.rs b/crates/a11y/src/lib.rs index 50e632dec8..d3492f50e7 100644 --- a/crates/a11y/src/lib.rs +++ b/crates/a11y/src/lib.rs @@ -15,6 +15,7 @@ use pdfcraft_cos::Document; mod alt; mod autotag; mod autotag_fields; +mod autotag_roles; mod content; mod headings; mod layout_table; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 890eb33e9f..4a981aaf5b 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -50,7 +50,7 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { "", ); let result = autotag_text(&mut doc).unwrap(); - assert_eq!(result, AutoTagResult { pages: 2, paragraphs: 3 }); + assert_eq!(result, AutoTagResult { pages: 2, paragraphs: 2, headings: 1 }); let catalog = doc.get(doc.root().unwrap()); let root = catalog.as_dict().unwrap().reference(b"StructTreeRoot").unwrap(); let tree = doc.get(root); @@ -62,14 +62,18 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { assert_eq!(entries[2].as_int(), Some(1)); assert_eq!(entries[1].as_array().unwrap().len(), 2); assert_eq!(entries[3].as_array().unwrap().len(), 1); - for page in pdfcraft_model::pages(&doc) { + // The 18 pt line stands out from the 12 pt body: a heading, the rest paragraphs. + let first_marks = ["/H1 <> BDC", "/P <> BDC"]; + for (page, first) in pdfcraft_model::pages(&doc).iter().zip(first_marks) { let content = page.dict.get(b"Contents").unwrap(); let stream = doc.resolve(content); let Object::Stream(stream) = &*stream else { panic!("marked page content") }; let text = String::from_utf8(stream.decoded_strict().unwrap()).unwrap(); - assert!(text.contains("/P <> BDC"), "{text}"); + assert!(text.contains(first), "{text}"); assert!(text.contains("EMC"), "{text}"); } + let kinds: Vec = tags(&doc).iter().map(|t| t.kind.clone()).collect(); + assert_eq!(kinds, ["Document", "H1", "P", "P"]); let report = check(&doc, &Options::default()); assert_eq!(status(&report, Rule::TaggedPdf), Status::Passed); assert_eq!(status(&report, Rule::TaggedContent), Status::Passed); @@ -104,7 +108,7 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { let mut predicted = pdfcraft_cos::Stream::flate(Dict::new(), &rows); predicted.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); doc.set(ObjRef::new(4, 0), predicted); - assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1 }); + assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1, headings: 0 }); let page = &pdfcraft_model::pages(&doc)[0]; let written = doc.resolve(page.dict.get(b"Contents").unwrap()); let Object::Stream(written) = &*written else { panic!("page content") }; @@ -157,7 +161,7 @@ fn autotag_text_marks_vector_graphics_and_images_as_artifacts() { "", ); let result = autotag_text(&mut doc).unwrap(); - assert_eq!(result, AutoTagResult { pages: 1, paragraphs: 1 }); + assert_eq!(result, AutoTagResult { pages: 1, paragraphs: 1, headings: 0 }); let report = check(&doc, &Options { rules: [Rule::TaggedPdf, Rule::TaggedContent].into(), pages: None }); assert_eq!(status(&report, Rule::TaggedPdf), Status::Passed); assert_eq!(status(&report, Rule::TaggedContent), Status::Passed); diff --git a/crates/automation/src/a11y.rs b/crates/automation/src/a11y.rs index b17192aab0..cab0fb9a61 100644 --- a/crates/automation/src/a11y.rs +++ b/crates/automation/src/a11y.rs @@ -217,7 +217,7 @@ impl Automation { let tags = self.accessibility_tags(a)?; let out = json!({ "tags": tags["tags"], "count": tags["count"], - "note": "Text objects were tagged as paragraphs. Review heading roles, reading order and remaining accessibility findings manually.", + "note": "Text objects were tagged as headings (H1-H6, by size and weight against the body text) or paragraphs. Review heading roles, reading order and remaining accessibility findings manually.", }); Ok(with_warnings(out, warnings)) } diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index f749ed2953..5251bf96ce 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -509,6 +509,25 @@ fn autotag_apply_writes_reviewable_tags_and_survives_save_reopen() { assert!(pages.iter().all(|page| page.dict.contains(b"StructParents"))); } +/// #7062: Apply tags short text larger than the body as headings, ranked across the pages, and +/// the body text as paragraphs. +#[test] +fn autotag_apply_tags_headings_by_size_across_pages() { + let dir = workdir("autotag-headings"); + let body = "BT /FR 11 Tf 72 670 Td (Body text that runs long enough to be the most common style) Tj ET"; + let page1 = format!("BT /FB 18 Tf 72 700 Td (Introduction) Tj ET {body} BT /FB 14 Tf 72 640 Td (Background) Tj ET"); + let page2 = format!("BT /FB 18 Tf 72 700 Td (Results) Tj ET {body}"); + std::fs::write(dir.join("headed.pdf"), bold_and_regular_pdf(&page1, &page2)).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "headed.pdf" }))["doc"].as_u64().unwrap(); + let applied = ok(&mut a, "accessibility_autotag_apply", json!({ "doc": doc })); + let types: Vec<&str> = applied["tags"].as_array().unwrap().iter().map(|t| t["type"].as_str().unwrap()).collect(); + assert_eq!(types, ["Document", "H1", "P", "H2", "H1", "P"]); + let check = ok(&mut a, "accessibility_check", json!({ "doc": doc, "rules": ["heading-nesting"] })); + let nesting = check["results"].as_array().unwrap().iter().find(|r| r["rule"] == "heading-nesting").unwrap(); + assert_eq!(nesting["status"], "passed", "{check}"); +} + #[test] fn autotag_form_fields_writes_reviewable_widget_tags() { let dir = workdir("autotag-form-fields"); @@ -531,6 +550,12 @@ fn styled_headings_pdf() -> Vec { let page1 = "BT /FB 18 Tf 72 700 Td (Introduction) Tj ET BT /FR 11 Tf 72 670 Td (Some text) Tj ET \ BT /FB 14 Tf 72 640 Td (Background) Tj ET"; let page2 = "BT /FB 18 Tf 72 700 Td (Results) Tj ET BT /FR 11 Tf 72 670 Td (More text) Tj ET"; + bold_and_regular_pdf(page1, page2) +} + +/// A two-page PDF whose pages draw `page1` and `page2` with `/FB` (Helvetica-Bold) and `/FR` +/// (Helvetica). +fn bold_and_regular_pdf(page1: &str, page2: &str) -> Vec { let objs: Vec = vec![ "<< /Type /Catalog /Pages 2 0 R >>".into(), "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 612 792] /Resources << /Font << /FB 7 0 R /FR 8 0 R >> >> >>".into(), diff --git a/docs/gaps.md b/docs/gaps.md index 1a1c04a9da..6f2ab13219 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -250,7 +250,7 @@ update the numbers. Read Out Loud; full keyboard operation of PdfCraft itself and a screen-reader audit. - **Evidence:** all 32 checker rules, figure alt text, tag property and sibling order edits, and a bounded auto-tag preview exist. Apply writes valid structure and parent-tree links for - text-only files; it refuses mixed content and existing tags; + text-only files, tagging headings by size and weight; it refuses mixed content and existing tags; `a11y.keyboard-only` (P0) planned, `a11y.app-screen-reader` partial. - **Impact:** most untagged files still need another editor to build their structure; PdfCraft itself is not yet usable without a mouse. diff --git a/docs/roadmap.md b/docs/roadmap.md index 7ca83dedd9..abc6d13b9f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -296,7 +296,7 @@ total matches the parity estimate. | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded, text-only auto-tag proposal tool for one page. Marking a figure decorative rewrites only that stream (#1324). An XFA radio item whose value contains a space is written back unchanged (#1319). Done since: #2569: Growable multiline XFA text reflows after edits, with undo/redo, save/reopen and print coverage; oversized fields still become read-only page text. Structured-data synchronization warnings retain accepted whitespace without relayout from stale data. Done since: #2572: Every repeated page-master button resolves its own field to the shared XFA click script, including print buttons. Done since: #2570: Dynamic XFA rich-text fields carry exData/maxLength into AcroForm MaxLen for reported and enforced Unicode character limits, retained after save/reopen. Done since: #2589: Active XFA skips PDF-container fallback scripts while retaining requested messages and saved scripts; ordinary/static/unlaid-out documents and explicit console/field calls keep their existing behavior. Missing: writing proposed tags and parent tree, field autotag, visual Tags/Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Reading colours: pending page backgrounds follow the configured colour (#2059). | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers
Done since: CLI run and MCP reject missing, empty, repeated and option-valued --root arguments before any tool/script execution or MCP initialization. A single-hyphen directory name remains valid; synthetic CLI regressions preserve a would-be overwritten file. `text_find` also caps the per-page match ranges it materializes at the remaining result limit plus one continuation match. CLI robustness scans visit each directory once through overlapping roots and symlink aliases. | -| M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded text-only auto-tag proposal tool; text-only Apply writes marked content, paragraph tags and a parent tree, then opens the Tags panel for review. Missing: mixed-content and field autotag, heading/list/table/figure classification, Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Done since: CLI info, text and render use dynamic XFA layout pages, with synthetic count/text/pixel and copy-permission regressions (#1645).
Reading colours: pending page backgrounds follow the configured colour (#2059). | +| M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded text-only auto-tag proposal tool; text-only Apply writes marked content, heading and paragraph tags and a parent tree, then opens the Tags panel for review. Missing: mixed-content and field autotag, list/table/figure classification, Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Done since: CLI info, text and render use dynamic XFA layout pages, with synthetic count/text/pixel and copy-permission regressions (#1645).
Reading colours: pending page backgrounds follow the configured colour (#2059). | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers
Done since: CLI run and MCP reject missing, empty, repeated and option-valued --root arguments before any tool/script execution or MCP initialization. A single-hyphen directory name remains valid; synthetic CLI regressions preserve a would-be overwritten file. `text_find` also caps the per-page match ranges it materializes at the remaining result limit plus one continuation match. | | M14 | 1.0 polish: performance, localization, installers | 120–250 | 15% | 80–150 | Local customization: adjustable, floating or side-docked hotbar with optional full-side strip and persistent tool picker; freehand highlighting is grouped with Highlight. Estimates unchanged. Done: PhotoCraft's translation system (`i18n/`: TSV catalogs, `tl!`, command-id and plural entries, system-language detection, strict catalog tests); every dialog, panel and notice goes through `tl!`. Catalogs of ≈ 2,000–2,140 entries for Japanese, Simplified and Traditional Chinese, Russian, Bulgarian, German, Spanish, French, Telugu, Hungarian, Ukrainian and Arabic (logical catalog; explicit right-to-left labels and mirrored stock chrome; end-to-end RTL workflows unqualified); the 2,674-entry Brazilian Portuguese catalog covers the current key union; Czech covers the menus. Italian has 2,528 entries: every key of the other catalogs that the code looks up. Signed and notarized macOS builds, signed Windows installers (x64, x86, ARM64), AppImage, deb, rpm, Flatpak and FreeBSD packages. #307 (bounded large-PDF memory): shared resource indexes and parser metadata, lazy inspection, bounded caches; a 9,156-page file inspects in ≈ 1.8 s at ≈ 500 MiB instead of exceeding 10 GiB. Edits let go of the previous working file before writing the next (a 1 GiB file peaks at two copies while editing, not three; gap 13). Shortcut display uses localized modifier/key names across menus, toolbar, palette, Help and tooltips (#496). Lithuanian adds a 2,578-entry catalog and date names, reviewed by a native-speaking contributor. Norwegian Bokmål adds a catalog and date names (terminology reviewed by a native speaker). Validated candidate (macOS raster probe; Windows runtime unverified): the Windows policy changes only embedded JetBrains Mono; horizontal alpha-edge energy per ink improves 2.78%, 3.87% and 1.59% at 100%, 125% and 150%, and is down 0.79% at 200%. Inter is pixel-identical; layout hashes, glyph coverage and the 8 MiB atlas are unchanged. Timing and RSS ranges overlap, with no improvement claimed. Estimates unchanged. Candidate #2383 clamps GPU requests to supported limits while retaining the adapter texture extent; synthetic V3D/downlevel and Windows GPU-required tests pass, but Raspberry Pi startup is unverified. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation Linux/Unix UI fallback fonts also use Fontconfig directories, including Nix store paths, with bounded discovery and the existing candidate allowlist; relative includes and digit-prefixed config filtering have isolated regressions. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation XDG desktops use the configuration base for preferences with non-overwriting migration and original-data preservation (#1395). Traditional Chinese accessibility statuses, rule semantics and Taiwan form/table terms have catalog regressions. Long main-menu lists scroll in short windows (#3373). Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation. Candidate #3558: bounded-worker macOS per-file PDFCraft association recovery after desktop open/save/sign; metadata-only watcher handling and upstream #3042 shared-writer preservation; recovery parser limits do not narrow save limits. Native workflow qualified on macOS 27.0.1; other versions and physical SMB/FUSE volumes remain unqualified. Estimates unchanged. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean/Vietnamese, file-backed input and global memory budgets, performance budgets, keyboard-only operation. | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index e533ecf6c7..6eb57527d3 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -9158,8 +9158,8 @@ milestone = "M12" status = "partial" commands = ["a11y.autotag"] tools = ["accessibility_autotag_preview", "accessibility_autotag_apply"] -evidence = ["crates/automation/tests/it/automation.rs::autotag_preview_proposes_text_without_changing_the_pdf", "crates/automation/tests/it/automation.rs::autotag_apply_writes_reviewable_tags_and_survives_save_reopen", "crates/ui-egui/tests/it/accessibility.rs::automatic_text_tagging_opens_reviewable_tags"] -notes = "Auto-tag writes real marked content, a Document structure element, P tags and a parent tree for text-only PDFs, with undo and Tags-panel review. It refuses mixed content, forms, annotations and existing tags. The preview separately proposes headings. Images, tables, lists, links, fields and headings are not classified or tagged by Apply." +evidence = ["crates/automation/tests/it/automation.rs::autotag_preview_proposes_text_without_changing_the_pdf", "crates/automation/tests/it/automation.rs::autotag_apply_writes_reviewable_tags_and_survives_save_reopen", "crates/automation/tests/it/automation.rs::autotag_apply_tags_headings_by_size_across_pages", "crates/a11y/src/autotag_roles.rs::levels_rank_styles_largest_first_and_never_skip_a_level", "crates/ui-egui/tests/it/accessibility.rs::automatic_text_tagging_opens_reviewable_tags"] +notes = "Auto-tag writes real marked content, a Document structure element, H1-H6 and P tags and a parent tree for text-only PDFs, with undo and Tags-panel review. A text object is a heading when it is short and larger or bolder than the body text; levels rank those styles across the document and never skip a level. It refuses mixed content, forms, annotations and existing tags. The preview still proposes headings by its own single-line rule, so it can differ from Apply. Running headers and footers can still become headings (#7065); invisible text (render mode 3) is ignored. Images, tables, lists, links and fields are not classified or tagged by Apply." [[feature]] id = "a11y.autotag-fields"