From 315e83b88c8e06f308fbbd1826ddc3c864e4ba18 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 15:22:50 -0500 Subject: [PATCH 01/85] perf(cos): an object that can't be read is searched for once, not on every read Document::try_get kept only objects that loaded. An object whose entry points at something that doesn't parse falls back to a whole-file search (scan_for); that failed too, nothing was kept, and the next read searched the whole file again. A caller that reads such an object repeatedly, such as the window title of a file whose document information dictionary is damaged and that asks for its title to be shown, paid one whole-file search per read, several per frame. The cache now keeps the load result, a failure as well. It is the same map with the same bounds, clears and per-reader privacy, so every place that drops loaded objects (authentication, reconstruction, private scan readers, eviction) drops remembered failures too. Edits are read from the overlay before the cache, as before, so a set or freed number still answers for itself. A failure is kept in nested loads too, as a success already is; only a poisoned lock is never kept. Failures count towards the cache's object caps and hold no bytes of their own. Sharing the map between clones stays sound because entries, data and the stream limit don't change after open. Test: an_unreadable_object_is_searched_for_once counts whole-file searches through a test-only counter (like SECTION_READS): one for eleven reads and a clone's read. With failures not kept it counts 22. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document.rs | 10 +++++---- crates/cos/src/document/objects.rs | 35 +++++++++++++++++------------- crates/cos/src/document/state.rs | 12 +++++++++- crates/cos/src/document/tests.rs | 31 +++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 082d1910d1..6e5891d1a4 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,10 +24,10 @@ pub(crate) use reader::find; #[cfg(test)] use reader::rfind; pub(crate) use state::ObjectReader; -use state::{ObjStm, Slot}; -pub use state::{ReadState, SourceIdentity}; +use state::{Loaded, ObjStm, Slot}; #[cfg(test)] -use state::{SECTION_READS, objstm_member, owned_bytes}; +use state::{OBJECT_SEARCHES, SECTION_READS, objstm_member, owned_bytes}; +pub use state::{ReadState, SourceIdentity}; use xref::XrefIndex; pub use xref::{Revision, XrefEntry}; @@ -47,7 +47,9 @@ pub struct Document { /// `fill_xref_gaps`. located: Arc>, repair_log: Arc>, - cache: Arc>>>, + /// What loading each file object gave, a failure as well: a damaged object is otherwise + /// searched for across the whole file on every read (`Document::load`). + cache: Arc>>, objstms: Arc>>>, /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ac18cff03..2ca327de74 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,5 +1,5 @@ use super::reader::find; -use super::state::{LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, objstm_member, owned_bytes}; +use super::state::{LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, objstm_member}; use super::xref::{Revision, XrefEntry}; use super::{ Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, @@ -388,20 +388,23 @@ impl Document { Some(Slot::Freed(_)) => return Ok(Arc::new(Object::Null)), None => {} } - if let Some(o) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { - return Ok(o.clone()); + if let Some(loaded) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + return loaded.clone(); } // Loading can re-enter (indirect stream `/Length`, object streams); damaged files make // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let obj = self.load(num, 0)?; - let obj = Arc::new(obj); - // The editing cache is bounded like the scan reader's: re-parsing a file-backed view - // is cheap, and returned objects outlive eviction. - self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); - self.evict_editor_cache_if_over_budget(num, &obj)?; - Ok(obj) + let loaded = self.load(num, 0).map(Arc::new); + // A failure is kept like an object, or every read of a damaged object searches the whole + // file again. A poisoned lock says nothing about the object. + if !matches!(loaded, Err(CosError::Poisoned)) { + // The editing cache is bounded like the scan reader's: re-parsing a file-backed view + // is cheap, and returned objects outlive eviction. + self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone()); + self.evict_editor_cache_if_over_budget(num, &loaded)?; + } + loaded } /// Bounds for the editing cache: roomier than the scan reader's (128 objects / 8 MB) @@ -430,17 +433,17 @@ impl Document { /// byte budget can double. const EDITOR_LARGE_OBJECT: usize = 256 * 1024; - fn evict_editor_cache_if_over_budget(&self, fresh_num: u32, fresh: &Object) -> Result<(), CosError> { + fn evict_editor_cache_if_over_budget(&self, fresh_num: u32, fresh: &Loaded) -> Result<(), CosError> { // Fast path: the full fold is O(cache) per insert, so skip it for a small newcomer // below the count cap. `len` is copied out first; nothing here holds the lock. let len = self.cache.lock().map_err(|_| CosError::Poisoned)?.len(); - if len <= Self::EDITOR_CACHE_OBJECTS && owned_bytes(fresh, &self.data, 0) <= Self::EDITOR_LARGE_OBJECT { + if len <= Self::EDITOR_CACHE_OBJECTS && loaded_bytes(fresh, &self.data) <= Self::EDITOR_LARGE_OBJECT { return Ok(()); } // Walk a snapshot off-lock; re-lock only to evict. - let live: Vec<(u32, Arc)> = self.cache.lock().map_err(|_| CosError::Poisoned)?.iter().map(|(n, o)| (*n, o.clone())).collect(); + let live: Vec<(u32, Loaded)> = self.cache.lock().map_err(|_| CosError::Poisoned)?.iter().map(|(n, o)| (*n, o.clone())).collect(); let data = &self.data; - let owned: usize = live.iter().fold(0usize, |n, (_, o)| n.saturating_add(owned_bytes(o, data, 0))); + let owned: usize = live.iter().fold(0usize, |n, (_, o)| n.saturating_add(loaded_bytes(o, data))); if live.len() <= Self::EDITOR_CACHE_OBJECTS && owned <= Self::EDITOR_CACHE_BYTES { return Ok(()); } @@ -450,7 +453,7 @@ impl Document { let mut cache = self.cache.lock().map_err(|_| CosError::Poisoned)?; if cache.len() <= Self::EDITOR_CACHE_OBJECTS { let data = &self.data; - let owned: usize = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))); + let owned: usize = cache.values().fold(0usize, |n, o| n.saturating_add(loaded_bytes(o, data))); if owned <= Self::EDITOR_CACHE_BYTES { return Ok(()); } @@ -602,6 +605,8 @@ impl Document { /// Last-resort lookup: scan the file for `num G obj`. fn scan_for(&self, num: u32) -> Option<(ObjRef, Object)> { + #[cfg(test)] + super::OBJECT_SEARCHES.with(|c| c.set(c.get() + 1)); let needle = format!("{num} "); let data = &self.data; let mut found = None; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index b4323f982e..121a59c2e7 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -30,6 +30,14 @@ impl LoadGuard { } } +/// What loading a file object gave, kept in the document's cache either way. +pub(super) type Loaded = Result, CosError>; + +/// The bytes a load result holds of its own ([`owned_bytes`]); a failure holds none. +pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc>) -> usize { + loaded.as_ref().map_or(0, |o| owned_bytes(o, data, 0)) +} + impl Drop for LoadGuard { fn drop(&mut self) { let outermost = LOADING.with(|l| { @@ -165,7 +173,7 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))) > Self::STREAM_BYTES { + if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(loaded_bytes(o, data))) > Self::STREAM_BYTES { cache.clear(); } } @@ -177,6 +185,8 @@ impl ObjectReader { thread_local! { /// Cross-reference sections read on this thread. pub(super) static SECTION_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Whole-file searches for an object (`Document::scan_for`) on this thread. + pub(super) static OBJECT_SEARCHES: std::cell::Cell = const { std::cell::Cell::new(0) }; } /// Nesting [`owned_bytes`] walks: past the parser's own nesting limit, so every parsed object diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index cca4b87326..3cebe0136d 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -588,6 +588,31 @@ fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() { assert!(doc.xref_entry_is_exact(1) && doc.xref_entry_is_exact(2)); } +/// An object that can't be read at its entry or anywhere else is searched for in the whole file +/// once: every later read, in a clone too, gives the same error without searching again. An +/// edit of that number still answers for it. +#[test] +fn an_unreadable_object_is_searched_for_once() { + let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"]; + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap(); + // Counted wherever the whole-file search is done (`scan_for`). + OBJECT_SEARCHES.with(|c| c.set(0)); + let first = doc.try_get(3); + assert!(first.is_err(), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + } + let mut edited = doc.clone(); + assert_eq!(edited.try_get(3), first); + assert_eq!(OBJECT_SEARCHES.with(|c| c.get()), 1, "searches for an object that isn't there"); + edited.set(ObjRef::new(3, 0), Object::Int(7)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Int(7))); + assert_eq!(doc.try_get(3), first, "the edit is the clone's alone"); + edited.free(ObjRef::new(3, 0)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { @@ -1109,7 +1134,7 @@ fn temporary_reader_keeps_file_views_and_evicts_data_of_its_own() { // Data of its own (decrypted streams, strings) counts: past the budget the cache is // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned)); + reader.document.cache.lock().unwrap().insert(3, Ok(Arc::new(owned))); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } @@ -1198,8 +1223,8 @@ fn editor_cache_counts_deeply_nested_strings() { let Object::Ref(reference) = reference else { panic!("indirect object") }; drop(doc.get(*reference)); let cache = doc.cache.lock().unwrap(); - let owned = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &doc.data, 0))); - let strings = cache.values().filter(|o| o.as_array().is_some()).count(); + let owned = cache.values().flatten().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &doc.data, 0))); + let strings = cache.values().flatten().filter(|o| o.as_array().is_some()).count(); assert!(strings << 20 <= Document::EDITOR_CACHE_BYTES + (1 << 20), "{strings} MiB strings cached, {owned} bytes counted"); } } From 84d4b8cab9b8ee1d6d0c3fbed70ea3291c22a1b4 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 16:21:45 -0500 Subject: [PATCH 02/85] fix(cos): a load that read one clone's edits is not shared with the others Clones share the object cache and the parsed object streams. A clone that freed or replaced an object stream then read a member of it put the result in those shared maps: a failure (which this branch now caches) or, as on main, the replacement stream and the member parsed from it. Another clone whose stream is intact then read that instead of the file. A load that reads an unsaved edit (an edited object stream, or an edited indirect /Length) is no longer cached, and an edited object stream is parsed for its own document only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 30 ++++++++++++++++++++++-------- crates/cos/src/document/state.rs | 11 +++++++++++ crates/cos/src/document/tests.rs | 21 +++++++++++++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ca327de74..55a01e117c 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,5 +1,7 @@ use super::reader::find; -use super::state::{LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, objstm_member}; +use super::state::{ + LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, note_overlay_read, objstm_member, overlay_reads, +}; use super::xref::{Revision, XrefEntry}; use super::{ Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, @@ -383,10 +385,12 @@ impl Document { } pub fn try_get(&self, num: u32) -> Result, CosError> { - match self.overlay.get(&num) { - Some(Slot::Set(_, o)) => return Ok(o.clone()), - Some(Slot::Freed(_)) => return Ok(Arc::new(Object::Null)), - None => {} + if let Some(slot) = self.overlay.get(&num) { + note_overlay_read(); + return Ok(match slot { + Slot::Set(_, o) => o.clone(), + Slot::Freed(_) => Arc::new(Object::Null), + }); } if let Some(loaded) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { return loaded.clone(); @@ -395,10 +399,12 @@ impl Document { // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; + let edits_read = overlay_reads(); let loaded = self.load(num, 0).map(Arc::new); // A failure is kept like an object, or every read of a damaged object searches the whole - // file again. A poisoned lock says nothing about the object. - if !matches!(loaded, Err(CosError::Poisoned)) { + // file again. A poisoned lock says nothing about the object. Clones share the cache, so + // a load that read this document's edits (an edited object stream or `/Length`) is not kept. + if overlay_reads() == edits_read && !matches!(loaded, Err(CosError::Poisoned)) { // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone()); @@ -540,6 +546,9 @@ impl Document { None | Some(XrefEntry::Free { .. }) => Ok(Object::Null), Some(XrefEntry::InFile { offset, .. }) => { let resolve = |r: ObjRef| { + if self.overlay.contains_key(&r.num) { + note_overlay_read(); + } if let Some(known) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { return known; } @@ -570,7 +579,9 @@ impl Document { } pub(super) fn objstm(&self, num: u32) -> Result, CosError> { - if let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + // An edited or freed stream is this document's alone: clones share the parsed streams. + let edited = self.overlay.contains_key(&num); + if !edited && let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { return Ok(s.clone()); } // A file entry that cannot place the stream (one that says it is itself compressed @@ -596,6 +607,9 @@ impl Document { index.push(objstm_member(onum, off, first, data.len())); } let stm = Arc::new(ObjStm { index, data: Arc::new(data) }); + if edited { + return Ok(stm); + } // Same bound as the object cache: decoded object streams can be megabytes each. // Insert first so the retained `num` is actually in the map. self.objstms.lock().map_err(|_| CosError::Poisoned)?.insert(num, stm.clone()); diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 121a59c2e7..286973943e 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -8,6 +8,17 @@ thread_local! { /// retries a parse several ways, and each retry would resolve the same lengths again /// (exponential in the chain of streams whose lengths point at further streams). pub(super) static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); + /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). + static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Unsaved edits read so far on this thread: a load during which this changes depended on one. +pub(super) fn overlay_reads() -> u64 { + OVERLAY_READS.with(std::cell::Cell::get) +} + +pub(super) fn note_overlay_read() { + OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1))); } /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 3cebe0136d..ee7fd28750 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -613,6 +613,27 @@ fn an_unreadable_object_is_searched_for_once() { assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); } +/// Clones share the cache, and an edit is one clone's alone: a member of an object stream that +/// one clone freed or replaced still reads, in the others, from the file's stream. +#[test] +fn an_edited_object_stream_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); + let doc = Document::open(file.clone()).unwrap(); + let mut freed = doc.clone(); + freed.free(ObjRef::new(4, 0)); + assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4))); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the free reached another clone"); + let doc = Document::open(file).unwrap(); + let mut replaced = doc.clone(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + replaced.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec())); + assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no")); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { From 85df8ee66817a726af56a7101bab81a2a791f823 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:08:05 -0500 Subject: [PATCH 03/85] feat(cos): list the objects that could not be read in doc_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An object whose load fails reads as null (ISO 32000-2 §7.3.10) and the file opens, but nothing said so: the repair log records only document-level repairs, so a file with an unparseable /Info showed no repair and no reason for its missing title. pdfcraft-cos Document now records each object whose load fails, with its generation and the reason, in a set shared by clones and scans like the object cache: at most 100, keyed by number, first reason kept, cut to 160 characters, built only for an object not yet listed. It is reset where the caches are (authenticate, reconstruct); edited objects drop off a document's list. Document::unreadable_objects() gives neutral lines ("object 4 0 could not be read: ..."), the engine passes them on, and automation doc_info adds an unreadable_objects array, kept apart from repairs so the "has been repaired" notice and the clean-file checks keep their meaning. Tests: a cos unit test for listing, dedup, clones, scans and edits; a cos test for the 100 cap and the no-readable-copy reason; an end-to-end doc_info test on a synthetic broken file. Each fails with the fix reverted. The core.repair-log parity entry cites them and says what is not listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/automation/src/json.rs | 3 + crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/main.rs | 1 + .../automation/tests/it/unreadable_objects.rs | 64 +++++++++++++++++++ crates/cos/src/document.rs | 5 +- crates/cos/src/document/objects.rs | 48 +++++++++++++- crates/cos/src/document/open.rs | 3 + crates/cos/src/document/reader.rs | 3 + crates/cos/src/document/state.rs | 48 ++++++++++++++ crates/cos/src/document/tests.rs | 58 +++++++++++++++++ crates/engine/src/document.rs | 6 ++ parity/acrobat-features.toml | 4 +- 12 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 crates/automation/tests/it/unreadable_objects.rs diff --git a/crates/automation/src/json.rs b/crates/automation/src/json.rs index 9e93d4c347..c950e72f05 100644 --- a/crates/automation/src/json.rs +++ b/crates/automation/src/json.rs @@ -375,6 +375,9 @@ pub(crate) fn info(d: &Document) -> Value { })).collect::>(), "warnings": i.warnings, "repairs": d.repair_log(), + // Objects that could not be read (each reads as null): nothing was repaired, what they held is + // lost. Last, so it includes what reading the fields above failed on. + "unreadable_objects": d.unreadable_objects(), }) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 7cafcb4a37..06927dee90 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -142,7 +142,7 @@ pub fn tools() -> Vec { .cmd("file.open") .with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])), t("doc_list", "List open documents", "List the open documents with their ids, page counts and unsaved state.").ro().with(schema(json!({}), &[])), - t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security and repair notes.") + t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security, repair notes and the objects that could not be read.") .ro() .cmd("file.properties") .with(schema(json!({ "doc": doc() }), &["doc"])), diff --git a/crates/automation/tests/it/main.rs b/crates/automation/tests/it/main.rs index d36dbe93a9..14e485c9f3 100644 --- a/crates/automation/tests/it/main.rs +++ b/crates/automation/tests/it/main.rs @@ -12,3 +12,4 @@ mod optimize_content_boundaries; mod redact_after_stray_q_form; mod redact_text_after_q; mod render_warnings; +mod unreadable_objects; diff --git a/crates/automation/tests/it/unreadable_objects.rs b/crates/automation/tests/it/unreadable_objects.rs new file mode 100644 index 0000000000..f19a19078a --- /dev/null +++ b/crates/automation/tests/it/unreadable_objects.rs @@ -0,0 +1,64 @@ +//! `doc_info` lists the objects of a file that could not be read: each reads as null, so the file +//! opens, but what it held is lost and nothing was repaired. + +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +/// A one-page PDF whose document information dictionary (object 4) is `info`. +fn with_info(info: &str) -> Vec { + let bodies = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + info, + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, body) in bodies.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", bodies.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R /Info 4 0 R >>\nstartxref\n{xref}\n%%EOF\n", bodies.len() + 1).as_bytes()); + out +} + +fn info_of(a: &mut Automation, path: &str) -> Value { + let doc = match a.call("doc_open", &json!({ "path": path })).unwrap().remove(0) { + Content::Json(v) => v["doc"].as_u64().unwrap(), + other => panic!("expected JSON: {other:?}"), + }; + match a.call("doc_info", &json!({ "doc": doc })).unwrap().remove(0) { + Content::Json(v) => v, + other => panic!("expected JSON: {other:?}"), + } +} + +/// An unparseable `/Info` is listed with its number and why, the rest of the file reads, and +/// nothing is reported as repaired. +#[test] +fn doc_info_lists_an_object_that_could_not_be_read() { + let dir = std::env::temp_dir().join(format!("pdfcraft-unreadable-objects-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("broken.pdf"), with_info("<< /Title (Report) /Author ) >>")).unwrap(); + std::fs::write(dir.join("sound.pdf"), with_info("<< /Title (Report) >>")).unwrap(); + let mut a = Automation::new().with_root(&dir).unwrap(); + + let info = info_of(&mut a, "broken.pdf"); + let lines = info["unreadable_objects"].as_array().unwrap(); + assert_eq!(lines.len(), 1, "{info}"); + let line = lines[0].as_str().unwrap(); + assert!(line.starts_with("object 4 0 could not be read: syntax error"), "{line}"); + assert_eq!(info["repairs"], json!([]), "nothing was repaired"); + assert_eq!(info["pages"].as_array().unwrap().len(), 1, "the rest of the file reads"); + + let info = info_of(&mut a, "sound.pdf"); + assert_eq!(info["unreadable_objects"], json!([]), "{info}"); + assert_eq!(info["title"], "Report"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 082d1910d1..81eb1fd4d9 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,7 +24,7 @@ pub(crate) use reader::find; #[cfg(test)] use reader::rfind; pub(crate) use state::ObjectReader; -use state::{ObjStm, Slot}; +use state::{ObjStm, Slot, Unreadable}; pub use state::{ReadState, SourceIdentity}; #[cfg(test)] use state::{SECTION_READS, objstm_member, owned_bytes}; @@ -49,6 +49,9 @@ pub struct Document { repair_log: Arc>, cache: Arc>>>, objstms: Arc>>>, + /// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the + /// source, and is reset wherever they are. + unreadable: Arc>, /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). jbig2_globals: Arc>>>>, diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ac18cff03..fdda5f5cf5 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -86,6 +86,38 @@ impl Document { &self.repair_log } + /// Objects of the file that could not be read, lowest number first, as lines such as + /// "object 11 0 could not be read: syntax error at byte 512: ...". [`Self::get`] reads each + /// as null (ISO 32000-2 §7.3.10), so a document with one opens, but what it held is lost: + /// unlike [`Self::repair_log`], nothing was repaired. Lists the objects read so far, by this + /// document, its clones or a scan of it, that have not since been edited; past 100, a last + /// line says there were more. Opening again, authenticating or rebuilding the index starts + /// the list afresh, as it does the object cache. + pub fn unreadable_objects(&self) -> Vec { + let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() }; + unreadable.lines(|num| self.overlay.contains_key(&num)) + } + + /// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file + /// whose broken object is asked for again and again pays for it once. + fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { + // A poisoned lock only means the object goes unlisted; the read itself still fails. + let Ok(mut unreadable) = self.unreadable.lock() else { return }; + if unreadable.wants(num) { + unreadable.note(num, self.generation(num), &reason()); + } + } + + /// Why an object listed at `offset` could not be read: what parsing it there, without resolving + /// an indirect `/Length`, found. A best account, not the only one: the reader also tried + /// elsewhere. + fn why_unreadable(&self, offset: usize) -> String { + match parse_indirect_shared(&self.data, offset, &|_| None, &self.endstreams) { + Err(e) => e.to_string(), + Ok(_) => "no readable copy was found at its cross-reference offset or elsewhere in the file".into(), + } + } + /// Add a line to [`Self::repair_log`]: a leniency applied while reading that the user should /// know about (fidelity: never silent). pub fn note_repair(&mut self, line: String) { @@ -395,8 +427,15 @@ impl Document { // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let obj = self.load(num, 0)?; - let obj = Arc::new(obj); + let obj = match self.load(num, 0) { + Ok(obj) => Arc::new(obj), + Err(e) => { + if !matches!(e, CosError::Poisoned) { + self.note_unreadable(num, || e.to_string()); + } + return Err(e); + } + }; // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); @@ -554,7 +593,10 @@ impl Document { .or_else(|| if self.header_offset == 0 { None } else { at(off.saturating_add(self.header_offset)) }) .or_else(|| self.located.get(&num).and_then(|(o, _)| usize::try_from(*o).ok()).and_then(at)) .or_else(|| self.scan_for(num).map(|(id, o)| self.decrypted(id, o))) - .ok_or(CosError::MissingObject(num)) + .ok_or_else(|| { + self.note_unreadable(num, || self.why_unreadable(off)); + CosError::MissingObject(num) + }) } Some(XrefEntry::InStream { stream, index }) => { let stm = self.objstm(stream)?; diff --git a/crates/cos/src/document/open.rs b/crates/cos/src/document/open.rs index 395424b771..21c0dd83ba 100644 --- a/crates/cos/src/document/open.rs +++ b/crates/cos/src/document/open.rs @@ -19,6 +19,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: crate::object::MAX_DECODED, overlay: BTreeMap::new(), @@ -103,6 +104,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: stream_limit.min(crate::object::MAX_DECODED), overlay: BTreeMap::new(), @@ -200,6 +202,7 @@ impl Document { // Objects read while locating the catalog were read before decryption was possible. self.cache = Arc::default(); self.objstms = Arc::default(); + self.unreadable = Arc::default(); Ok(()) } diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index d21bf9403f..1f944fd870 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -1,5 +1,6 @@ #[cfg(test)] use super::state::SECTION_READS; +use super::state::Unreadable; use super::xref::{ MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData, }; @@ -633,6 +634,7 @@ impl Document { } self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()))); self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Objects inside object streams. for num in objstms { if let Ok(o) = self.try_get(num) @@ -650,6 +652,7 @@ impl Document { let object_count = entries.len(); self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries))); self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a // usable /Root (last in the file, whichever kind), else the catalog itself. One whose // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index b4323f982e..8dbba4a93a 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -61,6 +61,54 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: us (start < len).then_some((num, start)) } +/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). +/// Bounded: past [`Unreadable::MAX`] objects only the fact that there were more is kept. +#[derive(Debug, Default)] +pub(super) struct Unreadable { + pub(super) objects: BTreeMap, + pub(super) more: bool, +} + +impl Unreadable { + pub(super) const MAX: usize = 100; + /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. + const REASON: usize = 160; + + /// Whether `num` is to be recorded: not listed yet and there is room. A full list notes that + /// there were more, so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32) -> bool { + if self.objects.contains_key(&num) { + return false; + } + if self.objects.len() >= Self::MAX { + self.more = true; + return false; + } + true + } + + /// Record `num` once; the first reason given is kept. + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str) { + if self.wants(num) { + self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); + } + } + + /// One line per recorded object not `hidden`, then one saying there were more, if there were. + pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { + let mut lines: Vec = self + .objects + .iter() + .filter(|(num, _)| !hidden(**num)) + .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) + .collect(); + if self.more { + lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); + } + lines + } +} + /// Where an overlay slot came from. #[derive(Clone, Debug, PartialEq)] pub(super) enum Slot { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index cca4b87326..c400e029d2 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1701,3 +1701,61 @@ fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { }); rx.recv_timeout(std::time::Duration::from_secs(20)).expect("loading must not retry exponentially"); } + +/// A file whose object 3 (its `/Info`) can't be parsed: the rest reads. +fn with_unreadable_info() -> Vec { + build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Author ) >>"], + "/Root 1 0 R /Info 3 0 R", + ) +} + +/// An object that can't be parsed reads as null and is listed once, by number and generation, +/// with why. Clones and scans share the list, an edit takes the object off it (in that document +/// only), and a sound file lists nothing. +#[test] +fn unreadable_objects_are_listed_with_the_reason() { + let doc = Document::open(Arc::new(with_unreadable_info())).unwrap(); + assert!(doc.unreadable_objects().is_empty(), "nothing has read object 3 yet"); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert!(doc.try_get(3).is_err()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "read twice, listed once: {lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); + let scanned = doc.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); + assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); + assert_eq!(doc.unreadable_objects(), lines); + let sound = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) >>"], "/Root 1 0 R /Info 3 0 R"); + let sound = Document::open(Arc::new(sound)).unwrap(); + assert_eq!(sound.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 0); + assert!(sound.unreadable_objects().is_empty()); +} + +/// The list keeps at most 100 objects and then says there were more; an object whose +/// cross-reference offset holds no object is listed with that as its reason. +#[test] +fn the_unreadable_object_list_is_bounded() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 120)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 120); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 101, "{:?}", lines.last()); + assert!(lines[0].starts_with("object 3 0 could not be read"), "{}", lines[0]); + assert_eq!(lines[100], "more objects could not be read; only the first 100 are listed"); + + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let text = String::from_utf8_lossy(&base).into_owned(); + let gone = text.replacen("3 0 obj\n(three)", "9 0 obj\n(three)", 1); + let doc = Document::open(Arc::new(gone.into_bytes())).unwrap(); + assert!(doc.try_get(3).is_err()); + assert_eq!( + doc.unreadable_objects(), + ["object 3 0 could not be read: no readable copy was found at its cross-reference offset or elsewhere in the file"] + ); +} diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index fa98a18417..b3e93cab0b 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -631,6 +631,12 @@ impl Document { pub fn repair_log(&self) -> Vec { self.editor.as_ref().map(|e| e.cos.repair_log().to_vec()).unwrap_or_default() } + + /// Objects of the file that could not be read and read as null, one line each (see + /// [`pdfcraft_cos::Document::unreadable_objects`]): only those read so far. + pub fn unreadable_objects(&self) -> Vec { + self.editor.as_ref().map(|e| e.cos.unreadable_objects()).unwrap_or_default() + } } /// A title that names no document: what a web browser writes when it saves a blank page or a diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 290a410e14..451e84f4aa 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -70,8 +70,8 @@ milestone = "M1" status = "shipped" commands = [] tools = ["doc_info"] -evidence = ["crates/ui-egui/tests/it/viewer.rs::damaged_files_say_they_were_repaired"] -notes = "a notice says the file was repaired; Document Properties ▸ Advanced lists the repairs; doc_info reports them" +evidence = ["crates/ui-egui/tests/it/viewer.rs::damaged_files_say_they_were_repaired", "crates/cos/src/document/tests.rs::unreadable_objects_are_listed_with_the_reason", "crates/automation/tests/it/unreadable_objects.rs::doc_info_lists_an_object_that_could_not_be_read"] +notes = "a notice says the file was repaired; Document Properties ▸ Advanced lists the repairs; doc_info reports them, and lists apart (unreadable_objects) each object that could not be read and reads as null, with its number and why: only objects read so far, including any object a scan, save or audit read, at most 100; a stream whose own data fails to decode is not listed, but the objects of an object stream that cannot be decoded are" [[feature]] id = "core.xref-streams" From 8cc4d378476ddfa13c8a7540f05e8edfef6728cf Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:18:30 -0500 Subject: [PATCH 04/85] fix(cos): an object stream framed by one clone's edited /Length is not shared An unedited object stream whose indirect /Length is edited in one clone was parsed with that clone's length and put in the object-stream map the clones share, so the other clones read its members within the edited length. A container load that read the clone's edits is now kept to that clone, like an edited stream. Test: an_edited_object_stream_length_leaves_other_clones_members_alone fails without the fix (the other clone reads None) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 5 ++++- crates/cos/src/document/tests.rs | 32 ++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 55a01e117c..b8f9a64870 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -589,6 +589,7 @@ impl Document { // header, as a plain object with a wrong offset is. An edited or freed stream, in the // overlay or in a saved revision's table, is what that says it is: the file's copy // must not come back. + let edits_read = overlay_reads(); let loaded = match self.try_get(num) { Ok(o) if matches!(*o, Object::Stream(_)) => o, _ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => { @@ -596,6 +597,8 @@ impl Document { } _ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?), }; + // A stream whose load read this document's edits (its `/Length`) is as much its own as an edited one. + let own = edited || overlay_reads() != edits_read; let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) }; let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize; let first = usize::try_from(s.dict.int(b"First").unwrap_or(0).max(0)).ok(); @@ -607,7 +610,7 @@ impl Document { index.push(objstm_member(onum, off, first, data.len())); } let stm = Arc::new(ObjStm { index, data: Arc::new(data) }); - if edited { + if own { return Ok(stm); } // Same bound as the object cache: decoded object streams can be megabytes each. diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index ee7fd28750..15c9d8cd61 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -634,6 +634,20 @@ fn an_edited_object_stream_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); } +/// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: +/// the others still read its members within the file's length. +#[test] +fn an_edited_object_stream_length_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); + let doc = Document::open(file).unwrap(); + let mut edited = doc.clone(); + // Ends the stream at the `endstream` inside the member's string. + let cut = "3 0 << /Hidden (cut".len(); + edited.set(ObjRef::new(6, 0), Object::Int(cut as i64)); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { @@ -1367,8 +1381,14 @@ fn hybrid_reference_files_read_the_xref_stream_too() { /// stream puts it at position `member` of object stream 4, which holds `header` (the number /// and offset pairs) and then `members`. fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { + object_stream_file_with(header, members, member, false) +} + +/// [`object_stream_file`], with the stream's `/Length` in object 6 when `indirect_length`. +fn object_stream_file_with(header: &str, members: &str, member: u8, indirect_length: bool) -> Vec { let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 6]; + let size = if indirect_length { 7 } else { 6 }; + let mut offs = vec![0usize; size]; let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { offs[n] = out.len(); out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); @@ -1379,7 +1399,8 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); let count = header.split_whitespace().count() / 2; let inner = format!("{header}{members}"); - let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {} >>\nstream\n", header.len(), inner.len()).into_bytes(); + let length = if indirect_length { "6 0 R".to_string() } else { inner.len().to_string() }; + let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {length} >>\nstream\n", header.len()).into_bytes(); stm.extend_from_slice(inner.as_bytes()); stm.extend_from_slice(b"\nendstream"); obj(&mut out, 4, &stm); @@ -1388,13 +1409,16 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { xs.extend_from_slice(&data); xs.extend_from_slice(b"\nendstream"); obj(&mut out, 5, &xs); + if indirect_length { + obj(&mut out, 6, inner.len().to_string().as_bytes()); + } let table = out.len(); - out.extend_from_slice(b"xref\n0 6\n0000000000 65535 f \n"); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); for (n, off) in offs.iter().enumerate().skip(1) { let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; out.extend_from_slice(line.as_bytes()); } - out.extend_from_slice(format!("trailer << /Size 6 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); out } From 96d3d1e073b61a00ab4bea312fdb8cdf5e974013 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:56:44 -0500 Subject: [PATCH 05/85] fix(cos): a reused /Length keeps the edits it was read through The indirect /Length memo kept only the value. A length first resolved through one clone's edited object stream and reused later in the same load did not count as an edit read, so an object stream framed by it could still be parsed with the edited length and put in the object-stream map and cache the clones share. The memo now keeps whether resolving the length read edits, and a reuse counts it again. Test: a_reused_edited_length_leaves_other_clones_members_alone fails without the fix (the other clone reads None) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 10 ++++-- crates/cos/src/document/state.rs | 5 +-- crates/cos/src/document/tests.rs | 58 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index b8f9a64870..976e5e360c 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -549,11 +549,17 @@ impl Document { if self.overlay.contains_key(&r.num) { note_overlay_read(); } - if let Some(known) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { + // A length resolved through this document's edits counts as an edit read + // on every reuse. + if let Some((known, edits)) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { + if edits { + note_overlay_read(); + } return known; } + let edits_read = overlay_reads(); let length = self.try_get(r.num).ok().and_then(|o| o.as_int()); - LENGTHS.with(|m| m.borrow_mut().insert(r.num, length)); + LENGTHS.with(|m| m.borrow_mut().insert(r.num, (length, overlay_reads() != edits_read))); length }; let off = offset as usize; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 286973943e..29e2ea4879 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -6,8 +6,9 @@ thread_local! { static LOADING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; /// Indirect `/Length` values resolved during the current outermost load: a damaged file /// retries a parse several ways, and each retry would resolve the same lengths again - /// (exponential in the chain of streams whose lengths point at further streams). - pub(super) static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); + /// (exponential in the chain of streams whose lengths point at further streams). Each is + /// kept with whether resolving it read unsaved edits. + pub(super) static LENGTHS: std::cell::RefCell, bool)>> = std::cell::RefCell::new(HashMap::new()); /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 15c9d8cd61..84dc17cc7f 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -648,6 +648,64 @@ fn an_edited_object_stream_length_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); } +/// A `/Length` read through one clone's edits and reused within the same load still keeps that +/// load to the clone. Here the file's offsets are header-relative behind junk, so object 10's +/// offset first lands on object 12, whose length (object 8, in the edited object stream 9) is +/// read; object 10's own length then loads object stream 4, framed by object 8 too. +#[test] +fn a_reused_edited_length_leaves_other_clones_members_alone() { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 13]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let stream = |dict: &str, data: &str| format!("<< {dict} >>\nstream\n{data}\nendstream").into_bytes(); + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let inner = "11 0 3 2 5 << /Hidden (cut endstream here) >>"; + obj(&mut out, 4, &stream("/Type /ObjStm /N 2 /First 9 /Length 8 0 R", inner)); + let lengths = format!("8 0 {}", inner.len()); + obj(&mut out, 9, &stream(&format!("/Type /ObjStm /N 1 /First 4 /Length {}", lengths.len()), &lengths)); + obj(&mut out, 12, &stream("/Length 8 0 R", &"x".repeat(40))); + obj(&mut out, 10, &stream("/Length 11 0 R", "PDATA")); + // A cross-reference stream: a hybrid file's /XRefStm is not looked for behind junk. + let compressed = [(3, 4, 1), (8, 9, 0), (11, 4, 0)]; + let mut data = Vec::new(); + for (n, off) in offs.iter().enumerate() { + let row = match compressed.iter().find(|c| c.0 == n) { + Some(&(_, stream, index)) => [2, 0, stream, index], + None if *off == 0 => [0, 0, 0, 0], + None => { + let [.., hi, lo] = u32::try_from(*off).unwrap().to_be_bytes(); + [1, hi, lo, 0] + } + }; + data.extend_from_slice(&row); + } + let table = out.len(); + let dict = format!("<< /Type /XRef /Size 14 /W [1 2 1] /Index [0 13] /Root 1 0 R /Length {} >>", data.len()); + out.extend_from_slice(format!("13 0 obj\n{dict}\nstream\n").as_bytes()); + out.extend_from_slice(&data); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{table}\n%%EOF\n").as_bytes()); + let mut file = vec![b'J'; offs[10] - offs[12] - 1]; + file.push(b'\n'); + file.extend_from_slice(&out); + let doc = Document::open(Arc::new(file)).unwrap(); + let mut edited = doc.clone(); + let cut = "11 0 3 2 5 << /Hidden (cut".len(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + edited.set(ObjRef::new(9, 0), crate::Stream::from_raw(dict, format!("8 0 {cut}").into_bytes())); + assert!(matches!(&*edited.get(ObjRef::new(10, 0)), Object::Stream(s) if &s.raw[..] == b"PDATA")); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { From 649e97930f0f516ac684393f9b16e5c3c00120a6 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 18:03:00 -0500 Subject: [PATCH 06/85] fix(cos): a rebuilt index keeps the read failures its scan found Rebuilding the cross-reference index from object headers reset the unreadable-object list a second time, after adding the object-stream members, so an object the header scan failed to read dropped off unreadable_objects() unless something read it again. Those failures still stand against the full index: the added members leave every scanned object's entry as it was, and a /Length found only then changes how much of a stream is read, never whether it reads. The list is now reset once, before the scan; the object cache is still cleared after it. Test: a_rebuilt_file_lists_the_objects_its_scan_could_not_read fails without the fix (the list is empty) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/reader.rs | 3 ++- crates/cos/src/document/tests.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 1f944fd870..d556c1423b 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -651,8 +651,9 @@ impl Document { } let object_count = entries.len(); self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries))); + // Failures from the scan stand: the members added change no scanned entry, and a + // `/Length` found only now cannot make a stream unreadable. self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); - *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a // usable /Root (last in the file, whichever kind), else the catalog itself. One whose // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index c400e029d2..b587471c62 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1736,6 +1736,17 @@ fn unreadable_objects_are_listed_with_the_reason() { assert!(sound.unreadable_objects().is_empty()); } +/// A file rebuilt from its object headers still lists an object the scan could not read. +#[test] +fn a_rebuilt_file_lists_the_objects_its_scan_could_not_read() { + let text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n3 0 obj\n<< /A ) >>\nendobj\n%%EOF\n"; + let doc = Document::open(Arc::new(text.as_bytes().to_vec())).unwrap(); + assert!(doc.repair_log().iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{:?}", doc.repair_log()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); +} + /// The list keeps at most 100 objects and then says there were more; an object whose /// cross-reference offset holds no object is listed with that as its reason. #[test] From bad9a8b44c89cf8384252fcc57a20ad8cf33041b Mon Sep 17 00:00:00 2001 From: KenM76 Date: Tue, 3 Nov 2026 12:47:31 -0500 Subject: [PATCH 07/85] cos: an edited object stream forgets its members' cached failures A member that failed to load from the file's object stream was cached as a failure. Replacing or freeing that stream left the failure in place, so the member kept reading as null instead of from the edit (CodeRabbit on #5802). set and free now drop the cached failures of the stream's members. Cached objects are untouched; a failure holds no bytes, so the cache's total stands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 11 +++++++++++ crates/cos/src/document/state.rs | 5 +++++ crates/cos/src/document/tests.rs | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 96d3a9dee4..619e64eb5f 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -642,6 +642,7 @@ impl Document { pub fn set(&mut self, r: ObjRef, obj: impl Into) { let obj = self.adopt(obj.into()); self.overlay.insert(r.num, Slot::Set(r.generation, Arc::new(obj))); + self.forget_member_failures(r.num); if r.num >= self.next_num { self.next_num = r.num.saturating_add(1); } @@ -669,6 +670,16 @@ impl Document { /// Delete an object (its number becomes free with the next generation). pub fn free(&mut self, r: ObjRef) { self.overlay.insert(r.num, Slot::Freed(r.generation.saturating_add(1))); + self.forget_member_failures(r.num); + } + + /// Forget cached failures of the members `stream` held in the file: they were failures of + /// the file's copy, and an edit of `stream` may read. Clones share the cache, so theirs are + /// searched again too, which costs time only. A poisoned lock leaves the cache as it is. + fn forget_member_failures(&self, stream: u32) { + if let Ok(mut cache) = self.cache.lock() { + cache.forget_failures(|n| matches!(self.entries.lookup(*n, &self.data), Some(XrefEntry::InStream { stream: s, .. }) if s == stream)); + } } /// Free what was added since the file was read and nothing refers to any more: the objects diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 5f9bb49852..b3e6c1314a 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -250,6 +250,11 @@ impl ObjectCache { self.owned = 0; } + /// Drop the cached failures `stale` picks. A failure holds no bytes, so the total stands. + pub(crate) fn forget_failures(&mut self, mut stale: impl FnMut(&u32) -> bool) { + self.entries.retain(|num, (loaded, _)| loaded.is_ok() || !stale(num)); + } + pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) { self.entries.retain(|num, _| keep(num)); self.owned = self.entries.values().fold(0usize, |n, (_, owned)| n.saturating_add(*owned)); diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 3b18521a4e..62481df4bc 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -634,6 +634,26 @@ fn an_edited_object_stream_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); } +/// A member that failed to load from the file's object stream is kept as a failure, but replacing +/// or freeing that stream forgets it: the member then reads from the edit. +#[test] +fn an_edited_object_stream_forgets_its_members_failures() { + let file = Arc::new(object_stream_file("3 0 ", ") broken", 0)); + let mut doc = Document::open(file.clone()).unwrap(); + assert!(doc.try_get(3).is_err(), "the file's member does not parse"); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + doc.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (fixed) >>".to_vec())); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("fixed"), "the cached failure outlived the edit"); + assert!(doc.try_get(3).is_ok()); + let mut doc = Document::open(file).unwrap(); + assert!(matches!(doc.try_get(3), Err(CosError::Syntax { .. })), "the file's member does not parse"); + doc.free(ObjRef::new(4, 0)); + assert_eq!(doc.try_get(3), Err(CosError::MissingObject(4)), "the freed stream's member"); +} + /// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: /// the others still read its members within the file's length. #[test] From 57ca07dea3b56bfb678f7831bbc11767c03b476f Mon Sep 17 00:00:00 2001 From: KenM76 Date: Tue, 3 Nov 2026 12:51:45 -0500 Subject: [PATCH 08/85] cos: test that a clone's scan reaches an unread original The clone-scan check ran after the original had already listed the object, so it passed whether or not scan findings were shared (CodeRabbit on #5816). A fresh document is now cloned before anything reads it, only the clone is scanned, and the original must list the object. Seen failing with the scan's list unshared. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 50a69159f1..fda2128ebc 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1727,6 +1727,10 @@ fn unreadable_objects_are_listed_with_the_reason() { let scanned = doc.clone(); assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let unread = Document::open(Arc::new(with_unreadable_info())).unwrap(); + let scanned = unread.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(unread.unreadable_objects(), lines, "a clone's scan reaches the original"); let mut edited = doc.clone(); edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); From 7971fb73f2794cd9089838dd07365cfa9f41ca5a Mon Sep 17 00:00:00 2001 From: KenM76 Date: Wed, 4 Nov 2026 11:04:00 -0500 Subject: [PATCH 09/85] fix(cos): a clone's edited objects don't use up its unreadable-object allowance The unreadable list is shared by clones, and each one hides the entries it has edited. The 100-entry cap counted the whole shared list, so a clone that edited every listed object recorded none of its own failures and listed only "more objects could not be read". The cap now counts the entries the asking document still shows. Each document still lists at most 100 entries, then the "more" line. Test: objects_a_clone_edited_do_not_use_up_its_unreadable_allowance. It was seen failing with each half reverted (left: 1, right: 10; then left: 110, right: 101). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 5 +++-- crates/cos/src/document/state.rs | 30 ++++++++++++++++++------------ crates/cos/src/document/tests.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index f85b7b9191..6cf1df9b1e 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -103,8 +103,9 @@ impl Document { fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { // A poisoned lock only means the object goes unlisted; the read itself still fails. let Ok(mut unreadable) = self.unreadable.lock() else { return }; - if unreadable.wants(num) { - unreadable.note(num, self.generation(num), &reason()); + let hidden = |n: u32| self.overlay.contains_key(&n); + if unreadable.wants(num, hidden) { + unreadable.note(num, self.generation(num), &reason(), hidden); } } diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index a6171479ca..3627280960 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -62,7 +62,9 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: us } /// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). -/// Bounded: past [`Unreadable::MAX`] objects only the fact that there were more is kept. +/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that +/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole +/// list holds at most `MAX` plus the objects edited in clones. #[derive(Debug, Default)] pub(super) struct Unreadable { pub(super) objects: BTreeMap, @@ -74,13 +76,15 @@ impl Unreadable { /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. const REASON: usize = 160; - /// Whether `num` is to be recorded: not listed yet and there is room. A full list notes that - /// there were more, so a caller asks this before building the reason. - pub(super) fn wants(&mut self, num: u32) -> bool { + /// Whether `num` is to be recorded: not listed yet and there is room among the entries the + /// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]), + /// so a clone's edits never use up its own allowance. A full list notes that there were more, + /// so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool { if self.objects.contains_key(&num) { return false; } - if self.objects.len() >= Self::MAX { + if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX { self.more = true; return false; } @@ -88,21 +92,23 @@ impl Unreadable { } /// Record `num` once; the first reason given is kept. - pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str) { - if self.wants(num) { + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) { + if self.wants(num, hidden) { self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); } } /// One line per recorded object not `hidden`, then one saying there were more, if there were. pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { - let mut lines: Vec = self - .objects - .iter() - .filter(|(num, _)| !hidden(**num)) + // Clones share the list and each hides its own edits, so the list may hold more than + // `MAX` entries in all; each document still shows at most `MAX`. + let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num)); + let mut lines: Vec = shown + .by_ref() + .take(Self::MAX) .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) .collect(); - if self.more { + if self.more || shown.next().is_some() { lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); } lines diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 2f40b25b07..4dbc99bb4e 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1919,6 +1919,32 @@ fn the_unreadable_object_list_is_bounded() { ); } +/// The unreadable list is shared by clones, and each hides what it has edited. A clone that edited +/// every listed object still lists its own failures: hidden entries don't use up its allowance. +#[test] +fn objects_a_clone_edited_do_not_use_up_its_unreadable_allowance() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 110)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + for num in 3..103 { + assert!(doc.try_get(num).is_err()); + } + assert_eq!(doc.unreadable_objects().len(), 100); + let mut edited = doc.clone(); + for num in 3..103 { + edited.set(ObjRef::new(num, 0), Object::Null); + } + for num in 103..113 { + assert!(edited.try_get(num).is_err()); + } + let lines = edited.unreadable_objects(); + assert_eq!(lines.len(), 10, "{lines:?}"); + assert!(lines[0].starts_with("object 103 0 could not be read"), "{}", lines[0]); + let original = doc.unreadable_objects(); + assert_eq!(original.len(), 101, "the original still shows at most 100, then says there were more"); + assert_eq!(original[100], "more objects could not be read; only the first 100 are listed"); +} + /// The cache keeps the sum of what its objects hold as they come and go, so neither reader has /// to measure the cache again on every read (#5873). #[test] From 352355ab77163db6c7bfecdac0f0ca96a5f6aacf Mon Sep 17 00:00:00 2001 From: KenM76 Date: Wed, 4 Nov 2026 11:58:06 -0500 Subject: [PATCH 10/85] M10: copy keeps formatting as HTML (Copy with formatting) Copying selected page text now also puts an HTML fragment on the system clipboard, so a paste into a word processor or mail client keeps paragraphs, bold, italic, size, colour and font family. The plain text is unchanged and stays on the clipboard as the alternative. - export: clip.rs, the clipboard model (ClipText of paragraphs of styled spans), its builder with caps (20 000 spans, 2 MiB of text, 16 MiB of HTML), font_family (subset tag and style suffixes off, camel case spaced) and clip_html. Family names are reduced to letters, digits, space, '-', '_' and '.', so a font name cannot leave the CSS string; text goes through esc, which drops control characters. - engine: Document::rich_copy(page, text, glyphs). Glyphs (view space) take the style of the reading block their centre lies in; blocks are style-homogeneous. A line wrap inside a block is a space (none after a hyphen), a change of block a new paragraph. Refused when the document does not allow copying. compare.rs's display unit and weight/slant inference are shared, not duplicated. - automation: text_extract takes format "text" (default) or "html"; html adds each page's html and styles_incomplete, for the whole page or only what rect takes. Other values are InvalidArgs. - ui-egui: Copy of a run or column selection writes HTML plus the plain text in one arboard set (desktop only); the web build and Select All of the whole document stay plain text. last_copied_html records what was produced, for tests. - parity: view.copy-with-layout is partial; notes list what is missing (RTF, underline/strike-through, tables, Select All, web, glyphs outside reading blocks). Formatting inference and the HTML serialiser are adapted from pdfcer (MIT, same author) at 540b00ef1a405d1baa67c92f3858535037c7cde8. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/automation/src/pages.rs | 22 +- crates/automation/src/tools.rs | 5 +- crates/automation/tests/it/automation.rs | 30 +++ crates/engine/src/compare.rs | 24 +- crates/engine/src/lib.rs | 2 + crates/engine/src/rich_copy.rs | 220 ++++++++++++++++ crates/export/src/clip.rs | 322 +++++++++++++++++++++++ crates/export/src/lib.rs | 2 + crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/app/clipboard.rs | 23 +- crates/ui-egui/src/canvas/selection.rs | 15 ++ crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/tests/it/text_select.rs | 56 +++- parity/acrobat-features.toml | 10 +- 14 files changed, 718 insertions(+), 17 deletions(-) create mode 100644 crates/engine/src/rich_copy.rs create mode 100644 crates/export/src/clip.rs diff --git a/crates/automation/src/pages.rs b/crates/automation/src/pages.rs index 8c793be153..ad6a819027 100644 --- a/crates/automation/src/pages.rs +++ b/crates/automation/src/pages.rs @@ -201,6 +201,11 @@ impl Automation { } pub(crate) fn text_extract(&mut self, a: &Args) -> Result { + let html = match a.opt_str("format")?.unwrap_or("text") { + "text" => false, + "html" => true, + other => return Err(ToolError::InvalidArgs(format!("format must be \"text\" or \"html\", not {other:?}"))), + }; let doc = self.doc_to_copy(a)?; let id = doc.id; let pages = match a.opt_ints("pages")? { @@ -214,7 +219,22 @@ impl Automation { Some(r) => t.column_text(&t.glyphs_in(r)), None => t.plain_text(), }; - let out: Vec = pages.iter().zip(texts).map(|(p, t)| json!({ "page": p + 1, "text": text(&t) })).collect(); + let doc = self.doc(a)?; + let mut out = Vec::with_capacity(pages.len()); + for (&p, t) in pages.iter().zip(texts) { + let mut page = json!({ "page": p + 1, "text": text(&t) }); + if html { + // Copy with formatting: the same glyphs as an HTML fragment. + let glyphs: Vec = match rect { + Some(r) => t.glyphs_in(r), + None => (0..t.glyphs.len()).collect(), + }; + let copy = doc.rich_copy(p, &t, &glyphs).map_err(failed)?; + page["html"] = json!(copy.html); + page["styles_incomplete"] = json!(copy.styles_incomplete); + } + out.push(page); + } Ok(json!({ "pages": out })) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 1b4b6ac4ba..dae824cf5b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -195,13 +195,14 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc(), "pages": pages("to count (default: all)"), "exclude_tables": { "type": "boolean" }, "exclude_code": { "type": "boolean" }, "exclude_images": { "type": "boolean" } }), &["doc"])), - t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab.") + t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab. With format \"html\", each page also carries html, the same text as Copy with formatting puts on the clipboard (paragraphs, bold, italic, size, colour and font family), and styles_incomplete when some styles had to be left out.") .ro() .cmd("edit.column_select") .with(schema( json!({ "doc": doc(), "pages": pages("to extract (default: all)"), - "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." } + "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." }, + "format": { "type": "string", "enum": ["text", "html"], "description": "text (default) or html: also return each page's text as formatted HTML." } }), &["doc"], )), diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 13f9af4371..72abdf71a5 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -2420,6 +2420,7 @@ fn protecting_through_tools() { // Copying wasn't allowed either (#568): no text or images come out, though search works. for (tool, args) in [ ("text_extract", json!({ "doc": re })), + ("text_extract", json!({ "doc": re, "format": "html" })), ("text_word_count", json!({ "doc": re })), ("text_paragraphs", json!({ "doc": re, "page": 1 })), ("doc_export_text", json!({ "doc": re, "path": "out.txt" })), @@ -8978,6 +8979,35 @@ fn text_extract_with_a_rect_takes_one_column() { } } +/// Copy with formatting, headless: `format: "html"` adds each page's text as the HTML fragment +/// Copy puts on the clipboard, for the whole page or only what `rect` takes. +#[test] +fn text_extract_as_html_keeps_the_formatting() { + let dir = workdir("rich-copy"); + std::fs::write(dir.join("table.pdf"), table_pdf()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "table.pdf" }))["doc"].as_u64().unwrap(); + let page = &ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "format": "html" }))["pages"][0]; + let html = page["html"].as_str().unwrap(); + assert!(html.starts_with("

Name"), "{html}"); + assert!(html.contains("Pear") && html.contains("Qty"), "{html}"); + assert_eq!(page["text"], "Name\nApple\nPear\nQty\n12\n7", "the plain text is unchanged"); + assert_eq!(page["styles_incomplete"], false); + // Plain text by default: no html key. + assert!(ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1] }))["pages"][0].get("html").is_none()); + // Only what the rect takes; nothing inside gives empty HTML. + let column = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [120, 0, 300, 300], "format": "html" })); + let html = column["pages"][0]["html"].as_str().unwrap(); + assert!(html.contains("Qty") && !html.contains("Name"), "{html}"); + let none = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [0, 0, 5, 5], "format": "html" })); + assert_eq!(none["pages"][0]["html"], ""); + // An unknown format is refused. + for bad in [json!("rtf"), json!(1)] { + let e = a.call("text_extract", &json!({ "doc": doc, "format": bad })).unwrap_err(); + assert!(matches!(e, ToolError::InvalidArgs(_)), "{bad}: {e}"); + } +} + #[test] fn mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools() { use pdfcraft_cos::{Dict, Document, Object, Stream, write_incremental}; diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..fe5af55706 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -187,6 +187,20 @@ pub struct OfficeExport { /// `Err(false)`: it can't be encoded). type PreparedImage = Result<(&'static str, Arc<[u8]>), bool>; +/// Displayed points per user-space unit on `info`'s page (`/UserUnit`), 1 when the page has no +/// usable size. +pub(crate) fn display_unit(info: &pdfcraft_render::PageInfo) -> f64 { + let crop_width = f64::from(info.crop[2] - info.crop[0]); + let display_width = f64::from(if info.rotation.is_multiple_of(180) { info.width } else { info.height }); + if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 } +} + +/// Bold and italic as a font's name says them (`Arial-BoldItalicMT`). +pub(crate) fn weight_and_slant(base_font: &str) -> (bool, bool) { + let f = base_font.to_ascii_lowercase(); + (f.contains("bold") || f.contains("black") || f.contains("heavy"), f.contains("italic") || f.contains("oblique")) +} + impl crate::Document { /// The pages as paragraphs and images (for Word, HTML and RTF export), including what /// form XObjects draw. @@ -228,9 +242,7 @@ impl crate::Document { let b = info.user_to_view(r[2] as f32, r[3] as f32); [a[0].min(b[0]) as f64, (info.height - a[1].max(b[1])) as f64, a[0].max(b[0]) as f64, (info.height - a[1].min(b[1])) as f64] }; - let crop_width = f64::from(info.crop[2] - info.crop[0]); - let display_width = f64::from(if info.rotation % 180 == 0 { info.width } else { info.height }); - let unit = if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 }; + let unit = display_unit(info); let reading = pdfcraft_edit::reading_blocks_checked(cos, i).unwrap_or_default(); let forms_unread = reading.truncated; let blocks = reading @@ -238,7 +250,7 @@ impl crate::Document { .into_iter() .filter(|b| !b.text.trim().is_empty()) .map(|b| { - let f = b.base_font.to_ascii_lowercase(); + let (bold, italic) = weight_and_slant(&b.base_font); // `text` is trimmed; the editor text keeps the source's own spaces. let space_around = [b.editor_text.starts_with(char::is_whitespace), b.editor_text.ends_with(char::is_whitespace)]; pdfcraft_export::Block { @@ -246,8 +258,8 @@ impl crate::Document { text: b.text, rect: visible_rect(b.rect), size: b.size * unit, - bold: f.contains("bold") || f.contains("black") || f.contains("heavy"), - italic: f.contains("italic") || f.contains("oblique"), + bold, + italic, color: b.color, rotation: b.rotation - f64::from(info.rotation), } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index fe139164be..bf142bd037 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -43,6 +43,7 @@ mod pictures; pub mod print_raster; pub mod printcheck; mod protection; +mod rich_copy; mod scan_colours; mod scan_fields; mod session; @@ -66,6 +67,7 @@ pub use links::{ MAX_REMOTE_DOCUMENT_BYTES, RemoteDocumentPathError, read_remote_document, remote_document_path_is_absolute, resolve_remote_document_path, }; pub use pdfcraft_edit::strip_subset_tag; +pub use rich_copy::RichCopy; pub use signature_image::{ImageSignaturePreview, SignatureImage}; pub use signature_scan::{PhotoRect, ScanError, SignaturePhoto, Turn}; pub use stamp_preview::{CommentLayer, StampPreview}; diff --git a/crates/engine/src/rich_copy.rs b/crates/engine/src/rich_copy.rs new file mode 100644 index 0000000000..99c2938350 --- /dev/null +++ b/crates/engine/src/rich_copy.rs @@ -0,0 +1,220 @@ +//! Copy with formatting: the selected glyphs of a page as paragraphs of styled spans and an +//! HTML fragment ([`pdfcraft_export::clip_html`]) for the clipboard, next to the plain text. +//! +//! Glyphs come from the text layer (view space); style comes from the page's reading blocks, +//! which never mix styles, mapped into view space. A glyph takes the style of the block its +//! centre lies in; a glyph in no block is copied unstyled. + +use pdfcraft_export::{ClipBuilder, ClipStyle, ClipText, clip_html}; +use pdfcraft_render::{PageInfo, PageText}; + +use crate::EditError; +use crate::compare::{display_unit, weight_and_slant}; + +/// Block tests spent matching glyphs to blocks for one copy; past it the rest is unstyled. +const MAX_MATCH_STEPS: usize = 20_000_000; +/// How far outside a block's box (points) a glyph centre still belongs to it. +const SLACK: f32 = 0.5; + +/// Selected text with its formatting. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RichCopy { + /// The paragraphs and spans. + pub text: ClipText, + /// The HTML fragment for the clipboard (empty when nothing was selected). + pub html: String, + /// Styles were left out: reading stopped at its limits on this page, or matching glyphs to + /// blocks ran out of steps. + pub styles_incomplete: bool, +} + +struct StyledBlock { + rect: [f32; 4], + style: ClipStyle, +} + +impl crate::Document { + /// Selected `glyphs` of `page` (indices into `text`, in copy order) with their formatting. + /// Refused when the document does not allow copying. + pub fn rich_copy(&self, page: usize, text: &PageText, glyphs: &[usize]) -> Result { + self.check_copying()?; + let (blocks, mut incomplete) = self.styled_blocks(page); + let mut b = ClipBuilder::new(); + let mut steps = MAX_MATCH_STEPS; + // The previous glyph's line and block, and whether it was a hyphen. + let mut prev: Option<(usize, Option, bool)> = None; + for &i in glyphs { + let (Some(g), Some(&line)) = (text.glyphs.get(i), text.line_of.get(i)) else { continue }; + let found = block_at(&blocks, g.rect, prev.and_then(|(_, b, _)| b), &mut steps); + incomplete |= steps == 0; + let line = usize::try_from(line).unwrap_or(usize::MAX); + if let Some((prev_line, prev_block, hyphen)) = prev { + if line != prev_line { + if found == prev_block && found.is_some() { + // A paragraph wrapping: one space, none after a hyphen. + if !hyphen { + b.space(); + } + } else if found.is_none() && prev_block.is_none() { + b.line_break(); + } else { + b.paragraph(); + } + } else if text.space_before.get(i).copied().unwrap_or(false) { + b.space(); + } + } + let style = found.and_then(|k| blocks.get(k)).map(|s| &s.style); + if !b.push(&g.text, style.unwrap_or(&ClipStyle::default())) { + break; + } + prev = Some((line, found, g.text.ends_with('-'))); + } + let text = b.finish(); + Ok(RichCopy { html: clip_html(&text), text, styles_incomplete: incomplete }) + } + + /// `page`'s reading blocks in view space with their style, and whether reading stopped at + /// its limits. + fn styled_blocks(&self, page: usize) -> (Vec, bool) { + let (Some(editor), Some(info)) = (self.editor.as_ref(), self.info.pages.get(page)) else { return (Vec::new(), false) }; + let Ok(reading) = pdfcraft_edit::reading_blocks_checked(&editor.cos, page) else { return (Vec::new(), true) }; + let unit = display_unit(info); + let blocks = reading + .blocks + .into_iter() + .filter(|b| !b.text.trim().is_empty()) + .map(|b| { + let (bold, italic) = weight_and_slant(&b.base_font); + let style = ClipStyle { + family: pdfcraft_export::font_family(&b.base_font), + size: b.size * unit, + bold: b.bold || bold, + italic: b.italic || italic, + color: b.color, + }; + StyledBlock { rect: view_rect(info, b.rect), style } + }) + .collect(); + (blocks, reading.truncated) + } +} + +/// A user-space box as a view-space box (points from the displayed page's top-left, y down). +fn view_rect(info: &PageInfo, r: [f64; 4]) -> [f32; 4] { + let a = info.user_to_view(r[0] as f32, r[1] as f32); + let b = info.user_to_view(r[2] as f32, r[3] as f32); + [a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])] +} + +/// The block whose box holds the centre of `glyph` (view space): `hint` first, then the +/// smallest that holds it. Each block tested costs a step; with none left, no block. +fn block_at(blocks: &[StyledBlock], glyph: [f32; 4], hint: Option, steps: &mut usize) -> Option { + let (x, y) = ((glyph[0] + glyph[2]) / 2.0, (glyph[1] + glyph[3]) / 2.0); + let holds = |r: &[f32; 4]| x >= r[0] - SLACK && x <= r[2] + SLACK && y >= r[1] - SLACK && y <= r[3] + SLACK; + let area = |r: &[f32; 4]| (r[2] - r[0]) * (r[3] - r[1]); + if let Some(k) = hint + && *steps > 0 + { + *steps -= 1; + if blocks.get(k).is_some_and(|b| holds(&b.rect)) { + return Some(k); + } + } + let mut best: Option<(usize, f32)> = None; + for (k, b) in blocks.iter().enumerate() { + if *steps == 0 { + return None; + } + *steps -= 1; + if holds(&b.rect) && best.is_none_or(|(_, a)| area(&b.rect) < a) { + best = Some((k, area(&b.rect))); + } + } + best.map(|(k, _)| k) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::Session; + + /// A heading in bold, a body paragraph of two lines with HTML-special characters, and a red + /// word on the body's first line. + fn styled_page() -> Vec { + let content = b"BT /F1 24 Tf 20 250 Td (Title) Tj ET \ + BT /F2 12 Tf 20 200 Td (Body text) Tj ET \ + BT 1 0 0 rg /F2 12 Tf 90 200 Td (red) Tj ET \ + BT /F2 12 Tf 20 186 Td (a>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 300] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> >>") + .stream("", content) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .build() + } + + fn page_text(s: &Session, id: crate::DocId) -> Arc { + let doc = s.get(id).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), Default::default()); + let request = pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; + r.render(request).text.unwrap() + } + + #[test] + fn copy_keeps_headings_body_and_colour_as_html() { + let mut s = Session::new(); + let id = s.open("styled.pdf", None, Arc::new(styled_page()), None).unwrap(); + let text = page_text(&s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + let copy = s.get(id).unwrap().rich_copy(0, &text, &all).unwrap(); + let html = ©.html; + assert!(!copy.styles_incomplete); + assert!( + html.starts_with("

Title

\n"), + "{html}" + ); + assert!(html.contains("font-size:12pt\">Body text "), "{html}"); + assert!(html.contains("font-size:12pt;color:#FF0000\">red"), "{html}"); + assert!(html.contains("a<b & c"), "{html}"); + assert!(!html.contains("= 2, "{html}"); + // The heading and the body are separate paragraphs; the red word shares the body's. + let body = copy.text.paragraphs.iter().find(|p| p.iter().any(|s| s.text.contains("Body"))).unwrap(); + assert!(body.iter().any(|s| s.text == "red" && s.style.color == [1.0, 0.0, 0.0]), "{body:?}"); + + // A part of the selection: only those glyphs. + let part = s.get(id).unwrap().rich_copy(0, &text, &all[..5]).unwrap(); + assert_eq!(part.text.paragraphs.len(), 1); + assert!(part.html.contains("Title"), "{}", part.html); + // Nothing selected, and indices past the text: an empty copy, not a panic. + let none = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap(); + assert!(none.html.is_empty() && none.text.paragraphs.is_empty()); + let past = s.get(id).unwrap().rich_copy(7, &text, &[usize::MAX, text.glyphs.len()]).unwrap(); + assert!(past.html.is_empty()); + } + + #[test] + fn copy_with_formatting_is_refused_when_copying_is_withheld() { + let mut cos = pdfcraft_cos::Document::open(Arc::new(styled_page())).unwrap(); + cos.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0b0100, // print only + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(pdfcraft_cos::write_full(&cos, &Default::default()).unwrap()); + let mut s = Session::new(); + let id = s.open("locked.pdf", None, bytes, None).unwrap(); + let text = pdfcraft_render::PageText { glyphs: Vec::new(), line_of: Vec::new(), space_before: Vec::new() }; + let err = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap_err(); + assert!(matches!(err, crate::EditError::NotPermitted(_)), "{err:?}"); + } +} diff --git a/crates/export/src/clip.rs b/crates/export/src/clip.rs new file mode 100644 index 0000000000..0b214c2bcd --- /dev/null +++ b/crates/export/src/clip.rs @@ -0,0 +1,322 @@ +//! Copy with formatting: selected text as paragraphs of styled spans, written as an HTML +//! fragment for the clipboard (inline styles, so a word processor keeps font, size, weight, +//! slant and colour when it pastes). + +use crate::{esc, hex}; + +/// Spans kept for one copy; text past it is left out and [`ClipText::truncated`] is set. +pub const MAX_CLIP_SPANS: usize = 20_000; +/// Text bytes kept for one copy (before escaping). +pub const MAX_CLIP_TEXT_BYTES: usize = 2 << 20; +/// The HTML written for one copy never passes this many bytes. +pub const MAX_CLIP_HTML_BYTES: usize = 16 << 20; +/// A family name longer than this is not a family name. +const MAX_FAMILY_CHARS: usize = 64; + +/// How a span looks. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipStyle { + /// The family a word processor knows ([`font_family`]); `None` keeps the target's own. + pub family: Option, + /// Size in points; zero or not finite leaves the target's size. + pub size: f64, + pub bold: bool, + pub italic: bool, + /// Fill colour as RGB in 0..=1. + pub color: [f64; 3], +} + +/// A run of text in one style. +#[derive(Clone, Debug, PartialEq)] +pub struct ClipSpan { + pub text: String, + pub style: ClipStyle, +} + +/// The copied text: paragraphs of spans, and whether a cap left some out. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipText { + pub paragraphs: Vec>, + pub truncated: bool, +} + +/// Builds a [`ClipText`] piece by piece, merging pieces of equal style into one span and +/// stopping at the caps. +#[derive(Debug, Default)] +pub struct ClipBuilder { + out: ClipText, + /// The separator owed before the next piece in this paragraph (a space or a line break). + pending: Option, + new_paragraph: bool, + spans: usize, + bytes: usize, +} + +impl ClipBuilder { + pub fn new() -> Self { + Self { new_paragraph: true, ..Self::default() } + } + + /// A word space before the next piece (dropped at the start of a paragraph). + pub fn space(&mut self) { + self.pending.get_or_insert(' '); + } + + /// A line break inside the paragraph before the next piece. + pub fn line_break(&mut self) { + self.pending = Some('\n'); + } + + /// The next piece starts a new paragraph. + pub fn paragraph(&mut self) { + self.new_paragraph = true; + self.pending = None; + } + + /// Append `text` in `style`. Returns `false` once a cap is reached; nothing more is kept. + pub fn push(&mut self, text: &str, style: &ClipStyle) -> bool { + if self.out.truncated { + return false; + } + if text.is_empty() { + return true; + } + let sep = if self.new_paragraph { None } else { self.pending }; + let need = text.len().saturating_add(sep.map_or(0, char::len_utf8)); + let Some(bytes) = self.bytes.checked_add(need).filter(|b| *b <= MAX_CLIP_TEXT_BYTES) else { + self.out.truncated = true; + return false; + }; + if self.new_paragraph { + self.out.paragraphs.push(Vec::new()); + self.new_paragraph = false; + } + self.pending = None; + let Some(spans) = self.out.paragraphs.last_mut() else { return false }; + match spans.last_mut() { + Some(last) if last.style == *style => { + last.text.extend(sep); + last.text.push_str(text); + } + last => { + if self.spans >= MAX_CLIP_SPANS { + self.out.truncated = true; + return false; + } + // A separator belongs to the span before it, so a new style starts at its text. + if let (Some(last), Some(sep)) = (last, sep) { + last.text.push(sep); + } + spans.push(ClipSpan { text: text.to_owned(), style: style.clone() }); + self.spans += 1; + } + } + self.bytes = bytes; + true + } + + pub fn finish(self) -> ClipText { + self.out + } +} + +/// The family a word processor knows, from a font's `/BaseFont`: subset tag and style suffix +/// removed, `PSMT`/`MT`/`PS` dropped, camel case spaced (`ABCDEF+TimesNewRomanPSMT` → +/// `Times New Roman`). Only letters, digits, spaces, `-`, `_` and `.` are kept, so the name +/// cannot leave its CSS string. `None` when nothing is left. +pub fn font_family(base_font: &str) -> Option { + let name = match base_font.split_once('+') { + Some((tag, rest)) if tag.len() == 6 && tag.bytes().all(|b| b.is_ascii_uppercase()) => rest, + _ => base_font, + }; + let family = name.split_once([',', '-']).map_or(name, |(family, _)| family); + let family = family.strip_suffix("PSMT").or_else(|| family.strip_suffix("MT")).or_else(|| family.strip_suffix("PS")).unwrap_or(family); + let mut spaced = String::with_capacity(family.len().min(MAX_FAMILY_CHARS * 2)); + let mut prev_lower = false; + for c in family.chars().take(MAX_FAMILY_CHARS) { + if prev_lower && c.is_ascii_uppercase() { + spaced.push(' '); + } + prev_lower = c.is_ascii_lowercase(); + spaced.push(c); + } + safe_family(&spaced) +} + +/// `family` with only letters, digits, spaces, `-`, `_` and `.`, so it cannot leave a CSS +/// string; `None` when nothing is left. +fn safe_family(family: &str) -> Option { + let kept: String = family.chars().filter(|c| c.is_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.')).take(MAX_FAMILY_CHARS * 2).collect(); + let kept = kept.trim(); + (!kept.is_empty()).then(|| kept.to_owned()) +} + +/// The copied text as an HTML fragment: one `

` per paragraph, a `` per run with its +/// family, size and colour inline, bold as `` and italic as `` (so a target that +/// drops style attributes still keeps them). Text is escaped; a line break is `
` and a run +/// of spaces keeps its width. Never longer than [`MAX_CLIP_HTML_BYTES`]: a span that would pass +/// it ends the fragment. +pub fn clip_html(text: &ClipText) -> String { + const OPEN: &str = "

"; + const CLOSE: &str = "

\n"; + let mut out = String::new(); + for spans in &text.paragraphs { + let mut p = String::from(OPEN); + for span in spans { + let s = span_html(span); + if out.len().saturating_add(p.len()).saturating_add(s.len()).saturating_add(CLOSE.len()) > MAX_CLIP_HTML_BYTES { + if p.len() > OPEN.len() { + out.push_str(&p); + out.push_str(CLOSE); + } + return out; + } + p.push_str(&s); + } + p.push_str(CLOSE); + out.push_str(&p); + } + out +} + +fn span_html(span: &ClipSpan) -> String { + let st = &span.style; + let mut css = Vec::new(); + if let Some(f) = st.family.as_deref().and_then(safe_family) { + css.push(format!("font-family:'{f}'")); + } + if st.size.is_finite() && st.size > 0.0 { + css.push(format!("font-size:{}pt", num(st.size))); + } + if let Some(h) = hex(st.color) { + css.push(format!("color:#{h}")); + } + let mut body = text_html(&span.text); + if st.italic { + body = format!("{body}"); + } + if st.bold { + body = format!("{body}"); + } + if css.is_empty() { body } else { format!("{body}", esc(&css.join(";"))) } +} + +/// Escape for HTML text (control characters dropped by [`esc`]); `\n` becomes `
` and a +/// run of spaces keeps its width with ` `. +fn text_html(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut prev_space = false; + for (k, line) in text.split('\n').enumerate() { + if k > 0 { + out.push_str("
"); + prev_space = false; + } + for c in line.chars() { + if c == ' ' && prev_space { + out.push_str(" "); + } else { + out.push_str(&esc(c.encode_utf8(&mut [0; 4]))); + } + prev_space = c == ' '; + } + } + out +} + +/// A size with at most one decimal and no trailing `.0`. +fn num(v: f64) -> String { + let r = (v * 10.0).round() / 10.0; + if r.fract() == 0.0 { format!("{r:.0}") } else { format!("{r:.1}") } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn style(size: f64, bold: bool, italic: bool, color: [f64; 3]) -> ClipStyle { + ClipStyle { family: font_family("ABCDEF+Helvetica-Bold"), size, bold, italic, color } + } + + #[test] + fn heading_body_and_coloured_span_keep_their_formatting() { + let mut b = ClipBuilder::new(); + let heading = style(24.0, true, false, [0.0; 3]); + let body = style(11.5, false, false, [0.0; 3]); + let red = style(11.5, false, true, [1.0, 0.0, 0.0]); + b.push("Title", &heading); + b.paragraph(); + b.push("Body", &body); + b.space(); + b.push("text", &body); + b.space(); + b.push("red", &red); + let clip = b.finish(); + assert_eq!(clip.paragraphs.len(), 2); + assert_eq!(clip.paragraphs.get(1).map(Vec::len), Some(2), "{clip:?}"); + let html = clip_html(&clip); + assert_eq!( + html, + "

Title

\n\ +

Body text \ + red

\n" + ); + } + + #[test] + fn special_characters_are_escaped_and_cannot_leave_the_style() { + let st = ClipStyle { family: Some("Evil';}"#, - r#""#, + r#""#, ) .replace( r#"if (qty.rawValue === null) qty.rawValue = 2;"#, @@ -7337,9 +7338,11 @@ fn hostile_xfa_open_scripts_are_merged_capped_kept_from_printing_and_noted() { let mut s = Session::new().with_clock(|| 1_700_000_000); let started = std::time::Instant::now(); let id = s.open("hostile.pdf", None, bytes, None).expect("opens"); - assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + // 30 s and not 5 s: the run is wall-clock timed and a loaded test machine (all suites in + // parallel) can take several seconds for the 20 000 iterations; a real hang still fails. + assert!(started.elapsed() < std::time::Duration::from_secs(30), "{:?}", started.elapsed()); let doc = s.get(id).unwrap(); - assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["49999".to_string()]); + assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["19999".to_string()]); // One datasets stream for the scripts' revision, not one per value set. assert!(datasets_streams(&doc.bytes) <= 3, "{} datasets streams", datasets_streams(&doc.bytes)); // What opening changed is noted with the form's other XFA notes. diff --git a/crates/engine/tests/it/ocr_edit.rs b/crates/engine/tests/it/ocr_edit.rs index 53ca309326..2d358e327e 100644 --- a/crates/engine/tests/it/ocr_edit.rs +++ b/crates/engine/tests/it/ocr_edit.rs @@ -12,7 +12,14 @@ const WORD: [u32; 4] = [40, 40, 100, 60]; /// A 200 × 100 pt "scan" on tinted paper, with OCR text over its word. fn scanned() -> (Session, pdfcraft_engine::DocId) { - scanned_with(vec![PlacedWord { text: "John".into(), origin: [40.0, 40.0], across: [60.0, 0.0], up: [0.0, 20.0] }]) + scanned_with(vec![PlacedWord { + text: "John".into(), + origin: [40.0, 40.0], + across: [60.0, 0.0], + up: [0.0, 20.0], + confidence: None, + source: "ocrs".into(), + }]) } fn scanned_with(words: Vec) -> (Session, pdfcraft_engine::DocId) { @@ -86,11 +93,18 @@ fn an_explicit_colour_is_kept_for_the_new_text() { /// descender line when it has descenders, up to the cap height. fn scanned_word(text: &str, x0: f64, x1: f64, baseline: f64, size: f64) -> PlacedWord { let low = if text.chars().any(|c| "gjpqy".contains(c)) { 0.21 * size } else { 0.0 }; - PlacedWord { text: text.into(), origin: [x0, baseline - low], across: [x1 - x0, 0.0], up: [0.0, 0.718 * size + low] } + PlacedWord { + text: text.into(), + origin: [x0, baseline - low], + across: [x1 - x0, 0.0], + up: [0.0, 0.718 * size + low], + confidence: None, + source: "ocrs".into(), + } } fn word(text: &str, x0: f64, x1: f64, baseline: f64) -> PlacedWord { - PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0] } + PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0], confidence: None, source: "ocrs".into() } } /// OCR words keep the scan's layout: a phrase is one paragraph, a value in the next table cell and diff --git a/crates/fonts/README.md b/crates/fonts/README.md index b20b930a91..096f1a263e 100644 --- a/crates/fonts/README.md +++ b/crates/fonts/README.md @@ -59,7 +59,12 @@ set). Without that variable `CRAFT_FONTS` is empty and everything below copes: and shaped Thai Add Text content. Harfrust positions stacked vowel/tone marks and decomposes Sara Am; cluster ToUnicode entries preserve the original text. The shared Type 3 writer embeds outlines, so reopening does not depend on installed fonts. +- `craft_glyph(face, ch)` and `face_has(bytes, ch)`: bounded outlines and glyph coverage for any bundled face. - `SHIPPORI_MINCHO`: Shippori Mincho's bytes, or `None`. +- `TextPlan::for_words`: how the OCR output writes text beyond WinAnsi — per word the first + face (document face first) whose glyphs cover it, planned once per page into Type 3 fonts + (codes, advances, `?` stand-ins) that `pdfcraft-edit` registers and the `pdfcraft-ocr` layers + draw with; characters no face has are counted (`missing`) so the output reports them. Generated Japanese Type 3 fonts record the fallback face's family and style in an indirect `FontDescriptor`. `pdf::Metrics::base_font` uses its `FontName` when the Type 3 dictionary has diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index 61061f5509..45cb843ab0 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -14,6 +14,7 @@ pub mod embedding; mod encodings; pub mod fallback; pub mod pdf; +pub mod plan; mod script; mod shaping; pub mod source_shaping; @@ -24,8 +25,10 @@ pub use craft::{ document_japanese_font_for_style, document_japanese_fonts_for_style, document_latin_fonts, document_thai_font, ui_arabic_fonts, ui_chinese_fonts, ui_cjk_fonts, ui_devanagari_fonts, ui_hebrew_fonts, ui_japanese_fonts, ui_telugu_fonts, ui_thai_fonts, }; +pub use plan::{MAX_PLAN_GLYPHS, MAX_TYPE3_GLYPHS, PlannedType3, TextPlan}; pub use script::{ - GlyphError, GlyphOutline, INTER_REGULAR, MAX_SIGNATURE_CHARS, ScriptOutline, japanese_glyph, japanese_glyph_from, script_missing, script_outline, + GlyphError, GlyphOutline, INTER_REGULAR, MAX_SIGNATURE_CHARS, ScriptOutline, craft_glyph, face_has, japanese_glyph, japanese_glyph_from, + script_missing, script_outline, }; pub use shaping::{ ShapedCluster, ShapedScript, arabic_glyph, arabic_has, hebrew_has, shape_arabic, shape_hebrew, shape_thai, shaped_glyph, thai_has, @@ -273,6 +276,16 @@ pub fn set_undrawn_mark(d: &mut pdfcraft_cos::Dict, c: Option) { } } +/// The byte used by Unicode text plans, from the shared WinAnsi table (tabs become spaces). +pub fn win_ansi_char(c: char) -> Option { + if c == '\t' { Some(b' ') } else { win_ansi_byte(c) } +} + +/// Whether every character can be represented without substitution in WinAnsi. +pub fn fits_win_ansi(s: &str) -> bool { + s.chars().all(win_ansi_encodable) +} + /// Bytes as a PDF literal string, `(` … `)`, with delimiters escaped. pub fn literal(bytes: &[u8]) -> Vec { let mut out = Vec::with_capacity(bytes.len() + 2); diff --git a/crates/fonts/src/plan.rs b/crates/fonts/src/plan.rs new file mode 100644 index 0000000000..9c22a38fd3 --- /dev/null +++ b/crates/fonts/src/plan.rs @@ -0,0 +1,338 @@ +//! Planning visible Unicode text output for OCR: which characters the standard WinAnsi +//! fonts show as is, which need a Type3 font drawn from the craft-fonts faces, and which have +//! no glyph anywhere (written as `?` — and counted, never silently dropped). +//! +//! The plan is a pure function of the words ([`TextPlan::for_words`]). The content generators +//! (`pdfcraft_ocr::text_layer` / `visible_text_layer`) read the per-character codes and +//! advances from it, and whoever registers the fonts (`pdfcraft_edit::add_type3_fonts`) fills +//! in each face's resource name and object. The build must work without craft-fonts too: there +//! [`CRAFT_FONTS`] is empty, nothing can be planned, and every non-WinAnsi character is +//! reported as a substitution instead. + +use std::collections::HashMap; + +use pdfcraft_cos::ObjRef; + +use crate::{CraftFont, GlyphOutline, craft_glyph, document_japanese_font, face_has, fits_win_ansi, win_ansi_char}; + +/// Unique characters per Type3 font: the same bound the text-editing fallback uses, so one +/// hostile word cannot balloon into thousands of glyph streams. A face with more characters +/// simply gets another font. +pub const MAX_TYPE3_GLYPHS: usize = 240; + +/// Planned glyphs per plan in total (10 full fonts). A page's recognition cannot realistically +/// exceed it; hostile input beyond it degrades to `?` substitutions, which are counted. +pub const MAX_PLAN_GLYPHS: usize = 2400; + +/// One planned Type3 font: the glyphs of one craft-fonts face, one character per code +/// (`code = position + 1`, as in the text-editing fallback). `name` and `object` are filled in +/// when the font is added to a page's resources; until then the content generators fall back to +/// the WinAnsi path for the words that reference it. +pub struct PlannedType3 { + /// The face family the glyphs are drawn from ("Shippori Mincho"). + pub family: &'static str, + /// (character, outline with its advance) per code, in code order. + pub glyphs: Vec<(char, GlyphOutline)>, + /// The resource name the content references (`PCOcr`, `PCOcr1`, …); empty until registered. + pub name: String, + /// The built font object, shared by every page that uses the font; set at registration. + pub object: Option, +} + +impl PlannedType3 { + fn new(family: &'static str) -> Self { + PlannedType3 { family, glyphs: Vec::new(), name: String::new(), object: None } + } + + fn push(&mut self, ch: char, glyph: GlyphOutline) -> u8 { + self.glyphs.push((ch, glyph)); + self.glyphs.len() as u8 + } +} + +/// How the text layers write a plan's characters: each planned character is a +/// `(font index, code)` pair in its font's encoding; a character no face had a glyph for is +/// either the font's `?` code or absent from the run (both counted in [`TextPlan::missing`]). +#[derive(Default)] +pub struct TextPlan { + /// One planned font per face chunk, in first-use order. + pub fonts: Vec, + /// Per input word: `(font, code)` per character; empty for a word the standard WinAnsi + /// fonts write as is (or for which nothing could be planned). + words: Vec>, + /// character → where its glyph lives, across all fonts (codes are font-local). + index: HashMap, + /// Characters with no code in their word's font: they appear as `?` (or, when not even a + /// `?` glyph exists, are left out) and are reported, never silently lost. + missing: usize, +} + +impl TextPlan { + /// Plan the output for `words`, in order: characters the WinAnsi fonts can't show get codes + /// in Type3 fonts built from the craft-fonts faces (the document face preferred, then the + /// rest of the build input in manifest order); a word whose face can't show a character + /// substitutes its `?`. Faces are chosen per word, so a word of one script stays one face; + /// characters already planned for an earlier word keep their code. Built without + /// craft-fonts, the plan reports every non-WinAnsi character as a substitution. + pub fn for_words<'a>(words: impl IntoIterator) -> TextPlan { + let faces = document_faces(); + let mut coverage = Coverage::default(); + let mut plan = TextPlan::default(); + for word in words { + if fits_win_ansi(word) { + // The WinAnsi path substitutes '?' behind the caller's back; count it. + plan.missing += word.chars().filter(|&c| win_ansi_char(c).is_none()).count(); + plan.words.push(Vec::new()); + continue; + } + let Some(face) = choose_face(&faces, word, &mut coverage) else { + // No face for this word's script: the WinAnsi path writes '?' for what it + // can't encode. + plan.missing += word.chars().filter(|&c| win_ansi_char(c).is_none()).count(); + plan.words.push(Vec::new()); + continue; + }; + let mut enc: Vec<(usize, u8)> = Vec::with_capacity(word.len()); + let mut run: Option = None; + for ch in word.chars() { + if let Some(&where_) = plan.index.get(&ch) { + enc.push(where_); + run = Some(where_.0); + continue; + } + if let Ok(glyph) = craft_glyph(face, ch) + && plan.planned_glyphs() < MAX_PLAN_GLYPHS + && let Some((fi, code)) = plan.add_glyph(face, ch, glyph) + { + plan.index.insert(ch, (fi, code)); + enc.push((fi, code)); + run = Some(fi); + continue; + } + // The face has no glyph for the character (or the plan is full): substitute + // the font's '?', in the font the surrounding run is set in. + if let Some((fi, code)) = plan.question_mark(face, run.filter(|&f| plan.fonts[f].family == face.family)) { + enc.push((fi, code)); + run = Some(fi); + } + plan.missing += 1; + } + plan.words.push(enc); + } + plan + } + + /// The encoding of word `i` (parallel to the slice [`Self::for_words`] planned): a list of + /// `(font, code)` per character, or an empty slice for a word the WinAnsi fonts write. + /// Out-of-range indexes degrade to the empty slice rather than panicking — callers pass the + /// same word list they planned with. + pub fn encoding(&self, word: usize) -> &[(usize, u8)] { + self.words.get(word).map(Vec::as_slice).unwrap_or_default() + } + + /// How many characters no font could show (they appear as `?`, or are left out when the + /// word's face has no `?` glyph either). + pub fn missing(&self) -> usize { + self.missing + } + + /// The advance of one planned code, in em units (the Type3 glyphs' own widths). `0.0` for + /// anything out of range. + pub fn advance(&self, font: usize, code: u8) -> f64 { + self.fonts.get(font).and_then(|f| f.glyphs.get(usize::from(code).checked_sub(1)?)).map_or(0.0, |(_, g)| g.width) + } + + /// The sum of the advances of a word's planned codes, in em units. + pub fn advance_of(&self, encoding: &[(usize, u8)]) -> f64 { + encoding.iter().map(|&(f, c)| self.advance(f, c)).sum() + } + + /// Whether every font the encoding references has been registered (has a resource name). + /// Unregistered plans fall back to the WinAnsi path, the same output as before the + /// Unicode mapping existed. + pub fn registered(&self, encoding: &[(usize, u8)]) -> bool { + encoding.iter().all(|&(f, _)| self.fonts.get(f).is_some_and(|font| !font.name.is_empty())) + } + + /// Characters with a planned glyph so far. + fn planned_glyphs(&self) -> usize { + self.fonts.iter().map(|f| f.glyphs.len()).sum() + } + + /// Add a glyph to the face's current font (a fresh one when that font is full). + fn add_glyph(&mut self, face: &CraftFont, ch: char, glyph: GlyphOutline) -> Option<(usize, u8)> { + let fi = self.font_of_face(face.family)?; + let code = self.fonts[fi].push(ch, glyph); + Some((fi, code)) + } + + /// The index of the face's open font, or a fresh one. `None` without craft-fonts (the + /// face itself comes from there, so this cannot happen for a planned face). + fn font_of_face(&mut self, family: &'static str) -> Option { + if let Some(fi) = self.fonts.iter().rposition(|f| f.family == family && f.glyphs.len() < MAX_TYPE3_GLYPHS) { + return Some(fi); + } + if self.fonts.len() >= MAX_PLAN_GLYPHS / MAX_TYPE3_GLYPHS { + return None; + } + self.fonts.push(PlannedType3::new(family)); + Some(self.fonts.len() - 1) + } + + /// The code of the font's `?` stand-in, adding the glyph when the font lacks it. + /// `prefer` is the run's current font; `None` without craft-fonts. + fn question_mark(&mut self, face: &CraftFont, prefer: Option) -> Option<(usize, u8)> { + if let Some(fi) = prefer + && let Some(code) = self.fonts[fi].glyphs.iter().position(|(c, _)| *c == '?') + { + return Some((fi, code as u8 + 1)); + } + let fi = self.font_of_face(face.family)?; + if let Some(code) = self.fonts[fi].glyphs.iter().position(|(c, _)| *c == '?') { + return Some((fi, code as u8 + 1)); + } + let glyph = craft_glyph(face, '?').ok()?; + let code = self.fonts[fi].push('?', glyph); + Some((fi, code)) + } +} + +/// The faces that can draw document text, in preference order: the Japanese document face +/// (serif document text) first, then the rest of the build input in manifest order. +fn document_faces() -> Vec<&'static CraftFont> { + let mut out = Vec::new(); + if let Some(d) = document_japanese_font() { + out.push(d); + } + for f in crate::CRAFT_FONTS { + if !out.iter().any(|d| std::ptr::eq(*d, f)) { + out.push(f); + } + } + out +} + +/// The face for a word: the first (in preference order) whose glyphs cover every character, +/// else the one covering the most characters (ties go to the earlier face). `None` when no +/// face covers any character of the word. +fn choose_face<'f>(faces: &[&'f CraftFont], word: &str, coverage: &mut Coverage) -> Option<&'f CraftFont> { + if let Some(f) = faces.iter().enumerate().find(|(i, f)| word.chars().all(|c| coverage.has(*i, f, c))).map(|(_, f)| *f) { + return Some(f); + } + faces + .iter() + .enumerate() + .map(|(i, f)| (word.chars().filter(|&c| coverage.has(i, f, c)).count(), std::cmp::Reverse(i), *f)) + .max_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))) + .filter(|(covered, _, _)| *covered > 0) + .map(|(_, _, f)| f) +} + +/// Which faces have a glyph for which characters, remembered for one plan: a page of words +/// asks the same (face, character) pairs over and over, and each fresh answer parses the font. +#[derive(Default)] +struct Coverage { + known: std::collections::HashMap<(usize, char), bool>, + lookups: usize, +} + +impl Coverage { + /// Whether face number `i` (`face`) has `ch`; asks the font once per pair. + fn has(&mut self, i: usize, face: &CraftFont, ch: char) -> bool { + *self.known.entry((i, ch)).or_insert_with(|| { + self.lookups += 1; + face_has(face.bytes, ch) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Face coverage is asked once per (face, character): repeating words cost no new font + /// lookups. + #[test] + fn face_coverage_is_remembered_per_face_and_character() { + let faces = document_faces(); + let Some(face) = faces.first() else { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + }; + let mut coverage = Coverage::default(); + let first = (0..50).map(|_| choose_face(&faces, "日本語", &mut coverage)).last().flatten(); + assert!(first.is_some()); + assert!(coverage.lookups <= faces.len() * 3, "{} lookups for 3 characters", coverage.lookups); + assert_eq!(coverage.has(0, face, '日'), face_has(face.bytes, '日')); + } + + /// A page's vocabulary: German umlauts are WinAnsi, and a character no bundled face has + /// (here an emoji) is counted wherever it appears. + #[test] + fn win_ansi_words_get_no_encoding_and_count_only_real_losses() { + let plan = TextPlan::for_words(["Zürcher", "café", "size"]); + assert!(plan.words.iter().all(Vec::is_empty), "all WinAnsi: no Type3 fonts needed"); + assert!(plan.fonts.is_empty()); + assert_eq!(plan.missing(), 0); + let none = TextPlan::for_words(["plain text"]); + assert_eq!(none.missing(), 0); + // U+1F600 has no WinAnsi code and no glyph in any bundled face (with craft-fonts the + // JIS faces even cover the snowman): counted, shown as '?'. + let emoji = TextPlan::for_words(["\u{1f600}!"]); + assert_eq!(emoji.missing(), 1); + } + + #[test] + fn without_craft_fonts_every_non_win_ansi_character_is_reported() { + if !crate::CRAFT_FONTS.is_empty() { + eprintln!("built with craft-fonts: the planning tests with faces cover this instead"); + return; + } + let plan = TextPlan::for_words(["Привет", "日本", "abc"]); + assert!(plan.words.iter().all(Vec::is_empty)); + assert!(plan.fonts.is_empty()); + assert_eq!(plan.missing(), 8, "six Cyrillic + two CJK characters, each reported"); + } + + #[test] + fn encodings_are_aligned_and_total_the_advances() { + if crate::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + assert!(TextPlan::for_words(["日本"]).fonts.is_empty()); + return; + } + let plan = TextPlan::for_words(["日本語", "語"]); + assert_eq!(plan.encoding(0).len(), 3); + assert_eq!(plan.encoding(1).len(), 1); + // 語 planned once keeps its code (and font) in both words. + let (f, c) = plan.encoding(1)[0]; + assert_eq!(plan.encoding(0)[2], (f, c), "the repeated character keeps its code"); + assert!(plan.advance(f, c) > 0.0); + assert_eq!(plan.advance(f, 0), 0.0, "codes start at 1"); + assert_eq!(plan.encoding(7), &[], "out of range degrades to the WinAnsi path"); + assert!(plan.advance_of(plan.encoding(0)) > plan.advance_of(plan.encoding(1))); + } + + #[test] + fn hostile_input_stays_bounded() { + // Thousands of DISTINCT characters: the covered ones fill fonts up to the caps, the + // rest are reported substitutions. + let word: String = (0..4000u32).filter_map(|i| char::from_u32(0x4e00 + i)).collect(); + let plan = TextPlan::for_words([word.as_str()]); + assert!(plan.planned_glyphs() <= MAX_PLAN_GLYPHS); + assert!(plan.fonts.iter().all(|f| f.glyphs.len() <= MAX_TYPE3_GLYPHS)); + assert!(plan.missing() > 0, "characters beyond the caps are reported, never silently dropped"); + } + + #[test] + fn substitution_and_unknown_scripts_come_out_counted() { + if crate::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let plan = TextPlan::for_words(["日本\u{1f4a9}語"]); + // The emoji has no glyph in any bundled face; the kana/kanji do. + assert_eq!(plan.missing(), 1); + assert_eq!(plan.encoding(0).len(), 4, "the emoji still has an encoding: the font's ?"); + } +} diff --git a/crates/fonts/src/script.rs b/crates/fonts/src/script.rs index 70fc6ce092..94a8a93b8c 100644 --- a/crates/fonts/src/script.rs +++ b/crates/fonts/src/script.rs @@ -134,6 +134,17 @@ impl OutlinePen for Flatten { } } +/// Whether the face has a glyph for `ch` (a cheap character-map probe, no outline work). +pub fn face_has(bytes: &[u8], ch: char) -> bool { + FontRef::new(bytes).is_ok_and(|font| font.charmap().map(ch).is_some()) +} + +/// Return one glyph of `face` (a craft-fonts face), bounded so hostile replacement text cannot +/// allocate unbounded outline data. +pub fn craft_glyph(face: &crate::CraftFont, ch: char) -> Result { + glyph_in_face(face.bytes, ch) +} + /// Return one glyph of the Japanese document face ([`crate::document_japanese_font`], Shippori /// Mincho from craft-fonts), bounded so hostile replacement text cannot allocate unbounded /// outline data. [`GlyphError::NoFont`] when the build has no Japanese face. @@ -145,7 +156,11 @@ pub fn japanese_glyph(ch: char) -> Result { /// The bounded outline and advance from the explicitly selected craft-fonts face. /// Uses the same complexity and missing-glyph checks as [`japanese_glyph`]. pub fn japanese_glyph_from(face: &crate::CraftFont, ch: char) -> Result { - let Ok(font) = FontRef::new(face.bytes) else { return Err(GlyphError::Missing) }; + glyph_in_face(face.bytes, ch) +} + +fn glyph_in_face(bytes: &[u8], ch: char) -> Result { + let Ok(font) = FontRef::new(bytes) else { return Err(GlyphError::Missing) }; let loc = LocationRef::default(); let metrics = font.metrics(Size::unscaled(), loc); let scale = 1.0 / metrics.units_per_em.max(1) as f64; diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 750e78b96d..1b04dbb522 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -16,5 +16,9 @@ ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } rten = { version = "0.26.0", default-features = false, features = ["onnx_format", "rten_format"] } thiserror.workspace = true +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The page image is handed to the external tesseract process as a PNG. +png = "0.18" + [dev-dependencies] tempfile = "3.27" diff --git a/crates/ocr/README.md b/crates/ocr/README.md new file mode 100644 index 0000000000..cd33b186cf --- /dev/null +++ b/crates/ocr/README.md @@ -0,0 +1,104 @@ +# pdfcraft-ocr + +Layer L4 (may use `fonts` for the text layers' metrics): Scan & OCR — recognising the words in +a page raster and turning them back into page content (execution plan M10). + +```rust +// The caller renders a page to pixels and hands them over: +let image = OcrImage::new(w, h, rgba)?; // validated by construction +let lines = recognizer.recognize(&image, &options)?; // Vec: words in reading order +let placed: Vec = …; // PlacedWord::place maps boxes to user space +// The editable output draws real glyphs: plan the words, register the display faces into the +// plan (`pdfcraft_edit::add_type3_fonts`), then generate the layer. An unregistered plan falls +// back to WinAnsi, and characters no registered face has become '?'. +let plan = TextPlan::for_words(placed.iter().map(|w| w.text.as_str())); +add_type3_fonts(&mut doc, page, &mut plan)?; // names the plan's fonts on the page +let layer = visible_text_layer(&placed, &plan); // visible text: the editable output +// The searchable layer needs no display fonts: the engine's own layer (engine::ocr::layer) +// keeps every Unicode character with empty Type3 programs and a ToUnicode map beyond WinAnsi. +``` + +- **Engines behind one trait.** A [`Recognizer`] reports its `id`, whether it is `available`, + the languages it reads, and reads `Vec` from an image. Two exist: the pure-Rust + `OcrsRecognizer` (the `ocrs` crate, MIT/Apache-2.0, with its pre-trained models, + CC-BY-SA-4.0, fetched by `cargo xtask models`; found via `$PDFCRAFT_MODELS` or `models/` + beside the executable — see ATTRIBUTION.toml), and — off the web build — + `tesseract::TesseractCli`, which drives an installed `tesseract` binary as an external + process: never linked, bundled or downloaded. Implementations are shared across threads + (`Send + Sync`); nothing spawns on wasm32. +- **The tesseract process is fenced in.** The program comes from a user-configured location or + the PATH, never from document data (the engine validates a configured path as an absolute + path to an existing executable file before it is used). The language code is allow-listed + (`tesseract::language_code`, 12 codes) before anything reaches a process; the page image + goes to a private temp file (a fresh 0700 directory holding a 0600 file on Unix) that is + deleted afterwards; the run has a 120 s timeout, a 64 MiB output cap and a 50 000-word + reading cap (an incomplete reading is rejected rather than applied); every failure comes back through `OcrError` with its cause named. A configured + location that is invalid fails closed (it is never replaced by the PATH search). Availability + is probed with `--list-langs` (its list is read from stdout, or from stderr where older + versions print it); a failed probe is remembered for 30 s, so an install that appears later + is found without a restart. Known limit: at the timeout or the output cap only the direct + child is killed; a wrapper script's own children may outlive it. +- **Confidence drives review.** One classifier (`band_for`) puts a word in the High + (≥ 90), Medium (70–89) or Low (< 70) band — the review overlay's colours, the legend and the + counts; the batch flag rule (`low_confidence`, default threshold 60) is deliberately a + separate threshold. The `ocrs` models report no confidence (its words carry `None`); + a reading whose confidence is unknown is treated as unknown, never as good. +- **Two engines can be merged (ROVER).** Strategies: `PrimaryOnly` (the default), and two + ensembles — `ConfidenceWeighted` (the secondary engine joins when the primary's mean + confidence is under 70 or unknown; an empty primary keeps the fast path) and `RoverVote` + (both engines always run). The merge lets every primary word adopt the best-overlapping + unused secondary word at IoU ≥ 0.5; the higher confidence wins text, box and confidence (a + tie keeps the primary; against a primary word of unknown confidence — ocrs — the secondary + wins only at 70 or more, `UNKNOWN_CONFIDENCE_BAR`), the survivor is named `"ROVER"`, and + unclaimed secondary words are + appended under their engine's name. A merge is capped at 100 M word-pair comparisons (over + it, the primary reading stands alone), and a failing partner degrades to the primary with a + recorded note — a page is never skipped because a partner could not read it. +- **Preprocessing between the raster and recognition**, a pure function in fixed order: DPI + normalization to 300 (only when one of the steps below is on, since they are calibrated for + it; with all of them off the raster is read exactly as rendered, and a scan at or above 300 + is left alone), then the opt-in steps — + orientation (90°/180°/270° from projection-profile banding), deskew (± 5°, shear-scored, + below 0.1° is a no-op), denoise (3×3 median) and binarization (Sauvola, window 8, k 0.34). + Every geometric step composes one exact affine `InverseTransform`, and callers map every + word box back to the original raster before placing it. The heuristics are conservative and + panic-free: too small to judge, blank, or no clear winner is a no-op, never a guess. +- **Searchable output preserves Unicode without display fonts.** The engine uses empty + Type3 character programs and `/ToUnicode` maps for non-WinAnsi words, leaving the scan + unchanged. WinAnsi words use `/PCHelv`; the `/OCR` layer is fitted to each word's box. +- **Editable output uses display glyphs.** `visible_text_layer` writes visible text whose + size comes from the box height and whose horizontal scaling uses real font advances. + Beyond WinAnsi, attributed craft-fonts faces supply outline Type3 glyphs with `/ToUnicode`. + Uncovered characters become `?` and are reported in editable output notes; searchable + output retains them. The engine refuses nonfinite geometry or oversized input before + planning fonts: 4 MiB of text and 65,536 words, across all pages for editable export. + Empty geometry is skipped; rotated words carry `across`/`up` vectors and retain direction. +- **Hostile input is rejected, not panicked at.** `OcrImage` holds private, validated state: + non-zero sides, a 10 000-pixel side cap, a 64-megapixel area cap (checked arithmetic — the + counts come from documents, so they are untrusted), and an exact buffer length. + +## Where it is wired + +The engine's `ocr` module renders pages into `OcrImage`s, runs jobs (`OcrJob::run`) and applies +the words: searchable-image mode draws the invisible layer over the page's own image and leaves +that image untouched; editable mode writes a NEW `_ocr.pdf` of visible text (the scan +image is not carried over, the source is never overwritten). Skip probes keep pages — and +whole files — that already contain text out of the work unless Force OCR overrides them. A +searchable result applies as one undoable step; the Editable text output opens the OCR Verify +screen first, and Scan & OCR ▸ Correct recognized text reviews any recognition there before it +is applied (confidence-coloured boxes, corrections, a per-language user dictionary — a +plain-text file in the preferences folder, so words added from a document persist on disk +outside it until removed — Re-OCR of a region or a page, Accept/Reject), a +multiple-file run writes searchable copies as it goes, and the automation tools `ocr_recognize`, +`ocr_words`, `ocr_status` and `ocr_recognize_files` expose recognition headlessly. +`ocr_words` with `retain_review` returns a bounded session-local handle; `ocr_review_apply` +applies text corrections at the stored original boxes through its stale-document guard, +or discards the result. Empty corrections remove words. Searchable acceptance is one undo +step; editable acceptance writes a separate file. `ocr_dictionary` reads or atomically +updates the UI's per-language dictionary format through an explicitly root-confined path. + +Not yet: languages beyond the tested set (the `ocrs` models read only unaccented Latin; the +Tesseract side is bounded by its allow-list and whatever its installed language packs cover), +OCR in the browser (the tesseract module is compiled out for wasm32), the deep scan-cleanup +options (descreen, background removal, edge-shadow removal), and editable glyph coverage +beyond the available attributed faces. diff --git a/crates/ocr/src/confidence.rs b/crates/ocr/src/confidence.rs new file mode 100644 index 0000000000..1c0de605f5 --- /dev/null +++ b/crates/ocr/src/confidence.rs @@ -0,0 +1,106 @@ +//! Confidence: one classifier ([`band_for`]) drives the review overlay, the legend, the counts +//! and the uncertain-word navigation; the batch rule ([`low_confidence`]) is a separate +//! threshold with its own range check. The two rules are deliberately never merged. + +/// The three confidence bands (the review overlay's green / yellow / red). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ConfidenceBand { + Low, + Medium, + High, +} + +impl ConfidenceBand { + /// The legend line for this band, so the meaning never rests on colour alone. + pub fn legend(self) -> &'static str { + match self { + ConfidenceBand::High => "HIGH (>= 90%)", + ConfidenceBand::Medium => "MEDIUM (70-89%)", + ConfidenceBand::Low => "LOW (< 70%)", + } + } +} + +/// The one classifier: `>= 90` is High, `70..=89` Medium, `< 70` Low. Out-of-range values +/// clamp (negative → Low, above 100 → High); NaN is Low. +pub fn band_for(confidence: f32) -> ConfidenceBand { + if confidence >= 90.0 { + ConfidenceBand::High + } else if confidence >= 70.0 { + ConfidenceBand::Medium + } else { + ConfidenceBand::Low + } +} + +/// The batch rule, separate from [`band_for`]: a word is flagged when `0 <= conf <= threshold` +/// (default 60). Negative or above-threshold values are not flagged; NaN is not. +pub fn low_confidence(confidence: f32, threshold: f32) -> bool { + (0.0..=threshold).contains(&confidence) +} + +/// A word whose confidence lands in the Low band — a suspect for review. A word without a +/// reported confidence has no signal and is never a suspect. +pub fn is_suspect(confidence: Option) -> bool { + confidence.is_some_and(|c| band_for(c) == ConfidenceBand::Low) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The boundary matrix: 89 and 90, 69 and 70 straddle the bands. + #[test] + fn bands_straddle_the_boundaries() { + assert_eq!(band_for(89.0), ConfidenceBand::Medium); + assert_eq!(band_for(90.0), ConfidenceBand::High); + assert_eq!(band_for(69.0), ConfidenceBand::Low); + assert_eq!(band_for(70.0), ConfidenceBand::Medium); + assert_eq!(band_for(0.0), ConfidenceBand::Low); + assert_eq!(band_for(100.0), ConfidenceBand::High); + } + + /// Out-of-range values clamp, and NaN lands in the Low band instead of comparing equal to + /// nothing. + #[test] + fn out_of_range_confidences_clamp() { + assert_eq!(band_for(-0.5), ConfidenceBand::Low, "negative → Low"); + assert_eq!(band_for(1000.0), ConfidenceBand::High, "above 100 → High"); + assert_eq!(band_for(f32::NAN), ConfidenceBand::Low); + assert_eq!(band_for(f32::NEG_INFINITY), ConfidenceBand::Low); + assert_eq!(band_for(f32::INFINITY), ConfidenceBand::High); + } + + /// The batch threshold is its own rule: everything in `0..=threshold` flags, including + /// words band_for calls Medium (60 < 70), and negatives never do. + #[test] + fn batch_threshold_is_separate_from_the_bands() { + assert!(low_confidence(60.0, 60.0), "at the threshold it flags"); + assert!(low_confidence(0.0, 60.0)); + assert!(!low_confidence(60.5, 60.0)); + assert!(!low_confidence(-1.0, 60.0), "negative confidences are out of the rule's range"); + assert!(!low_confidence(f32::NAN, 60.0)); + // The rules disagree on real values: 65 is inside the Low band yet outside the batch + // rule with the default threshold (60), which is why the two must not be merged. + assert!(!low_confidence(65.0, 60.0)); + assert_eq!(band_for(65.0), ConfidenceBand::Low); + } + + /// Unconfident words are not suspects; Low-band words are. + #[test] + fn suspects_need_a_low_band_confidence() { + assert!(!is_suspect(None)); + assert!(!is_suspect(Some(95.0))); + assert!(!is_suspect(Some(70.0))); + assert!(is_suspect(Some(69.9))); + assert!(is_suspect(Some(0.0))); + } + + /// The legend names the thresholds, so the overlay never rests on colour alone. + #[test] + fn legend_text_matches_the_classifier() { + assert_eq!(ConfidenceBand::High.legend(), "HIGH (>= 90%)"); + assert_eq!(ConfidenceBand::Medium.legend(), "MEDIUM (70-89%)"); + assert_eq!(ConfidenceBand::Low.legend(), "LOW (< 70%)"); + } +} diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index 7f58270e1a..ff5b920dca 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -1,35 +1,50 @@ //! pdfcraft-ocr — Scan & OCR ▸ Recognize text (L4). //! -//! Recognition runs the ocrs engine (MIT/Apache-2.0) with its pre-trained models (CC-BY-SA-4.0, -//! fetched by `cargo xtask models`; see ATTRIBUTION.toml). The caller renders a page to pixels; -//! [`Ocr::recognize`] finds the words in them, and [`text_layer`] turns words placed in user space -//! into page content: invisible text (rendering mode 3) over each word, so the page becomes a +//! The caller renders a page to pixels ([`OcrImage`]) and hands it to a [`Recognizer`]. Two +//! engines exist: the pure-Rust `OcrsRecognizer` (ocrs, MIT/Apache-2.0, with its pre-trained +//! models, CC-BY-SA-4.0, fetched by `cargo xtask models`; see ATTRIBUTION.toml) and — off the +//! web — `tesseract::TesseractCli`, which drives an installed `tesseract` binary as an external +//! process. `merge` combines two engines' readings (ROVER), `preprocess` cleans the raster up +//! between rendering and recognition, and [`text_layer`] turns words placed in user space into +//! page content: invisible text (rendering mode 3) over each word, so the page becomes a //! searchable image (Acrobat's "Searchable Image (Exact)": the image is left untouched). //! -//! The models read the Latin alphabet (English and other languages written without accents). +//! Visible editable output uses [`pdfcraft_fonts::TextPlan`] and optional craft-fonts glyph +//! outlines; unsupported visible characters become `?` and are counted. The engine's invisible +//! searchable layer uses empty Type3 programs and ToUnicode maps, preserving Unicode without +//! display fonts. WinAnsi-only invisible text keeps the standard Helvetica layout here. +//! +//! The ocrs models read the Latin alphabet (English and other languages written without +//! accents); Tesseract (the `tesseract` module, off the web build) reads whatever its installed +//! language packs cover. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -use std::path::{Path, PathBuf}; +pub mod confidence; +pub mod merge; +mod ocrs; +pub mod preprocess; +#[cfg(not(target_arch = "wasm32"))] +pub mod tesseract; +pub use confidence::{ConfidenceBand, band_for, is_suspect, low_confidence}; +pub use merge::{MergeStrategy, StrategyOutcome, mean_confidence, recognize_with_strategy, rover_merge}; +pub use ocrs::{ALPHABET, DETECTION_MODEL, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL}; + +pub use pdfcraft_fonts::TextPlan; pub use pdfcraft_fonts::helvetica_width; -/// The model files, as named in ATTRIBUTION.toml. -pub const DETECTION_MODEL: &str = "text-detection.onnx"; -pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; +/// Hard cap on either side of an image any code in this crate will touch, in pixels; larger +/// images are rejected with [`OcrError::ImageTooLarge`] instead of being processed. +pub const MAX_SIDE: u32 = 10_000; -/// The recognition model's alphabet, as it was trained (ocrs-models' `DEFAULT_ALPHABET`): its -/// output classes follow this order. The ocrs crate's own copy has an `E` where the training had -/// `€`, which made a scanned "€250" read as "E250". -pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - -/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. -pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; +/// Hard cap on an image's TOTAL area, in pixels (64 MP): sides under [`MAX_SIDE`] can still +/// multiply out to a raster no engine should have to read; larger images are rejected with +/// [`OcrError::ImageTooLarge`] instead of being processed. +pub const MAX_PIXELS: u32 = 64_000_000; #[derive(Debug, thiserror::Error)] pub enum OcrError { - /// Shown to end users (release packages ship the models; a source build fetches them with - /// `cargo xtask models`). The UI catalogs key this exact text. #[error("the text recognition models are not installed (reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them)")] NoModels, #[error("loading {0}: {1}")] @@ -38,52 +53,93 @@ pub enum OcrError { Recognize(String), #[error("the image is empty")] EmptyImage, + #[error("the image is larger than the {MAX_SIDE}-pixel side or {MAX_PIXELS}-pixel limit")] + ImageTooLarge, + #[error("the image buffer holds {got} bytes, expected {want} for {width}x{height} RGBA")] + ImageSize { got: usize, want: usize, width: u32, height: u32 }, + #[error("the tesseract program is not available: {0}")] + NoTesseract(String), + #[error("tesseract did not finish within {0} seconds")] + Timeout(u64), + #[error("tesseract produced more than {0} bytes of output")] + OutputTooLarge(u64), + #[error("tesseract failed: {0}")] + Process(String), } -/// Where the two model files are. -#[derive(Clone, Debug, PartialEq)] -pub struct Models { - pub detection: PathBuf, - pub recognition: PathBuf, +/// An RGBA8 raster, row-major, top-left origin: what the engines read. The fields are private +/// so the invariants (non-zero, capped, exactly `width * height * 4` bytes) hold by +/// construction: an [`OcrImage`] in hand has always been validated, whatever produced it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OcrImage { + width: u32, + height: u32, + rgba: Vec, } -impl Models { - /// A complete model pair in `dir`, preferring current ONNX files over legacy RTEN files. - /// An incomplete update never mixes formats or hides an intact legacy pair. - pub fn in_dir(dir: &Path) -> Option { - [(DETECTION_MODEL, RECOGNITION_MODEL), ("text-detection.rten", "text-recognition.rten")].into_iter().find_map(|(detection, recognition)| { - let m = Models { detection: dir.join(detection), recognition: dir.join(recognition) }; - (m.detection.is_file() && m.recognition.is_file()).then_some(m) - }) - } - - /// Look for the models: `$PDFCRAFT_MODELS`, then where the release packages install them - /// beside the executable ([`Models::dirs_beside_exe`]), then the source tree's `assets/models/`. - pub fn find() -> Option { - Self::search_dirs().iter().find_map(|d| Self::in_dir(d)) - } - - /// The directories [`Models::find`] looks in, in order. - pub fn search_dirs() -> Vec { - let mut dirs = Vec::new(); - if let Some(d) = std::env::var_os("PDFCRAFT_MODELS") { - dirs.push(PathBuf::from(d)); +impl OcrImage { + /// Validate the sides ([`MAX_SIDE`]) and total area ([`MAX_PIXELS`]) and that `rgba` holds + /// exactly `width * height * 4` bytes (checked arithmetic; the counts come from documents, + /// so they are untrusted). + pub fn new(width: u32, height: u32, rgba: Vec) -> Result { + if width == 0 || height == 0 { + return Err(OcrError::EmptyImage); } - if let Some(exe) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { - dirs.extend(Self::dirs_beside_exe(&exe)); + if width > MAX_SIDE || height > MAX_SIDE { + return Err(OcrError::ImageTooLarge); + } + let pixels = (width as usize).checked_mul(height as usize).ok_or(OcrError::ImageTooLarge)?; + if pixels > MAX_PIXELS as usize { + return Err(OcrError::ImageTooLarge); + } + match pixels.checked_mul(4) { + Some(want) if rgba.len() == want => Ok(OcrImage { width, height, rgba }), + Some(want) => Err(OcrError::ImageSize { got: rgba.len(), want, width, height }), + None => Err(OcrError::ImageTooLarge), } - dirs.push(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/models")); - dirs } - /// Where the release packages put the models, relative to the directory holding the - /// executable (packaging/*): - /// - `models/` beside it: the Windows MSI and portable zip; - /// - `../Resources/models`: the macOS app bundle (`Contents/MacOS` → `Contents/Resources`); - /// - `../share/pdfcraft/models`: the FHS-style trees (`bin/` → `share/`) of the Linux - /// deb/rpm/tar.gz/AppImage/Flatpak and the FreeBSD tarball, wherever they are installed. - pub fn dirs_beside_exe(exe_dir: &Path) -> Vec { - vec![exe_dir.join("models"), exe_dir.join("../Resources/models"), exe_dir.join("../share/pdfcraft/models")] + /// The image's width, in pixels. + pub fn width(&self) -> u32 { + self.width + } + + /// The image's height, in pixels. + pub fn height(&self) -> u32 { + self.height + } + + /// The RGBA bytes, row-major, exactly `width * height * 4` of them. + pub fn rgba(&self) -> &[u8] { + &self.rgba + } + + /// The smallest side, in pixels. + pub fn min_side(&self) -> u32 { + self.width.min(self.height) + } + + /// One RGBA byte at integer pixel (`x`, `y`), edges clamped; 255 (white) outside. Only + /// whole-pixel positions and channel 0–3 make sense; anything else reads clamped. + pub fn byte(&self, x: f64, y: f64, ch: usize) -> u8 { + let stride = self.width as usize * 4; + if stride == 0 || self.height == 0 || ch > 3 { + return 255; + } + let xi = (x.round().max(0.0) as usize).min(self.width as usize - 1); + let yi = (y.round().max(0.0) as usize).min(self.height as usize - 1); + self.rgba.get(yi * stride + xi * 4 + ch).copied().unwrap_or(255) + } + + /// The grayscale value of the pixel at (`x`, `y`), 0 (ink) … 255 (paper), with the borders + /// clamped (edges replicate). `None` only when the image is empty. + pub fn gray(&self, x: i64, y: i64) -> Option { + let x = x.clamp(0, self.width as i64 - 1); + let y = y.clamp(0, self.height as i64 - 1); + let row = y as usize * self.width as usize * 4; + let o = row + x as usize * 4; + let px = self.rgba.get(o..o + 4)?; + Some(((299 * px[0] as u32 + 587 * px[1] as u32 + 114 * px[2] as u32) / 1000) as u8) } } @@ -92,6 +148,26 @@ impl Models { pub struct Word { pub text: String, pub rect: [f32; 4], + /// How sure the engine is, 0–100, when it reports a confidence at all (`None`: the ocrs + /// engine does not). + pub confidence: Option, + /// Which engine read this word: the recogniser's [`Recognizer::id`], or `"ROVER"` when two + /// engines were merged and the surviving text is the merged choice. + pub source: String, +} + +impl Word { + pub fn new(text: impl Into, rect: [f32; 4], source: &str) -> Word { + Word { text: text.into(), rect, confidence: None, source: source.into() } + } + + /// Set the confidence (builder style), clamped to the documented 0–100 scale (a negative + /// value is 0, anything over 100 is 100). NaN means no confidence was reported (`None`), + /// the same unknown the engines that report nothing carry. + pub fn with_confidence(mut self, confidence: f32) -> Word { + self.confidence = if confidence.is_nan() { None } else { Some(confidence.clamp(0.0, 100.0)) }; + self + } } /// A recognised line of words, in reading order. @@ -106,63 +182,34 @@ impl Line { } } -/// A loaded recogniser. Loading takes a moment; keep one and reuse it. -pub struct Ocr { - engine: ocrs::OcrEngine, +/// What a recogniser needs to know beyond the pixels. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecognizeOptions { + /// Language code from [`LANGUAGES`] (`"en"`, `"de"`, …). Engines map it to their own codes + /// (Tesseract: `tesseract::language_code`, allow-listed before anything reaches a process); + /// unknown codes fall back to the engine's default. + pub language: String, } -impl Ocr { - pub fn load(models: &Models) -> Result { - let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); - let params = ocrs::OcrEngineParams { - detection_model: Some(load(&models.detection)?), - recognition_model: Some(load(&models.recognition)?), - alphabet: Some(ALPHABET.into()), - ..Default::default() - }; - let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; - Ok(Ocr { engine }) +impl Default for RecognizeOptions { + fn default() -> Self { + RecognizeOptions { language: "en".into() } } +} - /// Load the models found by [`Models::find`]. - pub fn find() -> Result { - Self::load(&Models::find().ok_or(OcrError::NoModels)?) - } - - /// Recognise the text in an RGBA (or RGB, or grey) image, `width` × `height` pixels. - pub fn recognize(&self, pixels: &[u8], width: u32, height: u32) -> Result, OcrError> { - if width == 0 || height == 0 || pixels.is_empty() { - return Err(OcrError::EmptyImage); - } - // ocrs wants 1 or 3 channels. - let rgb: std::borrow::Cow<[u8]> = if pixels.len() == (width * height * 4) as usize { - pixels.as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() - } else { - pixels.into() - }; - let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); - let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; - let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; - let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; - let lines = self.engine.find_text_lines(&input, &found); - let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; - use ocrs::TextItem; - Ok(read - .into_iter() - .flatten() - .map(|line| Line { - words: line - .words() - .map(|w| { - let r = w.bounding_rect(); - Word { text: w.to_string(), rect: [r.left() as f32, r.top() as f32, r.right() as f32, r.bottom() as f32] } - }) - .filter(|w| !w.text.trim().is_empty()) - .collect(), - }) - .filter(|l| !l.words.is_empty()) - .collect()) - } +/// One text-recognition engine. Implementations are shared across threads +/// (`Arc`); `available` and `recognize` may spawn processes only off the web +/// build (see the `tesseract` module). +pub trait Recognizer: Send + Sync { + /// The engine's short name (`"ocrs"`, `"tesseract"`): what [`Word::source`] carries and + /// what settings and tools name it by. + fn id(&self) -> &'static str; + /// Whether this engine can run right now (models installed, binary found). + fn available(&self) -> bool; + /// The languages it reads, as `(code, name)` pairs (`"en"`, `"English"`). + fn languages(&self) -> Vec<(String, String)>; + /// Read the words in `image`, in reading order. + fn recognize(&self, image: &OcrImage, options: &RecognizeOptions) -> Result, OcrError>; } /// A word placed on a page, in PDF user space: the bottom-left corner of its box, and the @@ -173,6 +220,10 @@ pub struct PlacedWord { pub origin: [f64; 2], pub across: [f64; 2], pub up: [f64; 2], + /// Confidence 0–100, when the engine reported one. + pub confidence: Option, + /// Which engine read this word (see [`Word::source`]). + pub source: String, } impl PlacedWord { @@ -183,7 +234,14 @@ impl PlacedWord { let o = to_user(l, b); let br = to_user(r, b); let tl = to_user(l, t); - PlacedWord { text: word.text.clone(), origin: o, across: [br[0] - o[0], br[1] - o[1]], up: [tl[0] - o[0], tl[1] - o[1]] } + PlacedWord { + text: word.text.clone(), + origin: o, + across: [br[0] - o[0], br[1] - o[1]], + up: [tl[0] - o[0], tl[1] - o[1]], + confidence: word.confidence, + source: word.source.clone(), + } } } @@ -214,32 +272,193 @@ fn num(v: f64) -> String { if s == "-0" { "0".into() } else { s.into() } } -/// Page content that writes `words` as invisible text in `/PCHelv` (standard Helvetica, -/// WinAnsiEncoding), each word stretched to its box so selection and search highlight the -/// right place. Marked content `/OCR` so it can be told apart from the page's own text. -pub fn text_layer(words: &[PlacedWord]) -> Vec { +/// One run of a word's planned characters in one Type3 font: the resource name, the codes to +/// show (one byte per character), and the run's start as an offset in em units from the word's +/// origin (the accumulated advances of the runs before it). +struct PlannedRun { + font: String, + codes: Vec, + start: f64, +} + +/// The planned runs of word `i` (the same index [`pdfcraft_fonts::TextPlan::for_words`] planned +/// the words with), or `None` when the word is drawn in Helvetica: every character has a +/// WinAnsi code, or nothing could be planned/registered for it (the pre-Unicode output). +fn planned_runs(plan: &pdfcraft_fonts::TextPlan, word: usize) -> Option<(Vec, f64)> { + let enc = plan.encoding(word); + if enc.is_empty() || !plan.registered(enc) { + return None; + } + let mut runs: Vec = Vec::new(); + let mut total = 0.0; + for &(f, code) in enc { + let font = plan.fonts[f].name.as_str(); + match runs.last_mut() { + Some(last) if last.font == font => last.codes.push(code), + _ => runs.push(PlannedRun { font: font.to_string(), codes: vec![code], start: total }), + } + total += plan.advance(f, code); + } + (total.is_finite() && total > 0.0).then_some((runs, total)) +} + +/// Page content that writes `words` as VISIBLE text at their original boxes: the +/// editable-text output's page content. WinAnsi words are set in `/PCHelv` (standard +/// Helvetica); words with characters beyond WinAnsi are set in the plan's Type3 fonts, one run +/// per font. Each word fills its box — the font size comes from the box height and the +/// horizontal scaling is computed from the real advances of the font that draws the word +/// (Helvetica's approximated, the Type3 faces' exact) and emitted as an explicit `Tz` +/// percentage. Words whose box is empty (or whose text has no measurable width) are skipped; a +/// page that ends up with nothing simply shows nothing. +pub fn visible_text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec { + visible_text_layer_at(words, plan, 0) +} + +/// Visible text for words starting at `word_offset` in a shared document plan. +pub fn visible_text_layer_at(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan, word_offset: usize) -> Vec { + // The rendering mode and the font are graphics state, which `ET` doesn't reset: without + // `q … Q` they would stay set for every later content stream on the page, so text added + // after OCR (a header or footer) drew invisibly (#3029). + let mut out = b"/OCR BMC\nq\nBT\n0 Tr\n/PCHelv 1 Tf\n".to_vec(); + // The font and size the text state is set in, so a Tf is written only when they change. + // (The header's size-1 Helvetica covers layers of WinAnsi words only.) + let mut current: Option<(String, f64)> = Some(("PCHelv".to_string(), 1.0)); + for (i, w) in words.iter().enumerate() { + let height = w.up[0].hypot(w.up[1]); + let across = w.across[0].hypot(w.across[1]); + let (ux, uy) = (w.across[0] / across, w.across[1] / across); + let (vx, vy) = (w.up[0] / height, w.up[1] / height); + if !across.is_finite() + || !height.is_finite() + || across <= 0.0 + || height <= 0.0 + || !ux.is_finite() + || !vx.is_finite() + || !w.origin.iter().all(|v| v.is_finite()) + { + continue; + } + let size = height / BOX_EM; + let runs = planned_runs(plan, word_offset.saturating_add(i)); + // The word's natural advance at this size: Helvetica's approximation, or the exact sum + // of the planned glyphs' widths. + let natural = runs.as_ref().map_or_else(|| helvetica_width(&w.text, size), |(_, total)| total * size); + if natural.is_nan() || natural <= 0.0 { + continue; + } + // Horizontal scaling as a percentage of the natural advance at this size, so the word + // lands exactly on its box (100 = no scaling; a long box on a short word stretches it). + let tz = (across / natural * 100.0).max(1.0); + // The baseline sits above the box's bottom edge by the descender depth. + let o = [w.origin[0] + vx * DESCENT * size, w.origin[1] + vy * DESCENT * size]; + out.extend_from_slice(format!("{} Tz ", num(tz)).as_bytes()); + match &runs { + None => { + if current.as_ref().is_some_and(|(f, s)| f != "PCHelv" || *s != size) { + out.extend_from_slice(format!("/PCHelv {} Tf ", num(size)).as_bytes()); + current = Some(("PCHelv".to_string(), size)); + } + out.extend_from_slice(format!("{} {} {} {} {} {} Tm ", num(ux), num(uy), num(vx), num(vy), num(o[0]), num(o[1])).as_bytes()); + out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); + out.extend_from_slice(b" Tj\n"); + } + Some((planned, total)) => { + for run in planned { + if current.as_ref().is_some_and(|(f, s)| *f != run.font || *s != size) { + out.extend_from_slice(format!("/{} {} Tf ", run.font, num(size)).as_bytes()); + current = Some((run.font.clone(), size)); + } + // The run's origin: the word's baseline plus the across vector scaled by + // the advances before this run (the stretch applies to runs alike). + let frac = run.start / total; + out.extend_from_slice( + format!( + "{} {} {} {} {} {} Tm ", + num(ux), + num(uy), + num(vx), + num(vy), + num(o[0] + w.across[0] * frac), + num(o[1] + w.across[1] * frac) + ) + .as_bytes(), + ); + out.extend(pdfcraft_fonts::literal(&run.codes)); + out.extend_from_slice(b" Tj\n"); + } + } + } + } + out.extend_from_slice(b"ET\nQ\nEMC\n"); + out +} + +/// Page content that writes `words` as invisible text (rendering mode 3), each word stretched +/// to its box so selection and search highlight the right place. WinAnsi words are set in +/// `/PCHelv` (standard Helvetica); words with characters beyond WinAnsi are set in the plan's +/// Type3 fonts, one run per font, with the same exact box fit. Marked content `/OCR` so it can +/// be told apart from the page's own text. +pub fn text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec { // `3 Tr` and the font are graphics state, which `ET` doesn't reset: without `q … Q` they // would stay set for every later content stream on the page, so text added after OCR (a // header or footer) drew invisibly (#3029). let mut out = b"/OCR BMC\nq\nBT\n3 Tr\n/PCHelv 1 Tf\n".to_vec(); - for w in words { - let width = helvetica_width(&w.text, 1.0); + let mut current: Option<(String, f64)> = Some(("PCHelv".to_string(), 1.0)); + for (i, w) in words.iter().enumerate() { + let runs = planned_runs(plan, i); + let width = runs.as_ref().map_or_else(|| helvetica_width(&w.text, 1.0), |(_, total)| *total); let height = w.up[0].hypot(w.up[1]); - if width <= 0.0 || height <= 0.0 { + let across = w.across[0].hypot(w.across[1]); + // Only positive, finite geometry reaches the content stream: a zero-sized box cannot + // be drawn, and a NaN or infinite one would put non-finite numbers on the page. (NaN + // fails every comparison, so the positivity checks already refuse it; the finiteness + // checks keep that property explicit.) + if width <= 0.0 || !(height > 0.0 && height.is_finite()) || !(across > 0.0 && across.is_finite()) || !w.origin.iter().all(|v| v.is_finite()) { continue; } - // The em square: the word's own ink (cap height or x-height, descenders or not) spans the - // box, so the text is as tall as the scanned word; its width is stretched to fit the word. - let (top, bottom) = ink_extent(&w.text); + // The em square. Helvetica words: the word's own ink (cap height or x-height, + // descenders or not) spans the box, so the text is as tall as the scanned word. Planned + // (Type3) words: those Helvetica letter heights do not describe the face, so the box spans + // the ascender-descender height. Either way the width is stretched to fit the word. + let (top, bottom) = if runs.is_some() { (BOX_EM - DESCENT, -DESCENT) } else { ink_extent(&w.text) }; let em = [w.up[0] / (top - bottom), w.up[1] / (top - bottom)]; - let ax = [w.across[0] / width, w.across[1] / width]; let o = [w.origin[0] - em[0] * bottom, w.origin[1] - em[1] * bottom]; - let m = [ax[0], ax[1], em[0], em[1], o[0], o[1]]; - let nums: Vec = m.iter().map(|v| num(*v)).collect(); - out.extend_from_slice(nums.join(" ").as_bytes()); - out.extend_from_slice(b" Tm "); - out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); - out.extend_from_slice(b" Tj\n"); + match &runs { + None => { + let ax = [w.across[0] / width, w.across[1] / width]; + if current.as_ref().is_some_and(|(f, s)| f != "PCHelv" || *s != 1.0) { + out.extend_from_slice(b"/PCHelv 1 Tf "); + current = Some(("PCHelv".to_string(), 1.0)); + } + let m = [ax[0], ax[1], em[0], em[1], o[0], o[1]]; + let nums: Vec = m.iter().map(|v| num(*v)).collect(); + out.extend_from_slice(nums.join(" ").as_bytes()); + out.extend_from_slice(b" Tm "); + out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); + out.extend_from_slice(b" Tj\n"); + } + Some((planned, total)) => { + for run in planned { + if current.as_ref().is_some_and(|(f, s)| *f != run.font || *s != 1.0) { + out.extend_from_slice(format!("/{} 1 Tf ", run.font).as_bytes()); + current = Some((run.font.clone(), 1.0)); + } + let m = [ + w.across[0] / total, + w.across[1] / total, + em[0], + em[1], + o[0] + w.across[0] * run.start / total, + o[1] + w.across[1] * run.start / total, + ]; + let nums: Vec = m.iter().map(|v| num(*v)).collect(); + out.extend_from_slice(nums.join(" ").as_bytes()); + out.extend_from_slice(b" Tm "); + out.extend(pdfcraft_fonts::literal(&run.codes)); + out.extend_from_slice(b" Tj\n"); + } + } + } } out.extend_from_slice(b"ET\nQ\nEMC\n"); out @@ -253,10 +472,22 @@ mod tests { fn places_words_through_the_mapping() { // 2 pixels per point, page 612 × 792, y flipped. let to_user = |x: f32, y: f32| [x as f64 / 2.0, 792.0 - y as f64 / 2.0]; - let w = PlacedWord::place(&Word { text: "Hello".into(), rect: [100.0, 200.0, 300.0, 240.0] }, to_user); + let w = PlacedWord::place(&Word::new("Hello", [100.0, 200.0, 300.0, 240.0], "ocrs"), to_user); assert_eq!(w.origin, [50.0, 672.0]); assert_eq!(w.across, [100.0, 0.0]); assert_eq!(w.up, [0.0, 20.0]); + assert_eq!(w.source, "ocrs"); + assert_eq!(w.confidence, None); + } + + /// Confidence and source survive placement. + #[test] + fn placed_words_carry_confidence_and_source() { + let to_user = |x: f32, y: f32| [x as f64, y as f64]; + let w = Word::new("Hi", [1.0, 2.0, 3.0, 4.0], "tesseract").with_confidence(87.5); + let p = PlacedWord::place(&w, to_user); + assert_eq!(p.confidence, Some(87.5)); + assert_eq!(p.source, "tesseract"); } /// The text layer is Helvetica in WinAnsiEncoding: every character the model can recognise @@ -267,10 +498,27 @@ mod tests { assert_eq!(pdfcraft_fonts::first_undrawable(ALPHABET), None); } + /// `with_confidence` is clamped to the documented 0–100 scale: negatives (and −inf) are 0, + /// anything over 100 (and +inf) is 100, in-scale values pass through, and NaN means no + /// confidence was reported — the same `None` unknown the unreporting engines carry. + #[test] + fn with_confidence_clamps_and_treats_nan_as_unknown() { + let c = |v: f32| Word::new("w", [0.0; 4], "ocrs").with_confidence(v).confidence; + assert_eq!(c(0.0), Some(0.0)); + assert_eq!(c(50.0), Some(50.0)); + assert_eq!(c(100.0), Some(100.0)); + assert_eq!(c(-1.0), Some(0.0), "negatives clamp to 0"); + assert_eq!(c(f32::NEG_INFINITY), Some(0.0)); + assert_eq!(c(150.0), Some(100.0), "over 100 clamps to 100"); + assert_eq!(c(f32::INFINITY), Some(100.0)); + assert_eq!(c(f32::NAN), None, "NaN is unknown, not a confidence"); + assert_eq!(c(f32::EPSILON), Some(f32::EPSILON), "in-scale values are untouched"); + } + #[test] fn text_layer_is_invisible_text_fitted_to_each_box() { - let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); assert!(s.contains("3 Tr") && s.contains("/PCHelv 1 Tf"), "{s}"); let width = helvetica_width("Hi", 1.0); // "Hi" has no descender: its box spans the cap height, from the baseline up. @@ -283,8 +531,8 @@ mod tests { /// after it on the page starts from the graphics state it had before. #[test] fn the_layer_restores_the_graphics_state_it_changes() { - let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); let mut depth = 0_i32; for token in s.split_ascii_whitespace() { match token { @@ -309,78 +557,200 @@ mod tests { assert_eq!(ink_extent("Gallus.Ze@camtel.cm"), (CAP_HEIGHT, 0.0)); assert_eq!(ink_extent("Corporation"), (CAP_HEIGHT, -DESCENT)); assert_eq!(ink_extent("--"), (BOX_EM - DESCENT, -DESCENT)); - let w = PlacedWord { text: "gap".into(), origin: [0.0, 100.0], across: [20.0, 0.0], up: [0.0, 7.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "gap".into(), origin: [0.0, 100.0], across: [20.0, 0.0], up: [0.0, 7.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); let em = 7.3 / (X_HEIGHT + DESCENT); assert!(s.contains(&format!(" 0 0 {} 0 {} Tm (gap) Tj", num(em), num(100.0 + em * DESCENT))), "{s}"); } + /// A word whose box is not positive, finite geometry never reaches the content stream: + /// NaN, infinite and zero-sized boxes are dropped, so the layer never carries non-finite + /// numbers. + #[test] + fn text_layer_refuses_non_finite_and_degenerate_boxes() { + let word = |across: [f64; 2], up: [f64; 2], origin: [f64; 2]| PlacedWord { + text: "Hi".into(), + origin, + across, + up, + confidence: Some(90.0), + source: "tesseract".into(), + }; + for (why, w) in [ + ("NaN across", word([f64::NAN, 0.0], [0.0, 9.3], [0.0; 2])), + ("NaN up", word([30.0, 0.0], [f64::NAN, 1.0], [0.0; 2])), + ("inf up", word([30.0, 0.0], [0.0, f64::INFINITY], [0.0; 2])), + ("inf origin", word([30.0, 0.0], [0.0, 9.3], [0.0, f64::INFINITY])), + ("zero across", word([0.0, 0.0], [0.0, 9.3], [0.0; 2])), + ("NaN origin", word([30.0, 0.0], [0.0, 9.3], [0.0, f64::NAN])), + ] { + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); + assert_eq!(s, "/OCR BMC\nq\nBT\n3 Tr\n/PCHelv 1 Tf\nET\nQ\nEMC\n", "{why}: {s}"); + assert!(!s.contains("NaN") && !s.contains("inf"), "{why}: {s}"); + } + } + + /// The editable-text layer draws visible text whose horizontal scaling is an explicit + /// percentage computed from real Helvetica metrics: `Tz` × natural advance = box width. + #[test] + fn visible_text_layer_scales_by_a_percentage_of_real_metrics() { + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [40.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(visible_text_layer(&[w], &TextPlan::default())).unwrap(); + assert!(s.contains("0 Tr"), "visible, not invisible: {s}"); + let size = 9.3 / BOX_EM; + let natural = helvetica_width("Hi", size); + let tz = 40.0 / natural * 100.0; + assert!(s.contains(&format!("{} Tz", num(tz))), "{s}"); + assert!(s.contains("(Hi) Tj"), "{s}"); + assert!(s.starts_with("/OCR BMC") && s.ends_with("EMC\n")); + } + + /// Degenerate boxes and words with no measurable width are skipped, never divide by zero. + #[test] + fn visible_text_layer_skips_degenerate_words() { + let word = |across: [f64; 2], up: [f64; 2], text: &str| PlacedWord { + text: text.into(), + origin: [0.0; 2], + across, + up, + confidence: None, + source: "ocrs".into(), + }; + let words = vec![ + word([0.0, 0.0], [0.0, 9.0], "flat"), + word([10.0, 0.0], [0.0, 0.0], "short"), + word([f64::NAN, 0.0], [0.0, 9.0], "nan"), + word([10.0, 0.0], [0.0, 9.0], ""), + word([10.0, 0.0], [0.0, 9.0], "Hi"), + ]; + let s = String::from_utf8(visible_text_layer(&words, &TextPlan::default())).unwrap(); + assert_eq!(s.matches("Tj").count(), 1, "only the one drawable word is written: {s}"); + } + + /// A word whose ORIGIN is not finite never reaches the visible layer either: a NaN or + /// infinite origin would put "NaN"/"inf" into the content stream. + #[test] + fn visible_text_layer_refuses_non_finite_origins() { + let word = + |origin: [f64; 2]| PlacedWord { text: "Hi".into(), origin, across: [10.0, 0.0], up: [0.0, 9.0], confidence: None, source: "ocrs".into() }; + for (why, origin) in [("NaN x", [f64::NAN, 100.0]), ("inf y", [10.0, f64::INFINITY]), ("NaN both", [f64::NAN; 2])] { + let s = String::from_utf8(visible_text_layer(&[word(origin)], &TextPlan::default())).unwrap(); + assert!(!s.contains("NaN") && !s.contains("inf"), "{why}: {s}"); + assert_eq!(s.matches("Tj").count(), 0, "{why}: {s}"); + } + } + + /// After a Type3 word the text state goes back to Helvetica: the Latin words around a + /// planned word must not inherit its font. + #[test] + fn helvetica_is_restored_after_a_planned_word() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let word = |text: &str| PlacedWord { + text: text.into(), + origin: [10.0, 20.0], + across: [60.0, 0.0], + up: [0.0, 9.3], + confidence: None, + source: "ocrs".into(), + }; + let words = [word("Hi"), word("日本"), word("Ho")]; + let mut plan = TextPlan::for_words(["Hi", "日本", "Ho"]); + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + let s = String::from_utf8(text_layer(&words, &plan)).unwrap(); + assert!(s.contains("/PCOcr0 1 Tf"), "{s}"); + // The last Helvetica selection comes after the Type3 run, and the Latin word after it + // is drawn in Helvetica, not left in the planned font. + assert!(s.rfind("/PCOcr0 1 Tf").unwrap_or(0) < s.rfind("/PCHelv 1 Tf").unwrap_or(usize::MAX), "{s}"); + // The `Q` closes the `q … Q` scope that keeps `3 Tr` from leaking into later content. + assert!(s.ends_with("Tm (Ho) Tj\nET\nQ\nEMC\n"), "{s}"); + // The visible layer restores Helvetica with the word's size. + let v = String::from_utf8(visible_text_layer(&words, &plan)).unwrap(); + assert!(v.rfind("/PCOcr0").unwrap_or(0) < v.rfind("/PCHelv").unwrap_or(usize::MAX), "{v}"); + assert!(v.contains("(Ho) Tj"), "{v}"); + } + + /// Words with characters beyond WinAnsi draw in the plan's Type3 font, one run, with the + /// same exact box fit as the Helvetica words (the matrix x-axis spans across / advance). + #[test] + fn planned_words_draw_in_their_font_with_the_same_exact_fit() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let w = + PlacedWord { text: "日本".into(), origin: [10.0, 20.0], across: [60.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let mut plan = TextPlan::for_words(["日本"]); + // The names the registrar (pdfcraft_edit::add_type3_fonts) fills in — this crate + // cannot reach it, so the test sets them where it reads them. + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + let codes: Vec = plan.encoding(0).iter().map(|&(_, c)| c).collect(); + assert_eq!(codes.len(), 2, "{codes:?}"); + let total = plan.advance_of(plan.encoding(0)); + let s = String::from_utf8(text_layer(std::slice::from_ref(&w), &plan)).unwrap(); + assert!(s.contains("/PCOcr0 1 Tf"), "{s}"); + assert!(s.contains(&format!("{} 0 0 10 10 22.1 Tm ", num(60.0 / total))), "{s}"); + assert!(s.contains(&format!("Tm {} Tj\n", String::from_utf8_lossy(&pdfcraft_fonts::literal(&codes)))), "{s}"); + // The visible layer: size from the box height, Tz from the Type3's own advance. + let v = String::from_utf8(visible_text_layer(std::slice::from_ref(&w), &plan)).unwrap(); + let size = 9.3 / BOX_EM; + assert!(v.contains(&format!("/PCOcr0 {} Tf ", num(size))), "{v}"); + assert!(v.contains(&format!("{} Tz", num(60.0 / (total * size) * 100.0))), "{v}"); + } + + /// Without planned characters (an empty plan: built without craft-fonts, or before the + /// fonts are registered) the WinAnsi path stands, '?' included. + #[test] + fn unplanned_words_fall_back_to_win_ansi_question_marks() { + let w = + PlacedWord { text: "日本".into(), origin: [10.0, 20.0], across: [60.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); + assert!(s.contains("(??) Tj"), "{s}"); + assert!(!s.contains("PCOcr"), "{s}"); + } + + /// A character no face has a glyph for becomes the font's '?', still in the run, counted + /// by the plan. + #[test] + fn substitutions_are_written_as_the_font_s_question_mark() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let w = PlacedWord { + text: "日\u{1f4a9}語".into(), + origin: [0.0, 0.0], + across: [60.0, 0.0], + up: [0.0, 9.3], + confidence: None, + source: "ocrs".into(), + }; + let mut plan = TextPlan::for_words(["日\u{1f4a9}語"]); + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + assert_eq!(plan.missing(), 1, "the emoji has no glyph anywhere"); + assert_eq!(plan.encoding(0).len(), 3, "it still gets an encoding: the font's ?"); + let codes: Vec = plan.encoding(0).iter().map(|&(_, c)| c).collect(); + let s = String::from_utf8(text_layer(&[w], &plan)).unwrap(); + assert!(s.contains(&format!("Tm {} Tj\n", String::from_utf8_lossy(&pdfcraft_fonts::literal(&codes)))), "{s}"); + } + #[test] fn rotated_pages_keep_the_reading_direction() { // A page turned 90°: image x runs up the page. let to_user = |x: f32, y: f32| [y as f64, x as f64]; - let w = PlacedWord::place(&Word { text: "Up".into(), rect: [0.0, 0.0, 50.0, 10.0] }, to_user); + let w = PlacedWord::place(&Word::new("Up", [0.0, 0.0, 50.0, 10.0], "ocrs"), to_user); assert_eq!(w.across, [0.0, 50.0]); assert_eq!(w.up, [-10.0, 0.0]); } - /// The current ONNX pair wins when an installation also retains both legacy models. - #[test] - fn model_discovery_prefers_the_complete_onnx_pair() { - let dir = tempfile::tempdir().unwrap(); - for file in ["text-detection.onnx", "text-recognition.onnx"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - let expected = Some(Models { detection: dir.path().join("text-detection.onnx"), recognition: dir.path().join("text-recognition.onnx") }); - assert_eq!(Models::in_dir(dir.path()), expected, "a complete ONNX pair is sufficient"); - for file in ["text-detection.rten", "text-recognition.rten"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!(Models::in_dir(dir.path()), expected, "the complete ONNX pair takes precedence over legacy models"); - } - - /// An interrupted ONNX update still permits an intact legacy pair in the same directory. - #[test] - fn model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete() { - for current in ["text-detection.onnx", "text-recognition.onnx"] { - let dir = tempfile::tempdir().unwrap(); - for file in [current, "text-detection.rten", "text-recognition.rten"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!( - Models::in_dir(dir.path()), - Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }), - "{current} alone must not hide a complete legacy pair" - ); - } - } - - /// Detection and recognition must come from one complete format pair, never a mixed update. - #[test] - fn model_discovery_refuses_mixed_format_pairs() { - for pair in [["text-detection.onnx", "text-recognition.rten"], ["text-detection.rten", "text-recognition.onnx"]] { - let dir = tempfile::tempdir().unwrap(); - for file in pair { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!(Models::in_dir(dir.path()), None, "{pair:?} is not a complete model pair"); - } - } - - /// Existing source builds and PDFCRAFT_MODELS overrides retain their complete RTEN pair. - #[test] - fn model_discovery_keeps_legacy_only_installations() { - let dir = tempfile::tempdir().unwrap(); - assert_eq!(Models::in_dir(dir.path()), None); - std::fs::write(dir.path().join("text-detection.rten"), b"model").unwrap(); - assert_eq!(Models::in_dir(dir.path()), None, "one legacy model is incomplete"); - std::fs::write(dir.path().join("text-recognition.rten"), b"model").unwrap(); - assert_eq!( - Models::in_dir(dir.path()), - Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }) - ); - } - /// #103: every release package's layout puts the models where the installed app looks. /// Mirrors packaging/{windows/package.ps1,macos/package.sh,linux/package.sh,freebsd/package.sh}. #[test] @@ -443,6 +813,47 @@ mod tests { assert!(word.rect[0] >= 10.0 && word.rect[0] < 60.0, "{word:?}"); } + /// The recognised words name their engine. + #[test] + fn ocrs_recognizer_names_its_words() { + let Ok(rec) = OcrsRecognizer::find() else { + eprintln!("skipped: OCR models not installed"); + return; + }; + assert_eq!(rec.id(), "ocrs"); + assert!(rec.languages().iter().any(|(c, n)| c == "en" && n == "English"), "{:?}", rec.languages()); + let (w, h, px) = test_image::hello(); + let image = OcrImage::new(w, h, px).unwrap(); + let lines = rec.recognize(&image, &RecognizeOptions::default()).unwrap(); + let words: Vec<&Word> = lines.iter().flat_map(|l| &l.words).collect(); + assert!(!words.is_empty()); + assert!(words.iter().all(|w| w.source == "ocrs"), "{words:?}"); + } + + /// Hostile images are rejected with errors, never panics: zero sides, sides over the cap, + /// totals over the area cap (even with legal sides), buffers that do not fit, counts that + /// overflow. + #[test] + fn hostile_image_dimensions_are_rejected_not_panicked_at() { + assert!(matches!(OcrImage::new(0, 10, vec![]), Err(OcrError::EmptyImage))); + assert!(matches!(OcrImage::new(10, 0, vec![]), Err(OcrError::EmptyImage))); + assert!(matches!(OcrImage::new(u32::MAX, u32::MAX, vec![]), Err(OcrError::ImageTooLarge))); + assert!(matches!(OcrImage::new(MAX_SIDE + 1, 1, vec![0; 4]), Err(OcrError::ImageTooLarge))); + // 10 000 × 7 000 = 70 MP: legal on both sides, over the 64 MP area cap (no allocation + // is needed to reject it — the buffer is never inspected). + assert!(matches!(OcrImage::new(MAX_SIDE, 7000, vec![]), Err(OcrError::ImageTooLarge))); + // 8 000 × 8 000 = exactly the area cap: accepted shape (the short buffer proves the + // cap check passed), while one more row is refused. + assert!(matches!(OcrImage::new(8000, 8000, vec![0; 16]), Err(OcrError::ImageSize { .. }))); + assert!(matches!(OcrImage::new(8000, 8001, vec![0; 16]), Err(OcrError::ImageTooLarge))); + assert!(matches!(OcrImage::new(2, 2, vec![0; 15]), Err(OcrError::ImageSize { .. }))); + let ok = OcrImage::new(2, 2, vec![255; 16]).unwrap(); + assert_eq!((ok.width(), ok.height()), (2, 2)); + assert_eq!(ok.rgba().len(), 16); + assert_eq!(ok.gray(5, 5), Some(255), "clamped reads stay inside"); + assert_eq!(ok.gray(-1, -1), Some(255)); + } + mod test_image { /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. pub fn hello() -> (u32, u32, Vec) { diff --git a/crates/ocr/src/merge.rs b/crates/ocr/src/merge.rs new file mode 100644 index 0000000000..58fc17a4e5 --- /dev/null +++ b/crates/ocr/src/merge.rs @@ -0,0 +1,583 @@ +//! Combining engines. "ROVER" merges two engines' readings of one page: every primary word +//! adopts the best-overlapping unused secondary word when their boxes agree enough (IoU, below); +//! the higher confidence wins the text, box and confidence (a tie keeps the primary; against a +//! primary of unknown confidence the secondary must reach [`UNKNOWN_CONFIDENCE_BAR`]) and the +//! surviving word is named `"ROVER"`; secondary words nothing claimed are appended, keeping +//! their own engine's name. + +use crate::{Line, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// How the chosen engines combine. The default is [`MergeStrategy::PrimaryOnly`] with the +/// default engine (ocrs), which is today's behavior. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum MergeStrategy { + /// Only the primary engine runs. + #[default] + PrimaryOnly, + /// The secondary engine joins only when the primary's reading looks weak: its mean word + /// confidence is under 70, or its confidence is unknown (words were read but none reports + /// a confidence — the ocrs engine). An unknown reading is never counted as 100; an empty + /// primary keeps the fast path (no words, nothing to double-check). + ConfidenceWeighted, + /// Both engines always run and their readings merge (with no secondary engine this + /// degrades to primary only). + RoverVote, +} + +/// The mean of the confidences the words report, 0–100; `None` when the reading's confidence +/// is UNKNOWN — words were read but none reports a confidence (the ocrs engine's words). +/// Unknown is never counted as 100: there is no low-confidence signal, but a reading whose +/// confidence nobody knows is not therefore confident, so a confidence-weighted strategy +/// answers it by asking the secondary engine. An empty reading is `Some(100.0)`: no words +/// were read, so there is nothing to double-check and the fast path stands. +pub fn mean_confidence(lines: &[Line]) -> Option { + let mut total = 0usize; + let mut reported = (0.0f64, 0usize); + for word in lines.iter().flat_map(|l| &l.words) { + total += 1; + if let Some(c) = word.confidence { + reported.0 += c as f64; + reported.1 += 1; + } + } + match (total, reported.1) { + (0, _) => Some(100.0), + (_, 0) => None, + (_, n) => Some((reported.0 / n as f64) as f32), + } +} + +/// Intersection over union of two `[left, top, right, bottom]` boxes. An empty intersection is +/// 0; degenerate or NaN boxes give 0 rather than panicking or poisoning the comparison. +pub fn iou(a: &[f32; 4], b: &[f32; 4]) -> f64 { + let l = a[0].max(b[0]); + let t = a[1].max(b[1]); + let r = a[2].min(b[2]); + let btm = a[3].min(b[3]); + if !(r > l && btm > t) { + return 0.0; // disjoint, touching, inverted, or NaN (all comparisons false) + } + let inter = (r - l) as f64 * (btm - t) as f64; + let area = |x: &[f32; 4]| ((x[2] - x[0]).max(0.0) as f64) * ((x[3] - x[1]).max(0.0) as f64); + let union = area(a) + area(b) - inter; + if union <= 0.0 { + return 0.0; + } + inter / union +} + +/// The most primary×secondary word comparisons one ROVER merge will attempt. Known limit: a +/// merge over this cap is skipped — the primary reading stands alone, and the skip is noted, +/// never silent — so two enormous readings cannot turn the merge into quadratic work without +/// end. A real page is a few hundred words on each side, orders of magnitude under the cap. +pub const ROVER_MAX_COMPARISONS: usize = 100_000_000; + +/// Whether the two readings are too large to merge: their word counts multiply past +/// [`ROVER_MAX_COMPARISONS`]. Shared by [`rover_merge`] (which then keeps the primary) and +/// [`recognize_with_strategy`] (which then notes the skip). +fn over_cap(primary: &[Line], secondary: &[Line]) -> bool { + let primary_words: usize = primary.iter().map(|l| l.words.len()).sum(); + let secondary_words: usize = secondary.iter().map(|l| l.words.len()).sum(); + primary_words.saturating_mul(secondary_words) > ROVER_MAX_COMPARISONS +} + +/// The confidence a secondary word needs to replace a primary word whose confidence is +/// unknown (the ocrs engine reports none). It is the bar under which a reading counts as weak +/// ([`MergeStrategy::ConfidenceWeighted`] asks the second engine below it): a word the second +/// engine itself calls weak does not displace a reading nobody scored. +pub const UNKNOWN_CONFIDENCE_BAR: f32 = 70.0; + +/// True when the secondary word's confidence `a` beats the primary's `b`. Both known: the +/// strictly higher one wins (a tie keeps the primary). A known confidence against an unknown +/// primary wins only at or above [`UNKNOWN_CONFIDENCE_BAR`]: an unknown confidence is neither +/// 0 nor 100, so a 5% reading must not replace it, while a confident one may. NaN never wins, +/// and with no confidence on the secondary side the primary stands. +fn higher(a: Option, b: Option) -> bool { + match (a, b) { + (Some(a), Some(b)) => a.partial_cmp(&b) == Some(std::cmp::Ordering::Greater), + (Some(a), None) => a >= UNKNOWN_CONFIDENCE_BAR, + _ => false, + } +} + +/// Merge the two engines' readings. Primary words keep their order and lines; a word is +/// replaced by its overlapping secondary (IoU >= 0.5) only when that secondary is more +/// confident, and either way the merged word's [`Word::source`] is `"ROVER"`. Secondary words +/// no primary claimed are appended, in their own reading order and lines, under their own +/// engine's name (`secondary_id`). Known limit: over [`ROVER_MAX_COMPARISONS`] comparisons +/// (word counts multiplied) the primary reading is kept as is, unmerged; the strategy runner +/// names that skip in a note. +pub fn rover_merge(primary: Vec, secondary: Vec, secondary_id: &str) -> Vec { + if over_cap(&primary, &secondary) { + return primary; // over the comparison cap: the primary stands alone + } + let mut primary = primary; + // The secondary's words, marked `None` once a primary word claimed them, grouped by their + // own lines so the unclaimed ones can be appended in order. + let mut unused: Vec>> = secondary.into_iter().map(|l| l.words.into_iter().map(Some).collect()).collect(); + for line in &mut primary { + for word in &mut line.words { + // The best-overlapping unused secondary word, if any beats IoU 0.5. + let mut best: Option<(f64, usize, usize)> = None; + for (li, words) in unused.iter().enumerate() { + for (wi, slot) in words.iter().enumerate() { + let Some(w) = slot else { continue }; + let score = iou(&word.rect, &w.rect); + if score >= 0.5 && best.is_none_or(|(b, _, _)| score > b) { + best = Some((score, li, wi)); + } + } + } + let Some((_, li, wi)) = best else { continue }; + let claimed = unused.get_mut(li).and_then(|l| l.get_mut(wi)).and_then(|slot| slot.take()); + let Some(claim) = claimed else { continue }; + if higher(claim.confidence, word.confidence) { + *word = Word { source: "ROVER".into(), ..claim }; + } else { + // Tie (or the primary wins): keep the primary's text, box and confidence, but + // record that this reading is the engines' agreed one. + word.source = "ROVER".into(); + } + } + } + // Append the secondary words nothing claimed, line by line, under their engine's name + // (`secondary_id` backfills words that name no source of their own). + let mut merged = primary; + for line in unused { + let words: Vec = line + .into_iter() + .flatten() + .map(|mut w| { + if w.source.is_empty() { + w.source = secondary_id.into(); + } + w + }) + .collect(); + if !words.is_empty() { + merged.push(Line { words }); + } + } + merged +} + +/// What a strategy run produced: the merged (or primary-only) reading, whether the secondary +/// engine's reading is part of it, and the notes recorded along the way. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct StrategyOutcome { + pub lines: Vec, + /// Whether the secondary engine's reading is part of `lines`: true only when the merge + /// actually happened. A secondary that failed, or whose reading was discarded over the + /// comparison cap, is `false`, with a note in [`StrategyOutcome::notes`] saying why. + pub secondary_ran: bool, + /// Why the page's reading is partial, if it is. The secondary engine is an enhancement, + /// never a gate: when it is configured but fails, the primary reading is kept and the + /// failure is noted here — a page is never skipped because a partner could not read it. + /// Only a primary failure is an [`OcrError`]. + pub notes: Vec, +} + +/// The note recorded when an ensemble partner fails: the page keeps the primary reading and +/// the cause is named for the report. +fn degraded_note(id: &str, error: &OcrError) -> String { + format!("the {id} engine failed ({error}) — the page keeps the primary reading alone") +} + +/// The note recorded when the two readings are too large to merge: the page keeps the primary +/// reading alone and the discarded partner reading is named, never silent. +fn cap_note(id: &str, primary_words: usize, secondary_words: usize) -> String { + format!( + "the readings were too large to merge ({primary_words} × {secondary_words} comparisons, over the cap of {ROVER_MAX_COMPARISONS}) — the page keeps the primary reading alone and the {id} reading is unused" + ) +} + +/// Run the engines per `strategy` and merge. See [`StrategyOutcome`] for what comes back. +pub fn recognize_with_strategy( + primary: &dyn Recognizer, + secondary: Option<&dyn Recognizer>, + strategy: MergeStrategy, + image: &OcrImage, + options: &RecognizeOptions, +) -> Result { + // The shared shape of the ensemble strategies: the primary has read, the secondary joins + // or fails (a failure degrades to the primary, with a note). + let join = |p: Vec, sec: &dyn Recognizer| match sec.recognize(image, options) { + // Over the comparison cap the merge is skipped: the secondary's reading was read but + // is not part of the result, so the outcome says so — primary only, honestly not + // merged, with the skip named. + Ok(s) if over_cap(&p, &s) => { + let (primary_words, secondary_words) = (p.iter().map(|l| l.words.len()).sum::(), s.iter().map(|l| l.words.len()).sum::()); + StrategyOutcome { lines: p, secondary_ran: false, notes: vec![cap_note(sec.id(), primary_words, secondary_words)] } + } + Ok(s) => StrategyOutcome { lines: rover_merge(p, s, sec.id()), secondary_ran: true, notes: Vec::new() }, + Err(e) => StrategyOutcome { lines: p, secondary_ran: false, notes: vec![degraded_note(sec.id(), &e)] }, + }; + match strategy { + MergeStrategy::PrimaryOnly => Ok(StrategyOutcome { lines: primary.recognize(image, options)?, secondary_ran: false, notes: Vec::new() }), + MergeStrategy::RoverVote => match secondary { + None => Ok(StrategyOutcome { lines: primary.recognize(image, options)?, secondary_ran: false, notes: Vec::new() }), + Some(sec) => { + let p = primary.recognize(image, options)?; + Ok(join(p, sec)) + } + }, + MergeStrategy::ConfidenceWeighted => { + let p = primary.recognize(image, options)?; + match mean_confidence(&p) { + // Confident enough — or an empty reading: the primary stands alone. + Some(mean) if mean >= UNKNOWN_CONFIDENCE_BAR => Ok(StrategyOutcome { lines: p, secondary_ran: false, notes: Vec::new() }), + // A weak mean, or an unknown confidence (words, none reported): the secondary + // runs. + _ => match secondary { + None => Ok(StrategyOutcome { lines: p, secondary_ran: false, notes: Vec::new() }), + Some(sec) => Ok(join(p, sec)), + }, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// A fake engine: returns canned lines, counts recognize calls, never available. + struct Fake { + id: &'static str, + lines: Vec, + calls: AtomicUsize, + fail: bool, + } + + impl Fake { + fn new(id: &'static str, words: &[(&str, [f32; 4], f32)]) -> Fake { + Fake { + id, + lines: vec![Line { words: words.iter().map(|(t, r, c)| Word::new(*t, *r, id).with_confidence(*c)).collect() }], + calls: AtomicUsize::new(0), + fail: false, + } + } + + fn empty(id: &'static str) -> Fake { + Fake { id, lines: Vec::new(), calls: AtomicUsize::new(0), fail: false } + } + + /// An engine like ocrs: words, but no confidence reported on any of them. + fn unknown(id: &'static str, n: usize) -> Fake { + Fake { + id, + lines: vec![Line { + words: (0..n).map(|i| Word::new(format!("w{i}"), [i as f32 * 10.0, 0.0, i as f32 * 10.0 + 8.0, 10.0], id)).collect(), + }], + calls: AtomicUsize::new(0), + fail: false, + } + } + } + + impl Recognizer for Fake { + fn id(&self) -> &'static str { + self.id + } + fn available(&self) -> bool { + !self.fail + } + fn languages(&self) -> Vec<(String, String)> { + vec![] + } + fn recognize(&self, _image: &OcrImage, _options: &RecognizeOptions) -> Result, OcrError> { + self.calls.fetch_add(1, Ordering::SeqCst); + if self.fail { + return Err(OcrError::Recognize("the fake engine failed".into())); + } + Ok(self.lines.clone()) + } + } + + fn image() -> OcrImage { + OcrImage::new(4, 4, vec![255; 64]).unwrap() + } + + fn words(lines: &[Line]) -> Vec<&Word> { + lines.iter().flat_map(|l| &l.words).collect() + } + + #[test] + fn iou_of_disjoint_touching_and_nan_boxes_is_zero() { + let a = [0.0, 0.0, 10.0, 10.0]; + assert_eq!(iou(&a, &[20.0, 0.0, 30.0, 10.0]), 0.0, "disjoint"); + assert_eq!(iou(&a, &[10.0, 0.0, 20.0, 10.0]), 0.0, "touching edges are an empty intersection"); + assert_eq!(iou(&a, &[f32::NAN, 0.0, 10.0, 10.0]), 0.0, "NaN boxes score 0, not a panic"); + assert_eq!(iou(&a, &a), 1.0, "identical boxes"); + assert!((iou(&a, &[5.0, 0.0, 15.0, 10.0]) - 1.0 / 3.0).abs() < 1e-9, "half overlap is IoU 1/3"); + } + + /// With nothing on the primary side, every secondary word is appended under the secondary + /// engine's own name. + #[test] + fn rover_appends_everything_when_the_primary_is_empty() { + let primary = Vec::new(); + let secondary = vec![Line { words: vec![Word::new("sole", [0.0, 0.0, 9.0, 9.0], "tesseract").with_confidence(80.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!(words(&merged).len(), 1); + assert_eq!(merged[0].words[0].text, "sole"); + assert_eq!(merged[0].words[0].source, "tesseract", "unclaimed words keep their own engine"); + } + + /// Non-overlapping secondary words never displace primary words; they are appended. + #[test] + fn rover_appends_non_overlapping_secondary_words() { + let primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let secondary = vec![Line { words: vec![Word::new("extra", [100.0, 100.0, 120.0, 110.0], "tesseract").with_confidence(99.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2); + assert_eq!((w[0].text.as_str(), w[0].source.as_str()), ("kept", "ocrs"), "the more confident but disjoint word does not win"); + assert_eq!((w[1].text.as_str(), w[1].source.as_str()), ("extra", "tesseract")); + } + + /// The higher-confidence word of an overlapping pair wins text, box and confidence, and is + /// named ROVER. + #[test] + fn rover_higher_confidence_wins_and_is_named_rover() { + let primary = vec![Line { words: vec![Word::new("helo", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(60.0)] }]; + let secondary = vec![Line { words: vec![Word::new("hello", [1.0, 0.0, 11.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 1, "the secondary word was claimed, not appended"); + assert_eq!(w[0].text, "hello"); + assert_eq!(w[0].rect, [1.0, 0.0, 11.0, 10.0]); + assert_eq!(w[0].confidence, Some(95.0)); + assert_eq!(w[0].source, "ROVER"); + } + + /// A tie keeps the primary word (still named ROVER, as an agreed reading). + #[test] + fn rover_tie_keeps_the_primary() { + let primary = vec![Line { words: vec![Word::new("primary", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(80.0)] }]; + let secondary = vec![Line { words: vec![Word::new("secondary", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(80.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 1); + assert_eq!(w[0].text, "primary"); + assert_eq!(w[0].source, "ROVER"); + } + + /// Boxes under the 0.5 IoU bar never merge, even when the secondary is more confident. + #[test] + fn rover_below_half_iou_does_not_merge() { + let primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + // IoU 10·10 / (100 + 169 - 25) = 100/244 ≈ 0.41 < 0.5. + let secondary = vec![Line { words: vec![Word::new("other", [5.0, 0.0, 23.0, 20.0], "tesseract").with_confidence(99.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2); + assert_eq!(w[0].text, "kept"); + assert_eq!(w[0].source, "ocrs"); + } + + /// Merged words keep their line; the merge never panics on NaN boxes. + #[test] + fn rover_survives_nan_boxes() { + let primary = vec![Line { words: vec![Word::new("nan", [f32::NAN, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let secondary = vec![Line { words: vec![Word::new("real", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(90.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2, "a NaN primary box merges with nothing"); + assert_eq!(w[0].text, "nan"); + assert_eq!(w[1].text, "real"); + } + + /// Only reported confidences count. An empty reading is Some(100) (nothing to + /// double-check); words without a single confidence are UNKNOWN (`None`), never 100. + #[test] + fn mean_confidence_distinguishes_empty_from_unknown() { + assert_eq!(mean_confidence(&[]), Some(100.0)); + assert_eq!(mean_confidence(&[Line { words: vec![Word::new("a", [0.0; 4], "ocrs")] }]), None, "words, no confidences: unknown"); + let lines = vec![Line { + words: vec![ + Word::new("a", [0.0; 4], "e").with_confidence(40.0), + Word::new("b", [0.0; 4], "e").with_confidence(80.0), + Word::new("c", [0.0; 4], "e"), + ], + }]; + assert_eq!(mean_confidence(&lines), Some(60.0), "reported confidences are averaged"); + } + + /// ROVER's preference needs a meaningful comparison: a known confidence beats an unknown + /// one (the ocrs primary yields to a confident tesseract reading), and None vs None keeps + /// the primary. + #[test] + fn rover_known_confidence_beats_unknown_and_none_vs_none_keeps_the_primary() { + let primary = vec![Line { words: vec![Word::new("helo", [0.0, 0.0, 10.0, 10.0], "ocrs")] }]; + let secondary = vec![Line { words: vec![Word::new("hello", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!( + (merged[0].words[0].text.as_str(), merged[0].words[0].confidence, merged[0].words[0].source.as_str()), + ("hello", Some(95.0), "ROVER") + ); + let primary = vec![Line { words: vec![Word::new("primary", [0.0, 0.0, 10.0, 10.0], "ocrs")] }]; + let secondary = vec![Line { words: vec![Word::new("secondary", [0.0, 0.0, 10.0, 10.0], "tesseract")] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!( + (merged[0].words[0].text.as_str(), merged[0].words[0].source.as_str()), + ("primary", "ROVER"), + "unknown vs unknown is not a comparison: the primary stands" + ); + } + + /// Over the comparison cap the merge is skipped and the primary reading stands alone (a + /// documented known limit, bounding the merge's quadratic core). + #[test] + fn rover_over_the_comparison_cap_keeps_the_primary() { + // 11 000 × 11 000 = 121 M comparisons > the 100 M cap. + let words = |id: &str| (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], id)).collect::>(); + let primary = vec![Line { words: words("ocrs") }]; + let secondary = vec![Line { words: words("tesseract") }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!(merged.len(), 1); + assert!(merged[0].words.iter().all(|w| w.source == "ocrs"), "every primary word stands, unmerged"); + assert_eq!(merged[0].words.len(), 11_000); + // Under the cap, merging still happens (a small overlap is replaced and named ROVER). + let small_primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let small_secondary = vec![Line { words: vec![Word::new("wins", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(small_primary, small_secondary, "tesseract"); + assert_eq!((merged[0].words[0].text.as_str(), merged[0].words[0].source.as_str()), ("wins", "ROVER")); + } + + /// Over the comparison cap the merge is skipped honestly at the strategy level: the page + /// keeps the primary reading alone, the skip is named in a note, and `secondary_ran` does + /// not claim the discarded reading is part of the result (though the secondary really did + /// run — its work was spent, its reading unused). + #[test] + fn rover_cap_hit_reports_primary_only_with_a_note_and_an_honest_flag() { + let mut primary = Fake::empty("ocrs"); + primary.lines = + vec![Line { words: (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], "ocrs")).collect() }]; + let mut secondary = Fake::empty("tesseract"); + secondary.lines = + vec![Line { words: (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], "tesseract")).collect() }]; + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1, "the secondary really ran"); + assert_eq!(outcome.lines.len(), 1); + assert_eq!(outcome.lines[0].words.len(), 11_000, "primary only"); + assert!(outcome.lines[0].words.iter().all(|w| w.source == "ocrs"), "every primary word stands, unmerged"); + assert!(!outcome.secondary_ran, "the secondary's reading is not part of the result"); + assert_eq!(outcome.notes.len(), 1, "{:?}", outcome.notes); + assert!(outcome.notes[0].contains("merge"), "{:?}", outcome.notes[0]); + assert!(outcome.notes[0].contains("tesseract"), "{:?}", outcome.notes[0]); + // Under the cap the same strategy merges and reports the secondary honestly. + let primary = Fake::new("ocrs", &[("kept", [0.0, 0.0, 10.0, 10.0], 50.0)]); + let secondary = Fake::new("tesseract", &[("wins", [0.0, 0.0, 10.0, 10.0], 95.0)]); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran && outcome.notes.is_empty()); + assert_eq!((outcome.lines[0].words[0].text.as_str(), outcome.lines[0].words[0].source.as_str()), ("wins", "ROVER")); + } + + #[test] + fn primary_only_never_runs_the_secondary() { + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 90.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = + recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::PrimaryOnly, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(primary.calls.load(Ordering::SeqCst), 1); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 0); + assert!(!outcome.secondary_ran); + assert!(outcome.notes.is_empty()); + assert_eq!(words(&outcome.lines).len(), 1); + } + + #[test] + fn confidence_weighted_runs_the_secondary_only_on_a_weak_primary() { + let weak = Fake::new("ocrs", &[("w", [0.0; 4], 40.0)]); + let strong = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = + recognize_with_strategy(&weak, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran, "mean 40 < 70 → the secondary joins"); + let outcome = + recognize_with_strategy(&strong, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran, "mean 95 >= 70 → primary alone"); + // An empty primary counts as 100: no signal, no secondary run. + let empty = Fake::empty("ocrs"); + let outcome = + recognize_with_strategy(&empty, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1, "only the weak page ran the secondary"); + } + + /// The ocrs engine reports no confidences: unknown is not 100, so the secondary runs — + /// and its known reading wins the overlapping word (the ensemble's whole point). + #[test] + fn confidence_weighted_runs_the_secondary_on_an_unknown_primary() { + let ocrs_like = Fake::unknown("ocrs", 1); + let secondary = Fake::new("tesseract", &[("hello", [0.0, 0.0, 10.0, 10.0], 95.0)]); + let outcome = + recognize_with_strategy(&ocrs_like, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran, "an unknown confidence asks the secondary"); + assert_eq!((outcome.lines[0].words[0].text.as_str(), outcome.lines[0].words[0].confidence), ("hello", Some(95.0))); + assert_eq!(outcome.lines[0].words[0].source, "ROVER"); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1); + } + + /// Against a primary word of unknown confidence, a secondary word wins only when it is + /// confident (at or above the weak-reading bar): a 5% reading does not replace a word nobody + /// scored, a 70% one does. The merged word is named ROVER either way. + #[test] + fn an_unknown_primary_word_yields_only_to_a_confident_secondary() { + let merge = |conf: f32| { + let secondary = vec![Line { words: vec![Word::new("other", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(conf)] }]; + let primary = Fake::unknown("ocrs", 1).lines; + rover_merge(primary, secondary, "tesseract").remove(0).words.remove(0) + }; + let weak = merge(5.0); + assert_eq!((weak.text.as_str(), weak.confidence, weak.source.as_str()), ("w0", None, "ROVER"), "a 5% word keeps the primary"); + let below = merge(UNKNOWN_CONFIDENCE_BAR - 0.5); + assert_eq!(below.text, "w0"); + let at_bar = merge(UNKNOWN_CONFIDENCE_BAR); + assert_eq!((at_bar.text.as_str(), at_bar.confidence), ("other", Some(UNKNOWN_CONFIDENCE_BAR)), "a confident word wins"); + let nan = merge(f32::NAN); + assert_eq!(nan.text, "w0", "NaN (unknown after clamping) never wins"); + } + + #[test] + fn rover_vote_always_runs_both_and_degrades_without_a_secondary() { + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1); + // No secondary: primary only. + let outcome = recognize_with_strategy(&primary, None, MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran); + } + + /// A configured secondary that fails degrades to the primary reading, with the failure + /// recorded as a note — never a skipped page, never a silently partial reading without a + /// trace. Both ensemble strategies degrade; a primary failure is still an error. + #[test] + fn a_failing_secondary_degrades_to_the_primary_with_a_note() { + let mut secondary = Fake::empty("tesseract"); + secondary.fail = true; + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(words(&outcome.lines).len(), 1, "the primary reading stands"); + assert_eq!(outcome.lines[0].words[0].text, "w"); + assert!(!outcome.secondary_ran); + assert_eq!(outcome.notes.len(), 1); + assert!(outcome.notes[0].contains("tesseract"), "{:?}", outcome.notes[0]); + // The confidence-weighted strategy degrades the same way on its weak path. + let weak = Fake::new("ocrs", &[("w", [0.0; 4], 40.0)]); + let outcome = + recognize_with_strategy(&weak, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(words(&outcome.lines).len(), 1); + assert_eq!(outcome.notes.len(), 1, "{:?}", outcome.notes); + // A failing primary is still an error: there is nothing to degrade to. + let mut failing_primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + failing_primary.fail = true; + let r = recognize_with_strategy(&failing_primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()); + assert!(r.is_err()); + } +} diff --git a/crates/ocr/src/ocrs.rs b/crates/ocr/src/ocrs.rs new file mode 100644 index 0000000000..7f8feb47c4 --- /dev/null +++ b/crates/ocr/src/ocrs.rs @@ -0,0 +1,251 @@ +//! The pure-Rust engine: the ocrs crate with its pre-trained models, wrapped as a +//! [`Recognizer`] (`OcrsRecognizer`). Reading is offline and local; the models do not report a +//! confidence, so its words have `confidence: None`. + +use std::path::{Path, PathBuf}; + +use super::{Line, MAX_PIXELS, MAX_SIDE, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// The validated byte length of a `width` × `height` RGBA raster, with the same caps +/// [`OcrImage::new`] applies: zero sides are empty, sides over [`MAX_SIDE`] or totals over +/// [`MAX_PIXELS`] are too large. `Ocr::recognize` takes raw dimensions (they come from +/// callers, ultimately from documents, so they are untrusted), and the naive +/// `width * height * 4` overflows `u32` for absurd values — the caps are checked first, in +/// `usize`, so nothing overflows and nothing huge is ever touched. +fn checked_len(width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Err(OcrError::EmptyImage); + } + if width > MAX_SIDE || height > MAX_SIDE { + return Err(OcrError::ImageTooLarge); + } + let pixels = (width as usize).checked_mul(height as usize).ok_or(OcrError::ImageTooLarge)?; + if pixels > MAX_PIXELS as usize { + return Err(OcrError::ImageTooLarge); + } + pixels.checked_mul(4).ok_or(OcrError::ImageTooLarge) +} + +/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. +pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; + +/// The model files, as named in ATTRIBUTION.toml. +pub const DETECTION_MODEL: &str = "text-detection.onnx"; +pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; + +pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +/// Where the two model files are. +#[derive(Clone, Debug, PartialEq)] +pub struct Models { + pub detection: PathBuf, + pub recognition: PathBuf, +} + +impl Models { + pub fn dirs_beside_exe(exe_dir: &Path) -> Vec { + vec![exe_dir.join("models"), exe_dir.join("../Resources/models"), exe_dir.join("../share/pdfcraft/models")] + } + + /// The models in `dir`, preferring current ONNX files over legacy RTEN files. An incomplete + /// update never mixes formats or hides an intact legacy pair. + pub fn in_dir(dir: &Path) -> Option { + [(DETECTION_MODEL, RECOGNITION_MODEL), ("text-detection.rten", "text-recognition.rten")].into_iter().find_map(|(detection, recognition)| { + let m = Models { detection: dir.join(detection), recognition: dir.join(recognition) }; + (m.detection.is_file() && m.recognition.is_file()).then_some(m) + }) + } + + /// Look for the models: `$PDFCRAFT_MODELS`, then `models/` beside the executable (and + /// `Resources/models` in a macOS bundle), then the source tree's `assets/models/`. + pub fn find() -> Option { + Self::search_dirs().iter().find_map(|d| Self::in_dir(d)) + } + + /// The directories [`Models::find`] looks in, in order. + pub fn search_dirs() -> Vec { + let mut dirs = Vec::new(); + if let Some(d) = std::env::var_os("PDFCRAFT_MODELS") { + dirs.push(PathBuf::from(d)); + } + if let Some(exe) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { + dirs.extend(Self::dirs_beside_exe(&exe)); + } + dirs.push(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/models")); + dirs + } +} + +/// A loaded recogniser. Loading takes a moment; keep one and reuse it. +pub struct Ocr { + engine: ocrs::OcrEngine, +} + +impl Ocr { + pub fn load(models: &Models) -> Result { + let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); + let params = ocrs::OcrEngineParams { + detection_model: Some(load(&models.detection)?), + recognition_model: Some(load(&models.recognition)?), + alphabet: Some(ALPHABET.into()), + ..Default::default() + }; + let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; + Ok(Ocr { engine }) + } + + /// Load the models found by [`Models::find`]. + pub fn find() -> Result { + Self::load(&Models::find().ok_or(OcrError::NoModels)?) + } + + /// Recognise the text in an RGBA (or RGB, or grey) image, `width` × `height` pixels. + pub fn recognize(&self, pixels: &[u8], width: u32, height: u32) -> Result, OcrError> { + if pixels.is_empty() { + return Err(OcrError::EmptyImage); + } + let want = checked_len(width, height)?; + // ocrs wants 1 or 3 channels. + let rgb: std::borrow::Cow<[u8]> = if pixels.len() == want { + pixels.as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() + } else { + pixels.into() + }; + let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); + let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; + let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; + let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; + let lines = self.engine.find_text_lines(&input, &found); + let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; + use ocrs::TextItem; + Ok(read + .into_iter() + .flatten() + .map(|line| Line { + words: line + .words() + .map(|w| { + let r = w.bounding_rect(); + Word::new(w.to_string(), [r.left() as f32, r.top() as f32, r.right() as f32, r.bottom() as f32], "ocrs") + }) + .filter(|w| !w.text.trim().is_empty()) + .collect(), + }) + .filter(|l| !l.words.is_empty()) + .collect()) + } +} + +/// The ocrs engine as a [`Recognizer`] (the default engine). +pub struct OcrsRecognizer { + engine: Ocr, +} + +impl OcrsRecognizer { + /// Load the models from `models`. + pub fn load(models: &Models) -> Result { + Ok(OcrsRecognizer { engine: Ocr::load(models)? }) + } + + /// Load the models found by [`Models::find`]. + pub fn find() -> Result { + Self::load(&Models::find().ok_or(OcrError::NoModels)?) + } +} + +impl Recognizer for OcrsRecognizer { + fn id(&self) -> &'static str { + "ocrs" + } + + fn available(&self) -> bool { + Models::find().is_some() + } + + fn languages(&self) -> Vec<(String, String)> { + LANGUAGES.iter().map(|(c, n)| ((*c).to_string(), (*n).to_string())).collect() + } + + fn recognize(&self, image: &OcrImage, _options: &RecognizeOptions) -> Result, OcrError> { + self.engine.recognize(image.rgba(), image.width(), image.height()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Raw dimensions are untrusted (they come from callers, ultimately from documents): zero + /// sides, sides or pixel totals over the caps, and products that would overflow the naive + /// `width * height * 4` are errors, never panics, before anything is touched. + #[test] + fn raw_dimensions_are_capped_and_checked_not_panicked_at() { + assert!(matches!(checked_len(0, 10), Err(OcrError::EmptyImage))); + assert!(matches!(checked_len(10, 0), Err(OcrError::EmptyImage))); + assert!(matches!(checked_len(MAX_SIDE + 1, 1), Err(OcrError::ImageTooLarge))); + // 65536² would wrap u32 in the old `width * height * 4`; the side cap refuses it first. + assert!(matches!(checked_len(u32::MAX, u32::MAX), Err(OcrError::ImageTooLarge))); + // Legal sides whose product is over the 64 MP area cap. + assert!(matches!(checked_len(MAX_SIDE, 7000), Err(OcrError::ImageTooLarge))); + assert_eq!(checked_len(2, 2).unwrap(), 16); + // Exactly the area cap fits: 8000 × 8000 = 64 MP. + assert_eq!(checked_len(8000, 8000).unwrap(), 8000usize * 8000 * 4); + } + + /// The current ONNX pair wins when an installation also retains both legacy models. + #[test] + fn model_discovery_prefers_the_complete_onnx_pair() { + let dir = tempfile::tempdir().unwrap(); + for file in ["text-detection.onnx", "text-recognition.onnx"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + let expected = Some(Models { detection: dir.path().join("text-detection.onnx"), recognition: dir.path().join("text-recognition.onnx") }); + assert_eq!(Models::in_dir(dir.path()), expected, "a complete ONNX pair is sufficient"); + for file in ["text-detection.rten", "text-recognition.rten"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!(Models::in_dir(dir.path()), expected, "the complete ONNX pair takes precedence over legacy models"); + } + + /// An interrupted ONNX update still permits an intact legacy pair in the same directory. + #[test] + fn model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete() { + for current in ["text-detection.onnx", "text-recognition.onnx"] { + let dir = tempfile::tempdir().unwrap(); + for file in [current, "text-detection.rten", "text-recognition.rten"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!( + Models::in_dir(dir.path()), + Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }), + "{current} alone must not hide a complete legacy pair" + ); + } + } + + /// Detection and recognition must come from one complete format pair, never a mixed update. + #[test] + fn model_discovery_refuses_mixed_format_pairs() { + for pair in [["text-detection.onnx", "text-recognition.rten"], ["text-detection.rten", "text-recognition.onnx"]] { + let dir = tempfile::tempdir().unwrap(); + for file in pair { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!(Models::in_dir(dir.path()), None, "{pair:?} is not a complete model pair"); + } + } + + /// Existing source builds and PDFCRAFT_MODELS overrides retain their complete RTEN pair. + #[test] + fn model_discovery_keeps_legacy_only_installations() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!(Models::in_dir(dir.path()), None); + std::fs::write(dir.path().join("text-detection.rten"), b"model").unwrap(); + assert_eq!(Models::in_dir(dir.path()), None, "one legacy model is incomplete"); + std::fs::write(dir.path().join("text-recognition.rten"), b"model").unwrap(); + assert_eq!( + Models::in_dir(dir.path()), + Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }) + ); + } +} diff --git a/crates/ocr/src/preprocess.rs b/crates/ocr/src/preprocess.rs new file mode 100644 index 0000000000..280ee73fec --- /dev/null +++ b/crates/ocr/src/preprocess.rs @@ -0,0 +1,973 @@ +//! Preprocessing: a pure function between the page raster and recognition, in fixed order — +//! (a) DPI normalization, then the four opt-in steps (b) orientation, (c) deskew, (d) denoise, +//! (e) binarization. Normalization exists for those steps (their windows and thresholds are +//! calibrated for 300 dpi), so it runs only when at least one of them is on: with every step +//! off the raster is handed to the engines exactly as it was rendered. (Enlarging a render +//! on its own changes what the models read: a 72 dpi scan of "due: €250" read as "ue: ?250" +//! after a 4× bilinear enlargement.) Everything is plain Rust on the RGBA raster: no +//! Leptonica, no process, no new dependencies. +//! +//! Every geometric step records its inverse; the result carries one exact +//! [`InverseTransform`] (an affine map, composed in f64), and callers map every word box +//! through it back to the *original* raster coordinates before placing the words. +//! +//! The heuristics are deliberately conservative and panic-free: an image too small to judge, +//! a blank page, or a score without a clear margin is a no-op, never a guess. + +use crate::{MAX_PIXELS, MAX_SIDE, OcrError, OcrImage}; + +/// The four opt-in steps. All off by default: a scan is passed through as rendered. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PreprocessOptions { + /// Detect pages scanned at 90°/180°/270° and turn them upright. + pub auto_rotate: bool, + /// Straighten skewed text lines. + pub deskew: bool, + /// 3×3 median filter (salt-and-pepper noise). + pub denoise: bool, + /// Sauvola binarization (window 8, k 0.34). + pub binarize: bool, +} + +impl PreprocessOptions { + /// Whether any step is on (and so the raster is normalized to 300 dpi first). + pub fn any(&self) -> bool { + self.auto_rotate || self.deskew || self.denoise || self.binarize + } +} + +/// A preprocessed raster: the image to recognize, the exact way back to the input raster, and +/// the resolution the image now stands for. +#[derive(Debug)] +pub struct Preprocessed { + pub image: OcrImage, + /// Maps pixel coordinates in [`Preprocessed::image`] back to the input raster's. + pub inverse_transform: InverseTransform, + /// The image's resolution in dots per inch: 300 after normalization (only when a step is + /// on), or the source's own + /// when it already rendered at 300 or more. + pub dpi: f32, +} + +/// Preprocess `image` (rendered at `source_dpi`) per `options`, in the fixed order. The image +/// is consumed: the common path (every step off, or a no-op) hands the raster back without +/// copying it, and a step that rewrites pixels replaces it. +pub fn preprocess(image: OcrImage, source_dpi: f32, options: &PreprocessOptions) -> Result { + let mut current = image; + let mut inverse = InverseTransform::identity(); + // (a) DPI normalization: for the steps below, a scan under 300 dpi is enlarged by 300/src so + // they see letter-sized strokes. With every step off nothing is resampled. A broken dpi + // (zero, negative, NaN) is left alone. + let mut dpi = if source_dpi.is_finite() && source_dpi >= 1.0 { source_dpi } else { 300.0 }; + if options.any() && dpi < 300.0 { + let scale = f64::from(300.0 / dpi); + current = resize_bilinear(¤t, scale)?; + inverse = compose(&inverse, &InverseTransform::scale(1.0 / scale)); + dpi = 300.0; + } + // (b) Orientation: turn 90/180/270 scans upright (opt-in, skipped for small images). + if options.auto_rotate { + let (w, h) = (current.width, current.height); + let turns = detect_orientation(¤t); + if turns != 0 { + current = rotate_quarter_turns(¤t, turns); + inverse = compose(&inverse, &inverse_quarter_turns(turns, w, h)); + } + } + // (c) Deskew: straighten skewed lines (opt-in; no-op under 64 px, blank, or without a + // clear margin or below 0.1°). + if options.deskew + && let Some(angle) = estimate_skew(¤t) + { + let (straightened, back) = deskew_rotate(¤t, angle); + current = straightened; + inverse = compose(&inverse, &back); + } + // (d) Denoise: 3×3 median with edge clamping (opt-in; dimensions never change). + if options.denoise { + current = median3(¤t); + } + // (e) Binarize: Sauvola (window 8, k 0.34) via an integral image (opt-in); dark ink stays + // dark and paper stays light. + if options.binarize { + current = sauvola(¤t); + } + Ok(Preprocessed { image: current, inverse_transform: inverse, dpi }) +} + +/// An affine map `(x, y) → (a·x + c·y + e, b·x + d·y + f)`, composed in f64 so mapping a +/// word box back is exact up to float rounding. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct InverseTransform { + a: f64, + b: f64, + c: f64, + d: f64, + e: f64, + f: f64, +} + +impl InverseTransform { + pub fn identity() -> InverseTransform { + InverseTransform { a: 1.0, b: 0.0, c: 0.0, d: 1.0, e: 0.0, f: 0.0 } + } + + fn scale(s: f64) -> InverseTransform { + InverseTransform { a: s, b: 0.0, c: 0.0, d: s, e: 0.0, f: 0.0 } + } + + /// `compose(outer, inner)` maps `p` to `outer(inner(p))`: the accumulated inverse of the + /// earlier steps is the outer map (their spaces come later in the round trip), so each + /// step calls `compose(&inverse, &this_step_inverse)`. The scale step, for instance, + /// happens first, so its inverse is applied last: `scale⁻¹(rotate⁻¹(p))`. + fn compose_(outer: &InverseTransform, inner: &InverseTransform) -> InverseTransform { + InverseTransform { + a: outer.a * inner.a + outer.c * inner.b, + b: outer.b * inner.a + outer.d * inner.b, + c: outer.a * inner.c + outer.c * inner.d, + d: outer.b * inner.c + outer.d * inner.d, + e: outer.a * inner.e + outer.c * inner.f + outer.e, + f: outer.b * inner.e + outer.d * inner.f + outer.f, + } + } + + /// Map a point from the processed image back to the input raster. + pub fn apply(&self, x: f64, y: f64) -> (f64, f64) { + (self.a * x + self.c * y + self.e, self.b * x + self.d * y + self.f) + } + + /// Map a word box `[left, top, right, bottom]` back to the input raster: the four corners + /// go through [`Self::apply`] and their bounding box is rounded outward to whole pixels. + /// A box with no finite corners comes back unchanged (garbage in, garbage out — never a + /// panic). + pub fn map_rect(&self, rect: [f32; 4]) -> [f32; 4] { + let corners = [(rect[0], rect[1]), (rect[2], rect[1]), (rect[0], rect[3]), (rect[2], rect[3])]; + let mut lo = (f64::INFINITY, f64::INFINITY); + let mut hi = (f64::NEG_INFINITY, f64::NEG_INFINITY); + let mut finite = false; + for (x, y) in corners { + if !(x.is_finite() && y.is_finite()) { + continue; + } + let (mx, my) = self.apply(f64::from(x), f64::from(y)); + if mx.is_finite() && my.is_finite() { + finite = true; + lo = (lo.0.min(mx), lo.1.min(my)); + hi = (hi.0.max(mx), hi.1.max(my)); + } + } + if !finite { + return rect; + } + [lo.0.floor() as f32, lo.1.floor() as f32, hi.0.ceil() as f32, hi.1.ceil() as f32] + } +} + +/// The step inverses compose in pipeline order: each new step's inverse is applied *after* +/// the inverses accumulated so far. +fn compose(outer: &InverseTransform, inner: &InverseTransform) -> InverseTransform { + InverseTransform::compose_(outer, inner) +} + +/// The size `width` x `height` becomes under `scale`, refused (before any buffer exists) when +/// a side is under one pixel or over [`MAX_SIDE`], or the area is over [`MAX_PIXELS`]: two +/// sides under the side cap can still ask for hundreds of megabytes, which `OcrImage::new` +/// would only refuse after they were allocated. +fn scaled_size(width: u32, height: u32, scale: f64) -> Result<(u32, u32), OcrError> { + let new_w = (f64::from(width) * scale).ceil(); + let new_h = (f64::from(height) * scale).ceil(); + if !(1.0..=f64::from(MAX_SIDE)).contains(&new_w) || !(1.0..=f64::from(MAX_SIDE)).contains(&new_h) { + return Err(OcrError::ImageTooLarge); + } + let (nw, nh) = (new_w as u32, new_h as u32); + if (nw as usize).checked_mul(nh as usize).is_none_or(|pixels| pixels > MAX_PIXELS as usize) { + return Err(OcrError::ImageTooLarge); + } + Ok((nw, nh)) +} + +/// Bilinear resize by `scale`, clamped at the edges; the result stays inside [`scaled_size`]'s +/// limits (the side cap and the area cap). +fn resize_bilinear(image: &OcrImage, scale: f64) -> Result { + let (nw, nh) = scaled_size(image.width, image.height, scale)?; + let mut out = vec![255u8; (nw as usize).checked_mul(nh as usize).and_then(|p| p.checked_mul(4)).ok_or(OcrError::ImageTooLarge)?]; + for (y, row) in out.chunks_exact_mut(4 * nw as usize).enumerate() { + for (x, px) in row.as_chunks_mut::<4>().0.iter_mut().enumerate() { + // The source position of the destination pixel's centre, and the four + // neighbours bilinear interpolation blends (positions clamp at the edges). + let sx = (x as f64 + 0.5) / scale - 0.5; + let sy = (y as f64 + 0.5) / scale - 0.5; + let (x0, y0) = (sx.floor().max(0.0), sy.floor().max(0.0)); + let (fx, fy) = (sx - x0, sy - y0); + for (ch, o) in px.iter_mut().enumerate() { + let blend = |dx: f64, dy: f64| f64::from(image.byte(x0 + dx, y0 + dy, ch)); + let v = blend(0.0, 0.0) * (1.0 - fx) * (1.0 - fy) + + blend(1.0, 0.0) * fx * (1.0 - fy) + + blend(0.0, 1.0) * (1.0 - fx) * fy + + blend(1.0, 1.0) * fx * fy; + *o = v.round().clamp(0.0, 255.0) as u8; + } + } + } + OcrImage::new(nw, nh, out) +} + +/// Grayscale (Rec.601 luma) of every pixel. +fn grayscale(image: &OcrImage) -> Vec { + image.rgba.as_chunks::<4>().0.iter().map(|p| ((299 * p[0] as u32 + 587 * p[1] as u32 + 114 * p[2] as u32) / 1000).min(255) as u8).collect() +} + +/// How the page must be turned (quarter turns, clockwise) to read upright. +type QuarterTurns = u32; + +/// Detect the correction for pages scanned sideways or upside down. Method (documented, +/// panic-free, no OCR): ink is everything darker than the midpoint between the darkest and +/// lightest pixel; for each candidate correction (0/90/180/270 clockwise) the ink's row +/// profile is taken and scored by how sharply it bands (Σ Δ², the projection-profile score: +/// upright text lines band sharply, turned text does not). The best-banding axis wins when it +/// beats the other axis by a factor of 1.3; within the winning axis the line bands' +/// bottom-heaviness (real text carries more ink below each line's middle — baselines and +/// descenders) picks the quarter turn, with a 4%-of-ink margin. Anything less clear is a +/// no-op (`0`), and images under 32 px are never judged. +fn detect_orientation(image: &OcrImage) -> QuarterTurns { + if image.min_side() < 32 { + return 0; + } + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return 0 }; + let t = (*lo as u32 + *hi as u32) / 2; + let ink: Vec = gray.iter().map(|&g| (g as u32) < t).collect(); + let (w, h) = (image.width as usize, image.height as usize); + let total: u64 = ink.iter().filter(|i| **i).count() as u64; + if total == 0 { + return 0; + } + // The ink grid rotated by k quarter turns (clockwise), as row profiles. + let mut scores = [0.0f64; 4]; + let mut asyms = [0.0f64; 4]; + for k in 0u32..4 { + let k = k as usize; + let profile = rotated_row_profile(&ink, w, h, k as u32); + scores[k] = profile.windows(2).map(|p| (p[1] as f64 - p[0] as f64).powi(2)).sum(); + asyms[k] = band_bottom_heaviness(&profile); + } + // 0 and 180 share one profile shape (reversed), as do 90 and 270: pick the axis first. + let (vertical_pair, horizontal_pair) = ((scores[0], scores[2]), (scores[1], scores[3])); + let row_axis = vertical_pair.0.max(vertical_pair.1); + let column_axis = horizontal_pair.0.max(horizontal_pair.1); + if row_axis >= column_axis { + if row_axis < column_axis * 1.3 { + return 0; // the axes are too close to call + } + if asyms[0] > 0.15 * total as f64 { + 0 + } else if asyms[2] > 0.15 * total as f64 { + 2 + } else { + 0 // the bands are not clearly bottom-heavy either way: no-op + } + } else { + if column_axis < row_axis * 1.3 { + return 0; + } + if asyms[1] > 0.15 * total as f64 { + 1 + } else if asyms[3] > 0.15 * total as f64 { + 3 + } else { + 0 + } + } +} + +/// The row profile of the ink grid after `k` quarter turns clockwise. +fn rotated_row_profile(ink: &[bool], w: usize, h: usize, k: u32) -> Vec { + // The rotated image is `w` pixels tall for odd k and `h` for even k; its row index is + // always the new y of the source pixel. + let len = if k.is_multiple_of(2) { h } else { w }; + let mut profile = vec![0u64; len]; + for (y, row) in ink.chunks_exact(w).enumerate() { + for (x, on) in row.iter().enumerate() { + if !on { + continue; + } + // Where does (x, y) land after k quarter turns clockwise? + let ny = match k { + 1 => x, + 2 => h - 1 - y, + 3 => w - 1 - x, + _ => y, + }; + if let Some(p) = profile.get_mut(ny) { + *p += 1; + } + } + } + profile +} + +/// Sum over text-line bands of (ink in the band's lower half − ink in its upper half): +/// positive when the bands carry more ink near their bottoms, as real text does. +fn band_bottom_heaviness(profile: &[u64]) -> f64 { + let mut asym = 0.0; + let mut start: Option = None; + for (i, &p) in profile.iter().chain(std::iter::once(&0)).enumerate() { + match (start, p > 0) { + (None, true) => start = Some(i), + (Some(s), false) => { + let mid = (s + i) / 2; + let lower: u64 = profile.get(s..i).map_or(0, |b| b[mid.saturating_sub(s)..].iter().sum()); + let upper: u64 = profile.get(s..i).map_or(0, |b| b[..mid.saturating_sub(s)].iter().sum()); + asym += lower as f64 - upper as f64; // signed: a top-heavy band scores below zero + start = None; + } + _ => {} + } + } + asym +} + +/// Rotate the raster `k` quarter turns clockwise (an exact pixel permutation). +fn rotate_quarter_turns(image: &OcrImage, k: QuarterTurns) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + let (nw, nh) = if k.is_multiple_of(2) { (image.width, image.height) } else { (image.height, image.width) }; + let mut out = vec![255u8; nw as usize * nh as usize * 4]; + for (y, row) in image.rgba.chunks_exact(w * 4).enumerate() { + for (x, px) in row.as_chunks::<4>().0.iter().enumerate() { + let (nx, ny) = match k { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + 3 => (y, w - 1 - x), + _ => (x, y), + }; + if let Some(dst) = out.get_mut((ny * nw as usize + nx) * 4..(ny * nw as usize + nx) * 4 + 4) { + dst.copy_from_slice(px); + } + } + } + OcrImage::new(nw, nh, out).unwrap_or_else(|_| image.clone()) +} + +/// The inverse of a `k`-quarter-turn correction: where each corrected pixel came from. +fn inverse_quarter_turns(k: QuarterTurns, old_w: u32, old_h: u32) -> InverseTransform { + let (w, h) = (f64::from(old_w), f64::from(old_h)); + match k { + // Forward (content moves): (x, y) ↦ (h−1−y, x) for one turn clockwise. + 1 => InverseTransform { a: 0.0, b: -1.0, c: 1.0, d: 0.0, e: 0.0, f: h - 1.0 }, + 2 => InverseTransform { a: -1.0, b: 0.0, c: 0.0, d: -1.0, e: w - 1.0, f: h - 1.0 }, + 3 => InverseTransform { a: 0.0, b: 1.0, c: -1.0, d: 0.0, e: w - 1.0, f: 0.0 }, + _ => InverseTransform::identity(), + } +} + +/// Estimate the skew of the text lines, in degrees (−5…5, positive: lines run downhill to the +/// right). Method (documented, panic-free): for each candidate angle the ink is sheared into a +/// row profile (`r = y − (x − cx)·tan φ`); the projection-profile score (Σ profile²) peaks +/// where the shear realigns the lines. `None` — leave the page alone — when the image is under +/// 64 px on a side, blank, the winner is under 0.1°, or the winner does not clearly beat +/// leaving the page as is (score < 1.02 × the 0° score). +fn estimate_skew(image: &OcrImage) -> Option { + if image.min_side() < 64 { + return None; + } + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return None }; + let t = (*lo as u32 + *hi as u32) / 2; + let ink: Vec = gray.iter().map(|&g| (g as u32) < t).collect(); + if !ink.iter().any(|i| *i) { + return None; // a blank page has no skew + } + let (w, h) = (image.width, image.height); + let cx = f64::from(w) / 2.0; + let max_shift = (f64::from(w) / 2.0 * 5f64.to_radians().tan()).ceil() as i64 + 2; + let score = |phi_deg: f64| -> f64 { + let slope = phi_deg.to_radians().tan(); + let mut profile = vec![0u64; (h as i64 + 2 * max_shift + 1) as usize]; + for (y, row) in ink.chunks_exact(w as usize).enumerate() { + for (x, on) in row.iter().enumerate() { + if !on { + continue; + } + let r = y as f64 - (x as f64 - cx) * slope; + let idx = r.round() as i64 + max_shift; + if let Some(p) = profile.get_mut(idx as usize) { + *p += 1; + } + } + } + profile.iter().map(|p| (*p as f64).powi(2)).sum() + }; + let mut best = (0.0f64, score(0.0)); + let mut phi = -5.0; + while phi <= 5.0 + 1e-9 { + let s = score(phi); + if s > best.1 { + best = (phi, s); + } + phi += 0.25; + } + let (angle, best_score) = best; + if angle.abs() < 0.1 || best_score < 1.02 * score(0.0) { + return None; // straight enough, or no clear winner: no-op + } + Some(angle) +} + +/// Rotate the content by `−angle` about the centre (the deskew correction), nearest-neighbour +/// with white fill; also returns the exact dest→src map (the correction's inverse). +fn deskew_rotate(image: &OcrImage, angle_deg: f64) -> (OcrImage, InverseTransform) { + let phi = angle_deg.to_radians(); + let (cos, sin) = (phi.cos(), phi.sin()); + let (w, h) = (image.width, image.height); + let (cx, cy) = (f64::from(w) / 2.0, f64::from(h) / 2.0); + let back = InverseTransform { a: cos, b: sin, c: -sin, d: cos, e: cx - cos * cx + sin * cy, f: cy - sin * cx - cos * cy }; + let mut out = vec![255u8; w as usize * h as usize * 4]; + for (y, row) in out.chunks_exact_mut(w as usize * 4).enumerate() { + for (x, px) in row.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let (sx, sy) = back.apply(x as f64, y as f64); + let (xi, yi) = (sx.round() as i64, sy.round() as i64); + if xi < 0 || yi < 0 || xi >= w as i64 || yi >= h as i64 { + continue; // outside the scan: white fill + } + if let Some(src) = image.rgba.get((yi as usize * w as usize + xi as usize) * 4..(yi as usize * w as usize + xi as usize) * 4 + 4) { + px.copy_from_slice(src); + } + } + } + (OcrImage::new(w, h, out).unwrap_or_else(|_| image.clone()), back) +} + +/// 3×3 median filter on each colour channel, edges clamped (border pixels replicate); +/// dimensions and alpha are unchanged. +fn median3(image: &OcrImage) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + let mut out = image.rgba.clone(); + for (i, dst) in out.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let (x, y) = (i % w, i / w); + let mut rgba = [255u8; 3]; + for (ch, v) in rgba.iter_mut().enumerate() { + let mut window = [0u8; 9]; + for (n, (dx, dy)) in [(-1i64, -1i64), (0, -1), (1, -1), (-1, 0), (0, 0), (1, 0), (-1, 1), (0, 1), (1, 1)].iter().enumerate() { + let nx = ((x as i64) + dx).clamp(0, w as i64 - 1) as usize; + let ny = ((y as i64) + dy).clamp(0, h as i64 - 1) as usize; + window[n] = image.byte(nx as f64, ny as f64, ch); + } + window.sort_unstable(); + *v = window[4]; + } + let alpha = dst[3]; + dst.copy_from_slice(&[rgba[0], rgba[1], rgba[2], alpha]); + } + OcrImage::new(image.width, image.height, out).unwrap_or_else(|_| image.clone()) +} + +/// The Sauvola ink mask: local thresholding with the documented window — 8 pixels wide, the +/// 8 pixels from −4 to +3 (the end is exclusive, so `x0..x1` is exactly `x−4..=x+3`), k 0.34 — +/// through an integral image. A pixel is ink when its grey is at or under the local +/// threshold. +fn sauvola_mask(image: &OcrImage) -> Vec { + let (w, h) = (image.width as usize, image.height as usize); + let gray = grayscale(image); + // Integral sums and square sums, (w+1)×(h+1), so any window sum is four lookups. + let stride = w + 1; + let mut sums = vec![0u64; stride * (h + 1)]; + let mut squares = vec![0u64; stride * (h + 1)]; + for (y, row) in gray.chunks_exact(w).enumerate() { + let (mut row_sum, mut row_sq) = (0u64, 0u64); + for (x, &g) in row.iter().enumerate() { + row_sum += u64::from(g); + row_sq += u64::from(g) * u64::from(g); + let above = sums.get(y * stride + x + 1).copied().unwrap_or(0); + let above_sq = squares.get(y * stride + x + 1).copied().unwrap_or(0); + if let (Some(s), Some(sq)) = (sums.get_mut((y + 1) * stride + x + 1), squares.get_mut((y + 1) * stride + x + 1)) { + *s = above + row_sum; + *sq = above_sq + row_sq; + } + } + } + let (k, r) = (0.34f64, 128.0f64); + let half = 4usize; + let mut ink = vec![false; w * h]; + for (y, row) in ink.chunks_exact_mut(w).enumerate() { + for (x, dst) in row.iter_mut().enumerate() { + let (x0, y0) = (x.saturating_sub(half), y.saturating_sub(half)); + // Exclusive end: the window covers −4..=+3, 8 pixels on a side (an end of + // `x + half + 1` would make it 9 and disagree with the documented window). + let (x1, y1) = ((x + half).min(w), (y + half).min(h)); + let n = ((x1 - x0) * (y1 - y0)) as f64; + let sum = rect_sum(&sums, stride, x0, y0, x1, y1); + let sq = rect_sum(&squares, stride, x0, y0, x1, y1); + let mean = sum / n; + let var = (sq / n - mean * mean).max(0.0); + let threshold = mean * (1.0 + k * (var.sqrt() / r - 1.0)); + *dst = matches!(gray.get(y * w + x), Some(&g) if f64::from(g) <= threshold); + } + } + ink +} + +/// Sauvola binarization through [`sauvola_mask`], followed by a one-pixel dilation of the ink: +/// ink goes to black, paper to white, alpha preserved. +fn sauvola(image: &OcrImage) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + // One dilation keeps anti-aliased strokes whole: a thresholded edge pixel that flaked off + // rejoins its stroke (slightly bold beats broken, for reading). + let ink = dilate1(&sauvola_mask(image), w, h); + let mut out = image.rgba.clone(); + for (i, px) in out.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let v = if ink.get(i).copied().unwrap_or(false) { 0u8 } else { 255u8 }; + if let Some(dst) = px.get_mut(..3) { + dst.copy_from_slice(&[v, v, v]); + } + } + OcrImage::new(image.width, image.height, out).unwrap_or_else(|_| image.clone()) +} + +/// Grow the ink mask by one pixel (8-neighbourhood, edges clamped by the bounds checks). +fn dilate1(ink: &[bool], w: usize, h: usize) -> Vec { + let mut out = ink.to_vec(); + for y in 0..h { + for x in 0..w { + if !ink.get(y * w + x).copied().unwrap_or(false) { + continue; + } + for dy in [-1isize, 0, 1] { + for dx in [-1isize, 0, 1] { + let (nx, ny) = (x as isize + dx, y as isize + dy); + if nx < 0 || ny < 0 || nx >= w as isize || ny >= h as isize { + continue; + } + if let Some(o) = out.get_mut(ny as usize * w + nx as usize) { + *o = true; + } + } + } + } + } + out +} + +fn rect_sum(table: &[u64], stride: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> f64 { + let a = table.get(y1 * stride + x1).copied().unwrap_or(0); + let b = table.get(y0 * stride + x1).copied().unwrap_or(0); + let c = table.get(y1 * stride + x0).copied().unwrap_or(0); + let d = table.get(y0 * stride + x0).copied().unwrap_or(0); + (a + d - b - c) as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Sides under MAX_SIDE whose area passes MAX_PIXELS (9500 × 9500, 90 MP) are refused + /// when the size is worked out, before the 360 MB buffer is allocated rather than after. + #[test] + fn an_upscale_over_the_area_cap_is_refused_before_allocating() { + assert!(matches!(scaled_size(1000, 1000, 9.5), Err(OcrError::ImageTooLarge))); + assert_eq!(scaled_size(1000, 1000, 2.0).unwrap(), (2000, 2000), "under the caps it still scales"); + assert_eq!(scaled_size(8000, 8000, 1.0).unwrap(), (8000, 8000), "exactly 64 MP is allowed"); + assert!(matches!(scaled_size(8001, 8000, 1.0), Err(OcrError::ImageTooLarge)), "one row over is not"); + assert!(matches!(scaled_size(1000, 1000, 10.5), Err(OcrError::ImageTooLarge)), "a side over MAX_SIDE"); + assert!(matches!(scaled_size(1000, 1000, 0.0), Err(OcrError::ImageTooLarge)), "a side under one pixel"); + assert!(matches!(scaled_size(1000, 1000, f64::NAN), Err(OcrError::ImageTooLarge))); + let small = OcrImage::new(100, 100, vec![255; 100 * 100 * 4]).unwrap(); + assert!(matches!(resize_bilinear(&small, 100.5), Err(OcrError::ImageTooLarge))); + assert_eq!(resize_bilinear(&small, 2.0).unwrap().width, 200); + } + + fn white(w: u32, h: u32) -> OcrImage { + OcrImage::new(w, h, vec![255; (w * h * 4) as usize]).unwrap() + } + + /// Ink bounding box of a raster (None when blank). + fn ink_bbox(image: &OcrImage) -> Option<[f32; 4]> { + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return None }; + let t = (*lo as u32 + *hi as u32) / 2; + let (mut l, mut tp, mut r, mut b) = (u32::MAX, u32::MAX, 0u32, 0u32); + let mut any = false; + for (i, &g) in gray.iter().enumerate() { + if (g as u32) >= t { + continue; + } + any = true; + let (x, y) = ((i as u32) % image.width, (i as u32) / image.width); + l = l.min(x); + r = r.max(x); + tp = tp.min(y); + b = b.max(y); + } + any.then_some([l as f32, tp as f32, r as f32, b as f32]) + } + + /// A horizontal line of three bottom-heavy block glyphs (L shapes) on white — the fixture + /// for orientation detection (240 × 64 at 300 dpi). + fn l_line() -> OcrImage { + let (w, h) = (240u32, 64u32); + let mut img = white(w, h); + for gx in [20u32, 100, 180] { + fill(&mut img, gx, 12, 12, 40); + fill(&mut img, gx, 40, 36, 12); + } + img + } + + fn fill(img: &mut OcrImage, x0: u32, y0: u32, w: u32, h: u32) { + for y in y0..y0 + h { + for x in x0..x0 + w { + let o = ((y * img.width + x) * 4) as usize; + if let Some(p) = img.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[0, 0, 0]); + } + } + } + } + + /// The same fixture pre-rotated by k quarter turns clockwise, built independently of the + /// crate's rotation (so the test cannot pass by a shared bug). + fn rotated(k: u32) -> OcrImage { + let src = l_line(); + let (w, h) = (src.width as usize, src.height as usize); + let (nw, nh) = if k.is_multiple_of(2) { (src.width, src.height) } else { (src.height, src.width) }; + let mut out = white(nw, nh); + for (y, row) in src.rgba.chunks_exact(w * 4).enumerate() { + for (x, px) in row.as_chunks::<4>().0.iter().enumerate() { + if px[0] != 0 { + continue; + } + let (nx, ny) = match k { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + 3 => (y, w - 1 - x), + _ => (x, y), + }; + fill(&mut out, nx as u32, ny as u32, 1, 1); + } + } + out + } + + /// A ruled fixture sheared by `degrees` (lines running downhill to the right for positive + /// angles): four 8-px bars across a 300 × 120 page. + fn sheared(degrees: f64) -> OcrImage { + let (w, h) = (300u32, 120u32); + let mut img = white(w, h); + let slope = degrees.to_radians().tan(); + for (base, height) in [(20u32, 8u32), (45, 8), (70, 8), (95, 8)] { + for x in 20..280u32 { + let y = (base as f64 + (x as f64 - 150.0) * slope).round().clamp(0.0, f64::from(h - height - 1)) as u32; + fill(&mut img, x, y, 1, height); + } + } + img + } + + /// With the four opt-in flags off and a dpi at or above 300, the raster passes through + /// untouched and the inverse transform is exact. + #[test] + fn flags_off_and_adequate_dpi_is_an_identity_pass_through() { + let src = l_line(); + let p = preprocess(src.clone(), 300.0, &PreprocessOptions::default()).unwrap(); + assert_eq!(p.image.rgba, src.rgba); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + assert_eq!(p.inverse_transform.map_rect([3.0, 7.0, 41.0, 19.0]), [3.0, 7.0, 41.0, 19.0]); + } + + /// With a step on, a scan below 300 dpi is enlarged by 300/src first, and its boxes map + /// back to the original raster. + #[test] + fn low_dpi_scans_are_upscaled_for_the_steps_and_boxes_map_back() { + let src = white(100, 50); + let on = PreprocessOptions { denoise: true, ..Default::default() }; + let p = preprocess(src.clone(), 150.0, &on).unwrap(); + assert_eq!((p.image.width, p.image.height), (200, 100)); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.inverse_transform.map_rect([0.0, 0.0, 200.0, 100.0]), [0.0, 0.0, 100.0, 50.0]); + assert_eq!(p.inverse_transform.apply(2.0, 4.0), (1.0, 2.0)); + // 300 and 600 dpi are not resampled; a broken dpi is left alone. + assert_eq!(preprocess(src.clone(), 600.0, &on).unwrap().image.width, 100); + assert_eq!(preprocess(src, f32::NAN, &on).unwrap().image.width, 100); + } + + /// With every step off a low-dpi render is read exactly as rendered: not resampled, with + /// the identity as its way back. (Reading a 72 dpi scan through a 4x enlargement changed + /// what the models read: "due: \€250" came back as "ue: ?250".) + #[test] + fn with_every_step_off_a_low_dpi_render_is_not_resampled() { + let src = l_line(); + let p = preprocess(src.clone(), 72.0, &PreprocessOptions::default()).unwrap(); + assert_eq!(p.image.rgba, src.rgba); + assert_eq!((p.image.width, p.image.height), (src.width, src.height)); + assert_eq!(p.dpi, 72.0, "the render's own resolution"); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The upscale refuses to break the side cap. + #[test] + fn upscaling_respects_the_side_cap() { + let src = white(9999, 10); + let r = preprocess(src, 72.0, &PreprocessOptions { denoise: true, ..Default::default() }); + assert!(matches!(r, Err(OcrError::ImageTooLarge)), "upscale past the cap: {r:?}"); + } + + /// An upright page is left alone by the orientation detector. + #[test] + fn upright_pages_are_not_rotated() { + let p = preprocess(l_line(), 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Pages scanned at 90/180/270 are detected, turned upright, and their word boxes map back + /// onto the scanned raster's ink. + #[test] + fn quarter_turns_are_detected_corrected_and_mapped_back() { + for k in 1..=3 { + let scanned = rotated(k); + let p = preprocess(scanned.clone(), 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + let corrected = ink_bbox(&p.image).unwrap_or_else(|| panic!("k={k}: the corrected page still has ink")); + // The corrected reading is a horizontal line again: wider than tall. + assert!(corrected[2] - corrected[0] > corrected[3] - corrected[1], "k={k}: {corrected:?}"); + // And the box maps back onto the scanned raster's own ink box. + let back = p.inverse_transform.map_rect(corrected); + let original = ink_bbox(&scanned).unwrap(); + for i in 0..4 { + assert!((back[i] - original[i]).abs() <= 2.0, "k={k}: {back:?} vs {original:?}"); + } + } + } + + /// Images under 32 px on a side are never judged; the step is a no-op. + #[test] + fn tiny_images_skip_orientation() { + let src = white(31, 31); + let p = preprocess(src, 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Blank pages are inconclusive: every step leaves them alone. + #[test] + fn blank_pages_are_a_noop() { + let src = white(200, 200); + let p = preprocess(src, 300.0, &PreprocessOptions { auto_rotate: true, deskew: true, denoise: true, binarize: true }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The skew estimator sees a 3° shear (either direction) and leaves a straight page alone. + #[test] + fn skew_is_estimated_and_straight_pages_are_left_alone() { + for degrees in [3.0, -3.0] { + let angle = estimate_skew(&sheared(degrees)).unwrap_or_else(|| panic!("{degrees}° should be found")); + assert!((angle - degrees).abs() <= 0.5, "{degrees}°: got {angle}"); + } + assert!(estimate_skew(&sheared(0.0)).is_none(), "a straight page is a no-op"); + assert!(estimate_skew(&l_line()).is_none(), "glyph lines without a shear are a no-op"); + } + + /// Deskew straightens ±3° lines (the vertical span tightens) and the centre maps back to + /// itself. + #[test] + fn deskew_corrects_plus_and_minus_three_degrees() { + for degrees in [3.0, -3.0] { + let src = sheared(degrees); + let before = ink_bbox(&src).unwrap(); + let p = preprocess(src.clone(), 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + let after = ink_bbox(&p.image).unwrap(); + assert!(after[3] - after[1] < before[3] - before[1], "{degrees}°: {before:?} → {after:?}"); + // The exact inverse maps the corrected centre back to the scanned centre. + let (cx, cy) = (f64::from(src.width) / 2.0, f64::from(src.height) / 2.0); + let (bx, by) = p.inverse_transform.apply(cx, cy); + assert!((bx - cx).abs() < 1.0 && (by - cy).abs() < 1.0, "{degrees}°: ({bx}, {by})"); + } + } + + /// Deskew no-ops on small images (under 64 px) — the detector is never consulted. + #[test] + fn small_images_skip_deskew() { + let src = white(63, 63); + let p = preprocess(src, 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The median filter removes isolated specks, keeps solid strokes, and never changes the + /// dimensions. + #[test] + fn denoise_removes_specks_and_keeps_dimensions() { + let mut src = white(32, 32); + fill(&mut src, 15, 15, 1, 1); // salt + fill(&mut src, 4, 4, 5, 5); // a solid stroke + let p = preprocess(src, 300.0, &PreprocessOptions { denoise: true, ..Default::default() }).unwrap(); + assert_eq!((p.image.width, p.image.height), (32, 32)); + assert_eq!(p.image.byte(15.0, 15.0, 0), 255, "an isolated speck is gone"); + assert_eq!(p.image.byte(6.0, 6.0, 0), 0, "the inside of a solid block stays"); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Sauvola keeps the polarity (dark ink stays dark, paper stays light) and the metadata. + /// The fixture is a thin grey stroke: its windows mix ink and paper, so the local + /// threshold separates them (a perfectly flat mid-tone area has no contrast and whitens, + /// which is Sauvola working as designed). + #[test] + fn binarize_keeps_polarity_and_metadata() { + let (w, h) = (64u32, 64u32); + let mut src = white(w, h); + for y in 26..30 { + for x in 8..56 { + let o = ((y * w + x) * 4) as usize; + if let Some(p) = src.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[70, 70, 70]); // a grey scan's ink, far below paper + } + } + } + let p = preprocess(src, 300.0, &PreprocessOptions { binarize: true, ..Default::default() }).unwrap(); + assert_eq!((p.image.width, p.image.height), (w, h)); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.image.byte(30.0, 28.0, 0), 0, "ink stays dark"); + assert_eq!(p.image.byte(2.0, 2.0, 0), 255, "paper stays light"); + for px in p.image.rgba.as_chunks::<4>().0.iter() { + assert!((px[0] == 0 || px[0] == 255) && px[0] == px[1] && px[1] == px[2], "{px:?}"); + } + } + + /// The inverse transform composes: an upscaled, turned, deskewed page maps corrected + /// points back inside the original raster. + #[test] + fn inverse_transform_maps_points_back_through_every_step() { + let scanned = rotated(1); // a page photographed on its side + let p = preprocess(scanned.clone(), 150.0, &PreprocessOptions { auto_rotate: true, deskew: true, ..Default::default() }).unwrap(); + let (w, h) = (f64::from(p.image.width), f64::from(p.image.height)); + for (x, y) in [(0.0, 0.0), (w - 1.0, 0.0), (0.0, h - 1.0), (w - 1.0, h - 1.0), (w / 2.0, h / 2.0)] { + let (bx, by) = p.inverse_transform.apply(x, y); + assert!((-2.0..=f64::from(scanned.width) + 2.0).contains(&bx), "x {x} → {bx}"); + assert!((-2.0..=f64::from(scanned.height) + 2.0).contains(&by), "y {y} → {by}"); + } + // The scale step, alone, is exact. + assert_eq!(InverseTransform::scale(0.5).apply(4.0, 10.0), (2.0, 5.0)); + } + + /// A realistic letter page (one line of type on white, at 300 dpi) reads as upright: the + /// row axis bands decisively sharper than the column axis, and the near-balanced + /// bottom-heaviness of real mixed-case text is below the 15% flip bar, so the page is left + /// alone rather than risk flipping a good scan upside down. + #[test] + fn realistic_letter_pages_are_left_upright() { + let (w, h) = (1200u32, 1650u32); + let mut img = white(w, h); + let mut x = 120u32; + let base = 300u32; + let mut seed = 7u64; + let mut rnd = || { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (seed >> 33) as u32 + }; + while x < 1000 { + let word = rnd() % 5 + 2; + for _ in 0..word { + let kind = rnd() % 4; + let wl = rnd() % 8 + 3; + match kind { + 0 => fill(&mut img, x, base - 20, wl, 20), // x-height + 1 => fill(&mut img, x, base - 34, wl, 34), // ascender + 2 => fill(&mut img, x, base - 20, wl, 28), // descender + _ => { + fill(&mut img, x, base - 20, 2, 20); + fill(&mut img, x + 3, base - 14, 2, 14); // stems + } + } + x += wl + 2; + } + x += 12; // space + } + assert_eq!(detect_orientation(&img), 0); + let p = preprocess(img, 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Hostile inputs are errors or no-ops, never panics: 1×1 rasters, all-black pages, NaN + /// boxes through the inverse map. + #[test] + fn hostile_inputs_never_panic() { + let tiny = white(1, 1); + let all = PreprocessOptions { auto_rotate: true, deskew: true, denoise: true, binarize: true }; + let p = preprocess(tiny, 300.0, &all).unwrap(); + assert_eq!((p.image.width, p.image.height), (1, 1)); + // All-black: no paper, nothing to judge. + let mut black = white(80, 80); + for px in black.rgba.as_chunks_mut::<4>().0 { + px.copy_from_slice(&[0, 0, 0, 255]); + } + let p = preprocess(black, 300.0, &all).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity(), "all-black is inconclusive"); + // NaN boxes never poison downstream math: the finite corners bound the mapped box, + // and a box with no finite corners comes back unchanged. + let mapped = p.inverse_transform.map_rect([f32::NAN, 0.0, 5.0, 5.0]); + assert!(mapped.iter().all(|v| v.is_finite()), "{mapped:?}"); + let unchanged = InverseTransform::identity().map_rect([f32::NAN; 4]); + assert!(unchanged.iter().all(|v| v.is_nan()), "{unchanged:?}"); + } + + /// The one-pixel dilation is load-bearing, pinned with a fixture that fragments without + /// it: a thin anti-aliased-looking stroke on a grey scan (alternating dark and light + /// pixels) — the raw Sauvola mask drops the light runs (the local window sits on a grey + /// page, so the threshold falls below them), while the binarized image keeps the stroke + /// whole. + #[test] + fn binarize_without_the_dilation_the_stroke_fragments_and_with_it_stays_whole() { + let (w, h) = (64u32, 64u32); + let mut src = white(w, h); + let set = |img: &mut OcrImage, x: u32, y: u32, v: u8| { + let o = ((y * w + x) * 4) as usize; + if let Some(p) = img.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[v, v, v]); + } + }; + for px in src.rgba.as_chunks_mut::<4>().0.iter_mut() { + px[..3].copy_from_slice(&[210, 210, 210]); // a grey scan's paper + } + // The stroke: one pixel tall, alternating dark (reads as ink) and light (an + // anti-aliased shoulder the threshold alone drops) in runs of two. + for x in 8..56u32 { + let v = if (x / 2) % 2 == 0 { 110 } else { 170 }; + set(&mut src, x, 32, v); + } + let mask = sauvola_mask(&src); + let at = |x: u32, y: u32| mask[y as usize * w as usize + x as usize]; + let dropped: Vec = (12..52).filter(|&x| !at(x, 32)).collect(); + assert!(!dropped.is_empty(), "without the dilation the stroke fragments: these stroke pixels fall under the threshold at {dropped:?}"); + let p = preprocess(src, 300.0, &PreprocessOptions { binarize: true, ..Default::default() }).unwrap(); + // With the dilation the stroke is whole: every position on it has ink within its + // 8-neighbourhood. + for x in 12..=51u32 { + let ink_near = (-1i32..=1).any(|dy| { + (-1i32..=1).any(|dx| { + let (nx, ny) = (x as i32 + dx, 32 + dy); + p.image.byte(f64::from(nx), f64::from(ny), 0) == 0 + }) + }); + assert!(ink_near, "the stroke is broken at x={x}"); + } + } + + /// The deskew inverse, checked at OFF-CENTRE points (the centre maps to itself under any + /// rotation, so it cannot catch a wrong sign): a point on a straightened line maps back + /// onto the sheared source line, uphill and downhill. + #[test] + fn deskew_inverse_maps_off_centre_points_back_onto_the_source_line() { + for degrees in [3.0, -3.0] { + let src = sheared(degrees); + let p = preprocess(src, 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + let slope = degrees.to_radians().tan(); + // The corrected first line sits at y = 20 (the fixture's base); the same ink was + // at 20 + (x − 150)·slope in the source. Well off centre on purpose. The + // rotation moves x a little too (it mixes in y: at most sin(5°)·40 ≈ 3.5 px + // here), so the line is checked at the mapped x — a wrong sign still misses it + // by ±(x−150)·slope, 6+ px at these offsets. + for x in [30.0f64, 150.0, 270.0] { + let (sx, sy) = p.inverse_transform.apply(x, 20.0); + assert!((sx - x).abs() < 4.0, "{degrees}° x={x}: mapped to ({sx}, {sy})"); + let expected = 20.0 + (sx - 150.0) * slope; + assert!((sy - expected).abs() < 2.0, "{degrees}° x={x}: ({sx}, {sy}), the source line is at {expected}"); + } + } + } +} diff --git a/crates/ocr/src/tesseract.rs b/crates/ocr/src/tesseract.rs new file mode 100644 index 0000000000..9fbcc574d8 --- /dev/null +++ b/crates/ocr/src/tesseract.rs @@ -0,0 +1,1022 @@ +//! The Tesseract engine: an installed `tesseract` binary driven as an optional external +//! process (`tesseract stdout -l tsv`), never linked, never bundled, never +//! downloaded. The whole module is compiled out for wasm32, where nothing may spawn. +//! +//! Security posture (hard rules): the program path comes from the PATH or a user-configured +//! location, never from document data; the language code is allow-listed before it reaches a +//! process; the page image goes to a private temp file (a fresh 0700 directory holding a 0600 +//! file on Unix) that is deleted afterwards; the run has a timeout and an output size cap; and +//! every failure comes back through [`OcrError`] with its cause named. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::{Duration, Instant}; + +use super::{Line, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// How long one recognition may run before the process is killed. +const TIMEOUT: Duration = Duration::from_secs(120); +/// How much TSV output is accepted before the run is abandoned. +const MAX_OUTPUT: u64 = 64 * 1024 * 1024; +/// How much of the program's stderr is read back (its end): enough for a `--list-langs` list +/// printed there, small enough that error text stays a message, not a memory DoS. +const MAX_STDERR: u64 = 256 * 1024; +/// The most words one TSV reading may hold. Known limit: a reading with more words than this +/// is truncated at the cap — corrupt or runaway output must not become unbounded memory in +/// the word list, the text layer and the review UI. +pub const MAX_WORDS: usize = 50_000; +/// Tesseract's code for each language PdfCraft offers (ISO 639-1 → tessdata name). Only these +/// codes ever reach the process, whatever the settings say. +pub const LANGUAGE_CODES: &[(&str, &str)] = &[ + ("en", "eng"), + ("de", "deu"), + ("fr", "fra"), + ("es", "spa"), + ("it", "ita"), + ("pt", "por"), + ("ru", "rus"), + ("zh", "chi_sim"), + ("ja", "jpn"), + ("ko", "kor"), + ("ar", "ara"), + ("nl", "nld"), +]; + +/// English names for [`LANGUAGE_CODES`] (dropdowns show code + name). +pub const LANGUAGE_NAMES: &[(&str, &str)] = &[ + ("en", "English"), + ("de", "German"), + ("fr", "French"), + ("es", "Spanish"), + ("it", "Italian"), + ("pt", "Portuguese"), + ("ru", "Russian"), + ("zh", "Chinese (Simplified)"), + ("ja", "Japanese"), + ("ko", "Korean"), + ("ar", "Arabic"), + ("nl", "Dutch"), +]; + +/// Map a settings language code to a Tesseract one: trimmed, case-insensitive, allow-listed; +/// anything unknown falls back to `"eng"`. +pub fn language_code(code: &str) -> &'static str { + let c = code.trim().to_ascii_lowercase(); + LANGUAGE_CODES.iter().find(|(k, _)| *k == c).map(|(_, v)| *v).unwrap_or("eng") +} + +/// The user-configured program location, when valid: `$PDFCRAFT_TESSERACT` (a user +/// preference, never taken from document data). An invalid value — relative, missing, a +/// directory, not executable — means unavailable: an explicit setting is never silently +/// replaced by a PATH search. +pub fn configured_path() -> Option { + let value = std::env::var_os("PDFCRAFT_TESSERACT")?; + let program = PathBuf::from(value); + program_is_executable(&program).then_some(program) +} + +/// The program [`TesseractCli::find`] would run: a set `configured` location (the value of +/// `PDFCRAFT_TESSERACT`) decides alone, so an invalid one yields `None` and `search` (the PATH) +/// is not even consulted; only when nothing is configured does `search` decide. +fn locate(configured: Option, search: impl FnOnce() -> Option) -> Option { + match configured { + Some(value) => Some(PathBuf::from(value)).filter(|program| program_is_executable(program)), + None => search(), + } +} + +/// The first `tesseract` on the PATH that is an existing executable file. Entries that are not +/// absolute are refused: a relative PATH entry resolves against the caller's working +/// directory, which is neither stable nor a safe place to find a program. Also named by +/// `ocr_status` (the engine reports where it would look, without probing). +pub fn path_search() -> Option { + path_search_in(std::env::split_paths(&std::env::var_os("PATH")?)) +} + +/// [`path_search`] over an explicit directory list (so tests can pass one). +fn path_search_in(dirs: impl Iterator) -> Option { + for dir in dirs.filter(|d| d.is_absolute()) { + for name in program_file_names() { + let candidate = dir.join(name); + if program_is_executable(&candidate) { + return Some(candidate); + } + } + } + None +} + +/// The file names an installed `tesseract` binary goes by. +fn program_file_names() -> &'static [&'static str] { + #[cfg(windows)] + { + &["tesseract.exe"] + } + #[cfg(not(windows))] + { + &["tesseract"] + } +} + +/// Whether `program` is a usable program location (see [`program_is_executable`]); callers +/// that run a validated program by path (the engine's `--version` query) check it first. +pub fn program_is_usable(program: &Path) -> bool { + program_is_executable(program) +} + +/// Whether `program` is a usable configured location: an absolute path to an existing, +/// executable FILE (on Unix the executable bits must be set; elsewhere being a file is what +/// the OS requires). A directory or a missing file is simply unavailable — never a panic. +fn program_is_executable(program: &Path) -> bool { + if !program.is_absolute() || !program.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(program).is_ok_and(|m| m.permissions().mode() & 0o111 != 0) + } + #[cfg(not(unix))] + { + true + } +} + +/// One row of Tesseract's TSV output. `level` 5 rows are the words; `conf` is −1 on the header +/// rows (page, block, paragraph, line), which are not words. +struct TsvRow<'a> { + level: &'a str, + block: u32, + par: u32, + line: u32, + left: f32, + top: f32, + width: f32, + height: f32, + conf: f32, + text: &'a str, +} + +/// Parse one TSV row; `None` for anything unreadable (short rows, non-numeric fields), which +/// is skipped rather than trusted. A row with non-finite numbers or a negative width or height +/// is not a word either: geometry like that is output corruption, and trusting it puts NaN +/// into word boxes and on to the page. +fn tsv_row(row: &str) -> Option> { + let c: Vec<&str> = row.split('\t').collect(); + if c.len() < 12 { + return None; + } + let (block, par, line) = (c[2].trim().parse().ok()?, c[3].trim().parse().ok()?, c[4].trim().parse().ok()?); + let (left, top): (f32, f32) = (c[6].trim().parse().ok()?, c[7].trim().parse().ok()?); + let (width, height, conf): (f32, f32, f32) = (c[8].trim().parse().ok()?, c[9].trim().parse().ok()?, c[10].trim().parse().ok()?); + if !left.is_finite() || !top.is_finite() || !width.is_finite() || !height.is_finite() || !conf.is_finite() { + return None; // NaN/inf anywhere in the numbers + } + if width < 0.0 || height < 0.0 { + return None; // a word box has a non-negative size + } + Some(TsvRow { level: c[0], block, par, line, left, top, width, height, conf, text: c[11] }) +} + +/// Parse `tesseract … tsv` output into lines of words. Rows with confidence −1 are the +/// structural header rows and never become words; consecutive word rows that share their +/// (block, paragraph, line) form one [`Line`], in output (reading) order. Confidence clamps +/// into 0–100. Known limit: past [`MAX_WORDS`] the rest of the reading is dropped (truncated, +/// not an error — the words before the cap are still placed). +pub fn parse_tsv(tsv: &str) -> Vec { + parse_tsv_bounded(tsv).0 +} + +/// The parser utility keeps its bounded-prefix API; recognition also checks completeness. +fn parse_tsv_bounded(tsv: &str) -> (Vec, bool) { + let mut lines: Vec = Vec::new(); + let mut key: Option<(u32, u32, u32)> = None; + let mut words = 0usize; + for row in tsv.lines() { + let Some(r) = tsv_row(row) else { continue }; + if r.level != "5" || r.conf < 0.0 || r.text.trim().is_empty() { + continue; + } + if words >= MAX_WORDS { + return (lines, true); + } + let this = (r.block, r.par, r.line); + if key != Some(this) { + lines.push(Line::default()); + key = Some(this); + } + let word = Word::new(r.text, [r.left, r.top, r.left + r.width, r.top + r.height], "tesseract").with_confidence(r.conf.clamp(0.0, 100.0)); + if let Some(last) = lines.last_mut() { + last.words.push(word); + words += 1; + } + } + (lines, false) +} + +/// Parse `tesseract --list-langs` output: the header line ("List of available languages…"), +/// then one code per line. Lines with spaces are not codes (a header, a warning) and are left +/// out. +pub fn parse_list_langs(out: &str) -> Vec { + out.lines() + .skip_while(|l| !l.trim().is_empty() && l.trim().starts_with("List of available")) + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty() && !l.contains(char::is_whitespace)) + .collect() +} + +/// The languages a `--list-langs` run reported: its stdout, or, when that holds none, its +/// stderr (Tesseract 3.x prints the list there). +pub fn list_langs_from(stdout: &str, stderr: &str) -> Vec { + let codes = parse_list_langs(stdout); + if codes.is_empty() { parse_list_langs(stderr) } else { codes } +} + +/// The first non-empty line of what `tesseract --version` printed: on stdout, or on stderr +/// where older versions print it. +fn version_line(stdout: &str, stderr: &str) -> Option { + let first = |text: &str| text.lines().map(str::trim).find(|line| !line.is_empty()).map(str::to_owned); + first(stdout).or_else(|| first(stderr)) +} + +/// The Tesseract program. Build one (via [`TesseractCli::find`] for the PATH copy, or +/// [`TesseractCli::new`] for a user-configured path) and reuse it: `--list-langs` runs at most +/// once. +pub struct TesseractCli { + program: PathBuf, + /// The timeout and the output cap are fields so tests can shorten them. + timeout: Duration, + max_output: u64, + /// Whether `--list-langs` has succeeded yet: 0 untried, 1 yes, 2 no. + probed: AtomicU8, + langs: Mutex>, +} + +impl TesseractCli { + /// The program at `path` (a user preference; never taken from document data). + pub fn new(program: PathBuf) -> TesseractCli { + TesseractCli { program, timeout: TIMEOUT, max_output: MAX_OUTPUT, probed: AtomicU8::new(0), langs: Mutex::new(Vec::new()) } + } + + /// The program this engine runs (`ocr_status` names it). + pub fn program(&self) -> &std::path::Path { + &self.program + } + + /// The program at a user-configured `path`, if it is usable there: only an absolute path + /// to an existing, executable file is accepted (a directory, a missing file or a relative + /// path is unavailable, never a panic), and it must answer `--list-langs`. + pub fn at(program: PathBuf) -> Option { + if !program_is_executable(&program) { + return None; + } + let t = TesseractCli::new(program); + let langs = t.list_langs().ok()?; + *t.langs.lock().unwrap_or_else(|e| e.into_inner()) = langs; + t.probed.store(1, Ordering::Relaxed); + Some(t) + } + + /// The program this machine offers, if it answers `--list-langs`: the user-configured + /// location ([`configured_path`] / `$PDFCRAFT_TESSERACT`) when valid, else the first + /// absolute PATH entry that holds an executable `tesseract`. + /// + /// # The PATH-lookup risk + /// + /// Resolving through the PATH trusts every directory on it: a binary planted earlier on + /// the PATH runs as a subprocess with this application's privileges — it sees the page + /// image and anything the process can reach. The language code and the page image still + /// never come from document data, but a user who handles untrusted documents should pin + /// the exact program with `PDFCRAFT_TESSERACT`, which is validated (absolute, existing, + /// executable file) before any process starts. + /// + /// A `PDFCRAFT_TESSERACT` that is set but invalid fails closed: it is never replaced by + /// the PATH search. + pub fn find() -> Option { + TesseractCli::at(locate(std::env::var_os("PDFCRAFT_TESSERACT"), path_search)?) + } + + /// Run `tesseract --list-langs` and parse it; also decides [`TesseractCli::available`]. + fn list_langs(&self) -> Result, OcrError> { + let (out, err) = self.run_capture(&["--list-langs"], self.timeout, self.max_output)?; + let codes = list_langs_from(&out, &err); + if codes.is_empty() { + return Err(OcrError::NoTesseract("--list-langs reported no languages".into())); + } + Ok(codes + .into_iter() + .map(|c| { + let name = LANGUAGE_NAMES.iter().find(|(k, _)| *k == c.as_str()).map(|(_, n)| (*n).to_string()).unwrap_or_else(|| c.clone()); + (c, name) + }) + .collect()) + } + + /// The program's first `--version` line (named by `ocr_status`); `None` when the program + /// does not answer. Not cached: status asks rarely, and a re-install mid-session should + /// show up without a restart. + pub fn version(&self) -> Option { + let (out, err) = self.run_capture(&["--version"], self.timeout, self.max_output).ok()?; + version_line(&out, &err) + } + + /// Run the program with fixed argv, its stdout in a private temp file, under `timeout` + /// seconds and an output cap of `max_output` bytes. Returns the (capped) stdout. + fn run(&self, args: &[&str], timeout: Duration, max_output: u64) -> Result { + self.run_capture(args, timeout, max_output).map(|(out, _)| out) + } + + /// [`Self::run`], also returning what the program wrote to stderr (up to + /// [`MAX_STDERR`] bytes, its end kept): some Tesseract versions print the `--list-langs` + /// list there instead of on stdout. + fn run_capture(&self, args: &[&str], timeout: Duration, max_output: u64) -> Result<(String, String), OcrError> { + let dir = TempDir::create()?; + let out_path = dir.path.join("out.txt"); + let err_path = dir.path.join("err.txt"); + let out_file = std::fs::File::create(&out_path).map_err(|e| OcrError::Process(format!("creating the output file failed: {e}")))?; + // stderr goes to a file, not a pipe: an undrained pipe fills (around 64 KiB) and then + // blocks the child mid-write, which a watcher can only read as a timeout. A file never + // blocks; it is read back as a short tail after the run. + let err_file = std::fs::File::create(&err_path).map_err(|e| OcrError::Process(format!("creating the error file failed: {e}")))?; + // Known limit: only the direct child is killed at the timeout or on an early cap. A + // wrapper script's children (grandchildren of this process) may survive the kill; the + // cap and the timeout still bound what this run reads and how long it waits. + let mut child = + std::process::Command::new(&self.program) + .args(args) + .stdin(std::process::Stdio::null()) + .stdout(out_file) + .stderr(err_file) + .current_dir(&dir.path) + // A minimal environment: nothing from the document or the caller's shell beyond + // what an installed tesseract needs — its data prefix, the PATH (bare-name lookup + // and DLL loading), the Windows system root, and a temp dir it may write to. + .env_clear() + .envs(std::env::vars().filter(|(k, _)| { + k.eq_ignore_ascii_case("TESSDATA_PREFIX") || k.eq_ignore_ascii_case("PATH") || k.eq_ignore_ascii_case("SystemRoot") + })) + .env("TMP", &dir.path) + .env("TEMP", &dir.path) + .spawn() + .map_err(|e| OcrError::NoTesseract(format!("{} could not start: {e}", self.program.display())))?; + let started = Instant::now(); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) => { + // The output caps are enforced while the child runs: a flood is killed as + // soon as a file passes its cap, not after it has finished writing. The + // read-back below caps what this run KEEPS of stderr; this cap bounds what + // the child can write there in the first place. + if std::fs::metadata(&out_path).map(|m| m.len()).unwrap_or(0) > max_output + || std::fs::metadata(&err_path).map(|m| m.len()).unwrap_or(0) > MAX_STDERR + { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::OutputTooLarge(max_output)); + } + if started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::Timeout(timeout.as_secs())); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::Process(format!("waiting for {} failed: {e}", self.program.display()))); + } + } + }; + let status = status?; + // Whatever the exit, the stderr tail names the cause (capped: error text stays a + // message, not a memory DoS). + let stderr = std::fs::File::open(&err_path) + .and_then(|mut f| { + use std::io::{Read, Seek, SeekFrom}; + let len = f.metadata()?.len().min(MAX_STDERR); + f.seek(SeekFrom::End(-(len as i64)))?; + let mut buf = Vec::with_capacity(len as usize); + f.take(len).read_to_end(&mut buf)?; + Ok::<_, std::io::Error>(buf) + }) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).into_owned(); + // The message names only the end of it. + let tail = stderr.char_indices().rev().nth(2047).map_or(0, |(i, _)| i); + let note = stderr.get(tail..).unwrap_or_default().trim(); + if !status.success() { + return Err(OcrError::Process(format!( + "{} exited with {}: {}", + self.program.display(), + status.code().map(|c| c.to_string()).unwrap_or_else(|| "a signal".into()), + if note.is_empty() { "no error message" } else { note } + ))); + } + let len = std::fs::metadata(&out_path).map(|m| m.len()).unwrap_or(0); + if len > max_output { + return Err(OcrError::OutputTooLarge(max_output)); + } + let bytes = std::fs::read(&out_path).map_err(|e| OcrError::Process(format!("reading the output failed: {e}")))?; + Ok((String::from_utf8_lossy(&bytes).into_owned(), stderr)) + } + + /// Retry `--list-langs` once (an install may have appeared since the last try). + fn refresh(&self) -> Result<(), OcrError> { + let langs = self.list_langs()?; + *self.langs.lock().unwrap_or_else(|e| e.into_inner()) = langs; + self.probed.store(1, Ordering::Relaxed); + Ok(()) + } +} + +impl Recognizer for TesseractCli { + fn id(&self) -> &'static str { + "tesseract" + } + + /// Whether the program answers `--list-langs` (remembered; one process at most after the + /// first call). + fn available(&self) -> bool { + match self.probed.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => self.refresh().is_ok(), + } + } + + fn languages(&self) -> Vec<(String, String)> { + if self.probed.load(Ordering::Relaxed) == 2 { + return Vec::new(); + } + let langs = self.langs.lock().unwrap_or_else(|e| e.into_inner()); + if langs.is_empty() { + drop(langs); + return self.refresh().map_or(Vec::new(), |()| self.langs.lock().unwrap_or_else(|e| e.into_inner()).clone()); + } + langs.clone() + } + + /// Write the page as a PNG and read it back as TSV. + fn recognize(&self, image: &OcrImage, options: &RecognizeOptions) -> Result, OcrError> { + let lang = language_code(&options.language); + let png = encode_png(image)?; + let dir = TempDir::create()?; + let png_path = dir.path.join("page.png"); + { + let mut f = std::fs::File::create(&png_path).map_err(|e| OcrError::Process(format!("creating the image file failed: {e}")))?; + set_private(&mut f); + f.write_all(&png).map_err(|e| OcrError::Process(format!("writing the image failed: {e}")))?; + } + // Fixed argv, no shell: tesseract stdout -l tsv. + let tsv = self.run(&[&png_path.to_string_lossy(), "stdout", "-l", lang, "tsv"], self.timeout, self.max_output)?; + let (lines, truncated) = parse_tsv_bounded(&tsv); + if truncated { + return Err(OcrError::Recognize(format!("the {MAX_WORDS}-word limit was exceeded; the incomplete reading was rejected"))); + } + Ok(lines) + } +} + +/// Encode an RGBA raster as PNG (the interchange format Tesseract reads best). +fn encode_png(image: &OcrImage) -> Result, OcrError> { + let mut out = Vec::new(); + let mut enc = png::Encoder::new(&mut out, image.width, image.height); + enc.set_color(png::ColorType::Rgba); + enc.set_depth(png::BitDepth::Eight); + let mut writer = enc.write_header().map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + writer.write_image_data(&image.rgba).map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + writer.finish().map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + Ok(out) +} + +/// Make a just-created file owner-only, where the OS supports it (Unix). Windows relies on the +/// user's private %TEMP% ACLs; the fresh directory keeps other sessions' files out of sight. +fn set_private(f: &mut std::fs::File) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = f.set_permissions(std::fs::Permissions::from_mode(0o600)); + } + #[cfg(not(unix))] + let _ = f; +} + +/// A fresh directory under the system temp dir, deleted (with its contents) when dropped. +/// On Unix it is created 0700, so the page image inside is private. +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn create() -> Result { + // The recognizer is shared across threads (`Arc`), so two `recognize` calls can run at + // once, and on a coarse clock they can read the same nanosecond. One counter value per + // attempt keeps the names of one process apart without trusting the clock. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + // Created owner-only in one step on Unix (no window in which the directory exists with + // the default, wider mode); `create` refuses a path that already exists. + let mut builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + // Elsewhere the default mode stands (Windows relies on the user's private %TEMP% ACLs). + #[cfg(not(unix))] + let _ = &mut builder; + // A leftover directory of an earlier process with the same pid can still take a name; + // that one collision is retried with the next counter value, anything else is fatal. + for _ in 0..4 { + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("pdfcraft-ocr-{}-{nanos}-{seq}", std::process::id())); + match builder.create(&path) { + Ok(()) => return Ok(TempDir { path }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(OcrError::Process(format!("creating the private temp directory failed: {e}"))), + } + } + Err(OcrError::Process("creating the private temp directory failed: every candidate name already exists".into())) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + // The Unix process tests below build fake scripts and name their paths. + #[cfg(unix)] + use std::path::Path; + + /// Several directories alive at once never share a name: the recognizer is shared across + /// threads (`Arc`), so two `recognize` calls can create their directories concurrently, + /// and the counter in the name — not the clock alone — tells them apart. + #[test] + fn temp_directories_alive_at_once_have_distinct_names() { + let dirs: Vec = (0..32).map(|_| TempDir::create().expect("create")).collect(); + let distinct: std::collections::BTreeSet = dirs.iter().map(|d| d.path.clone()).collect(); + assert_eq!(distinct.len(), dirs.len(), "{:?}", dirs.iter().map(|d| &d.path).collect::>()); + } + + /// A tiny image with one dark square (a synthetic "word"); the Unix process tests use it. + #[cfg(unix)] + fn blot(w: u32, h: u32, x0: u32, y0: u32, bw: u32, bh: u32) -> OcrImage { + let mut rgba = vec![255u8; (w * h * 4) as usize]; + for y in y0..(y0 + bh).min(h) { + for x in x0..(x0 + bw).min(w) { + let o = ((y * w + x) * 4) as usize; + rgba[o..o + 3].copy_from_slice(&[0, 0, 0]); + } + } + OcrImage::new(w, h, rgba).unwrap() + } + + /// The literal TSV fixture: two header rows (conf −1) and two lines of words. + const TSV: &str = "level\tpage\tblock\tpar\tline\tword\tleft\ttop\twidth\theight\tconf\ttext +1\t1\t0\t0\t0\t0\t0\t0\t100\t50\t-1\t +5\t1\t1\t1\t1\t1\t10\t10\t40\t12\t96.5\tHello +5\t1\t1\t1\t1\t2\t55\t10\t30\t12\t91.0\tworld +5\t1\t1\t2\t2\t1\t10\t30\t20\t12\t42.123\tsecond"; + + #[test] + fn tsv_parses_words_into_lines() { + let lines = parse_tsv(TSV); + assert_eq!(lines.len(), 2, "two (block, par, line) groups: {lines:?}"); + let first = &lines[0].words; + assert_eq!(first.len(), 2); + assert_eq!(first[0].text, "Hello"); + assert_eq!(first[0].rect, [10.0, 10.0, 50.0, 22.0], "left + width, top + height"); + assert_eq!(first[0].confidence, Some(96.5)); + assert_eq!(first[0].source, "tesseract"); + assert_eq!(first[1].text, "world"); + assert_eq!(lines[1].words.len(), 1, "a new par starts a new line"); + assert_eq!(lines[1].words[0].text, "second"); + assert_eq!(lines[1].words[0].confidence, Some(42.123)); + } + + /// Conf −1 rows and level ≠ 5 rows are structure, not words; blank text is dropped. + #[test] + fn tsv_treats_header_rows_and_blank_text_as_not_words() { + let tsv = "5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t-1\tskipped\n\ + 4\t1\t1\t1\t1\t0\t0\t0\t10\t10\t95.0\tline row\n\ + 5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t95.0\t \n\ + 5\t1\t1\t1\t1\t2\t0\t0\t10\t10\t88\tkept"; + let lines = parse_tsv(tsv); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert_eq!(lines[0].words.len(), 1); + assert_eq!(lines[0].words[0].text, "kept"); + } + + /// Garbage rows (short, non-numeric) are skipped; CRLF endings survive; boxes saturate + /// instead of overflowing. + #[test] + fn tsv_skips_garbage_and_never_panics() { + let lines = parse_tsv("not a tsv row\n\n5\t1\t1\n5\t1\t1\t1\t1\t1\tx\ty\tw\th\tzzz\ttext\r\n"); + assert!(lines.is_empty(), "{lines:?}"); + let big = parse_tsv(&format!("5\t1\t1\t1\t1\t1\t{}\t0\t{}\t10\t50\tbig", u32::MAX, u32::MAX)); + let rect = big[0].words[0].rect; + assert!(rect[2].is_finite() && rect[3].is_finite(), "no overflow: {rect:?}"); + // A row whose text field is empty yields no words. + assert!(parse_tsv("5\t1\t1\t1\t1\t1\t1\t1\t2\t2\t50\t\t").is_empty()); + } + + /// Non-finite numbers and negative sizes are not words: a row with NaN/inf coordinates or + /// a negative width/height never reaches the output (and so never reaches the content + /// stream). `1e39` overflows f32 to infinity, so it is refused the same way. + #[test] + fn tsv_rejects_non_finite_and_negative_geometry() { + let row = |left: &str, top: &str, w: &str, h: &str, conf: &str| format!("5\t1\t1\t1\t1\t1\t{left}\t{top}\t{w}\t{h}\t{conf}\tword"); + for (why, r) in [ + ("NaN left", row("NaN", "0", "10", "10", "50")), + ("NaN width", row("0", "0", "NaN", "10", "50")), + ("NaN height", row("0", "0", "10", "NaN", "50")), + ("NaN confidence", row("0", "0", "10", "10", "NaN")), + ("inf left", row("inf", "0", "10", "10", "50")), + ("inf width", row("0", "0", "inf", "10", "50")), + ("1e39 width overflows to inf", row("0", "0", "1e39", "10", "50")), + ("negative width", row("0", "0", "-10", "10", "50")), + ("negative height", row("0", "0", "10", "-10", "50")), + ] { + let lines = parse_tsv(&r); + assert!(lines.is_empty(), "{why}: {lines:?}"); + } + // A sane row still parses. + assert_eq!(parse_tsv(&row("0", "0", "10", "10", "50"))[0].words[0].text, "word"); + } + + /// A reading longer than the word cap is truncated at [`MAX_WORDS`] (a known limit, not an + /// error): the words before the cap are kept, the rest is dropped. + #[test] + fn tsv_truncates_readings_past_the_word_cap() { + let mut tsv = String::new(); + for i in 0..MAX_WORDS + 5 { + tsv.push_str(&format!("5\t1\t1\t1\t1\t{}\t{}\t10\t40\t12\t96.5\tw{i}\n", i % 7 + 1, i % 500)); + } + let lines = parse_tsv(&tsv); + let count: usize = lines.iter().map(|l| l.words.len()).sum(); + assert_eq!(count, MAX_WORDS, "truncated at the cap"); + assert_eq!(lines.last().and_then(|l| l.words.last()).map(|w| w.text.as_str()), Some(&*format!("w{}", MAX_WORDS - 1))); + } + + #[test] + fn language_table_maps_and_falls_back() { + assert_eq!(language_code("en"), "eng"); + assert_eq!(language_code(" EN "), "eng", "trimmed, case-insensitive"); + assert_eq!(language_code("de"), "deu"); + assert_eq!(language_code("ZH"), "chi_sim"); + assert_eq!(language_code("ja"), "jpn"); + assert_eq!(language_code("ko"), "kor"); + assert_eq!(language_code("nl"), "nld"); + assert_eq!(language_code("klingon"), "eng", "unknown → eng"); + assert_eq!(language_code(""), "eng"); + // The security property: whatever the settings say, only allow-listed codes reach a + // process. + for code in ["en", "de", "fr", "es", "it", "pt", "ru", "zh", "ja", "ko", "ar", "nl", "klingon", "", " eng ", "EN/;rm"] { + let mapped = language_code(code); + assert!(LANGUAGE_CODES.iter().any(|(_, v)| *v == mapped), "{code:?} → {mapped:?} is not allow-listed"); + } + } + + #[test] + fn list_langs_come_from_stderr_when_stdout_has_none() { + let listing = "List of available languages in \"/usr/share/tessdata/\" (2):\neng\nosd\n"; + assert_eq!(list_langs_from(listing, ""), ["eng", "osd"]); + assert_eq!(list_langs_from("", listing), ["eng", "osd"], "3.x prints the list on stderr"); + assert_eq!(list_langs_from("", "Error opening data file\n"), Vec::::new(), "a warning is not a language"); + assert_eq!(list_langs_from("eng\n", "deu\n"), ["eng"], "stdout wins when it has the list"); + } + + /// The private temp directory is created owner-only (Unix) and removed on drop. + #[test] + fn temp_dir_is_private_and_removed() { + let dir = TempDir::create().unwrap(); + let path = dir.path.clone(); + assert!(path.is_dir()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o700); + } + drop(dir); + assert!(!path.exists()); + } + + /// A set `PDFCRAFT_TESSERACT` decides alone: when it is invalid, [`TesseractCli::find`] does + /// not fall back to a PATH copy (which would run a program the user did not name); the PATH + /// search only runs when nothing is configured. + #[test] + fn an_invalid_configured_program_fails_closed() { + let Ok(real) = std::env::current_exe() else { return }; + let searched = std::cell::Cell::new(false); + let on_path = || { + searched.set(true); + Some(real.clone()) + }; + assert_eq!(locate(None, on_path), Some(real.clone()), "nothing configured: the PATH search decides"); + assert!(searched.get()); + searched.set(false); + assert_eq!(locate(Some(real.clone().into_os_string()), on_path), Some(real.clone()), "a valid location is used as is"); + assert!(!searched.get(), "and the PATH is not searched"); + let dir = real.parent().map(Path::to_path_buf).unwrap_or_default().into_os_string(); + for bad in [std::ffi::OsString::from("relative/tesseract"), std::ffi::OsString::from(""), dir, "/no/such/program-0123456789".into()] { + assert_eq!(locate(Some(bad.clone()), on_path), None, "{bad:?} is invalid, so there is no program"); + assert!(!searched.get(), "{bad:?}: an invalid location never falls back to the PATH"); + } + } + + #[test] + fn the_version_comes_from_stdout_or_stderr() { + assert_eq!(version_line("tesseract 5.4.0\n leptonica-1.84\n", ""), Some("tesseract 5.4.0".into())); + assert_eq!(version_line("", "\ntesseract 4.0.0\n"), Some("tesseract 4.0.0".into()), "older versions print it on stderr"); + assert_eq!(version_line(" \n", ""), None); + } + + #[test] + fn list_langs_output_parses() { + let out = "List of available languages in \"C:\\Program Files\\Tesseract-OCR\\tessdata/\" (2):\neng\nosd\n"; + assert_eq!(parse_list_langs(out), vec!["eng".to_string(), "osd".to_string()]); + assert_eq!(parse_list_langs("List of available languages (1):\neng\n"), vec!["eng".to_string()]); + assert!(parse_list_langs("").is_empty()); + } + + /// The real binary, when it is installed: a run end to end, with words that carry a + /// confidence and the engine's name (skipped where `tesseract` is missing). + #[test] + fn recognize_runs_the_installed_binary() { + let Some(t) = TesseractCli::find() else { + eprintln!("skipped: the tesseract program is not installed"); + return; + }; + assert_eq!(t.id(), "tesseract"); + assert!(t.available()); + // 300 dpi-ish block capitals, black on white. + let (w, h, rgba) = hello_world_image(); + let image = OcrImage::new(w, h, rgba).unwrap(); + let lines = t.recognize(&image, &RecognizeOptions::default()).unwrap(); + let text: Vec = lines.iter().map(Line::text).collect(); + assert!(text.iter().any(|t| t.to_uppercase().contains("HELL")), "{text:?}"); + let words: Vec<&Word> = lines.iter().flat_map(|l| &l.words).collect(); + assert!(words.iter().all(|w| w.source == "tesseract")); + assert!(words.iter().all(|w| matches!(w.confidence, Some(c) if (0.0..=100.0).contains(&c))), "{words:?}"); + } + + /// `available`/`languages` come from `--list-langs` (skipped without the binary). + #[test] + fn languages_come_from_list_langs() { + let Some(t) = TesseractCli::find() else { + eprintln!("skipped: the tesseract program is not installed"); + return; + }; + let langs = t.languages(); + assert!(!langs.is_empty()); + assert!(langs.iter().all(|(c, n)| !c.is_empty() && !n.is_empty()), "{langs:?}"); + } + + #[cfg(unix)] + #[test] + fn recognize_refuses_a_truncated_reading() { + let dir = TempDir::create().unwrap(); + let script = dir.path.join("synthetic-recognizer"); + let mut tsv = String::new(); + for i in 0..=MAX_WORDS { + use std::fmt::Write as _; + writeln!(tsv, "5\t1\t1\t1\t1\t{i}\t0\t0\t1\t1\t95\tword{i}").unwrap(); + } + let exact = tsv.lines().take(MAX_WORDS).collect::>().join("\n"); + std::fs::write(dir.path.join("reading.tsv"), tsv).unwrap(); + std::fs::write(&script, "#!/bin/sh\ncat \"$(dirname \"$0\")/reading.tsv\"\n").unwrap(); + make_executable(&script); + let recognizer = TesseractCli::new(script); + let result = recognizer.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + let status = result.as_ref().map(|lines| lines.iter().map(|line| line.words.len()).sum::()).map_err(ToString::to_string); + assert!( + matches!(result, Err(OcrError::Recognize(ref reason)) if reason.contains("word limit")), + "a capped reading must be reported, never accepted as complete: {status:?}" + ); + std::fs::write(dir.path.join("reading.tsv"), exact).unwrap(); + let complete = recognizer.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()).unwrap(); + assert_eq!(complete.iter().map(|line| line.words.len()).sum::(), MAX_WORDS); + } + + /// A hung binary is killed at the timeout and reported (Unix: a fake `tesseract` script; + /// this test needs a real process, so it is off the web and off Windows). + #[cfg(unix)] + #[test] + fn a_hung_program_is_killed_at_the_timeout() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-hang"); + std::fs::write(&script, "#!/bin/sh\nsleep 30\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.timeout = Duration::from_millis(500); + let started = Instant::now(); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(matches!(r, Err(OcrError::Timeout(_))), "{r:?}"); + assert!(started.elapsed() < Duration::from_secs(10), "killed, not waited out"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Output past the cap kills the run EARLY, while the child is still writing (Unix only, + /// same fake-script trick): the script floods far past the cap and would sleep for 30 s + /// after; a cap checked only after the exit would come back as a timeout instead. + #[cfg(unix)] + #[test] + fn flooding_output_is_capped_and_killed_early() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-cap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-flood"); + std::fs::write(&script, "#!/bin/sh\nhead -c 100000000 /dev/zero | tr '\\0' 'x'\nsleep 30\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.max_output = 1024; + t.timeout = Duration::from_secs(20); + let started = Instant::now(); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(matches!(r, Err(OcrError::OutputTooLarge(_))), "{r:?}"); + assert!(started.elapsed() < Duration::from_secs(10), "killed early, at {:?}", started.elapsed()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// An stderr flood does not block the child, so it does not cause a false timeout (Unix + /// only): the script writes megabytes of stderr — far more than a pipe would swallow — + /// then exits successfully with a small output, which must come back as a reading. + #[cfg(unix)] + #[test] + fn an_stderr_flood_does_not_cause_a_false_timeout() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-err-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-err-flood"); + // (below MAX_STDERR, past which the child is stopped: `an_stderr_flood_is_killed_while_the_child_runs`) + std::fs::write(&script, "#!/bin/sh\nhead -c 200000 /dev/zero | tr '\\0' 'e' >&2\necho flooded away\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.timeout = Duration::from_secs(10); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(r.is_ok(), "stderr flood must not fail the run: {r:?}"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The child runs with a minimal environment (its data prefix, the PATH, the Windows + /// system root, and a temp dir), never the caller's whole environ. This is checked with a + /// script that prints its own environ as the "TSV" (Unix only). + #[cfg(unix)] + #[test] + fn the_child_gets_a_minimal_environment() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-env-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-env"); + std::fs::write(&script, "#!/bin/sh\nenv | grep -v '^_='\n").unwrap(); + make_executable(&script); + let t = TesseractCli::new(script); + let png = encode_png(&blot(8, 8, 0, 0, 4, 4)).unwrap(); + std::fs::write(dir.join("page.png"), png).unwrap(); + let png_arg = dir.join("page.png").to_string_lossy().into_owned(); + let tsv = t.run(&[png_arg.as_str(), "stdout", "-l", "eng", "tsv"], t.timeout, t.max_output).unwrap(); + for (why, name) in [("the child's temp dir", "TMP="), ("temp", "TEMP="), ("lookup path", "PATH=")] { + assert!(tsv.contains(name), "{why} must be passed: {tsv}"); + } + // cargo always sets CARGO_* for the test process; env_clear must drop them. + assert!(!tsv.contains("CARGO_"), "the caller's environment must not leak: {tsv}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + fn make_executable(p: &Path) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)); + } + + /// The temp directory and its contents are gone after a run (private files do not linger). + #[cfg(unix)] + #[test] + fn temp_files_are_deleted_and_private() { + let dir = TempDir::create().unwrap(); + assert!(dir.path.is_dir()); + std::fs::write(dir.path.join("x"), b"secret").unwrap(); + let p = dir.path.clone(); + drop(dir); + assert!(!p.exists(), "the temp directory is deleted after use"); + } + + /// A configured program location is only usable as an absolute path to an existing, + /// executable file: a relative path, a missing file and a directory are unavailable (and + /// [`TesseractCli::at`] refuses them without spawning anything). + #[test] + fn a_configured_program_must_be_an_absolute_executable_file() { + assert!(!program_is_executable(Path::new(".")), "relative"); + assert!(!program_is_executable(Path::new("definitely/missing/tesseract")), "missing"); + let temp = std::env::temp_dir(); + assert!(!program_is_executable(&temp), "a directory is not a program"); + let file = temp.join(format!("pdfcraft-ocr-not-a-program-{}", std::process::id())); + std::fs::write(&file, b"").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)); + assert!(!program_is_executable(&file), "a file without executable bits is not a program"); + } + #[cfg(not(unix))] + assert!(program_is_executable(&file), "an existing file is usable on this platform"); + let _ = std::fs::remove_file(&file); + // `at` validates before any process starts, so these are None and not panics. + assert!(TesseractCli::at(PathBuf::from(".")).is_none()); + assert!(TesseractCli::at(temp).is_none()); + assert!(TesseractCli::at(PathBuf::from("definitely/missing/tesseract")).is_none()); + } + + /// The PATH search resolves through absolute entries only and skips relative ones: a + /// relative PATH entry would resolve against the caller's working directory. + #[test] + fn path_search_skips_relative_entries() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-path-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let program = dir.join(program_file_names()[0]); + std::fs::write(&program, b"").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)); + } + let found = path_search_in([PathBuf::from("relative/dir"), PathBuf::from("definitely/missing"), dir.clone()].into_iter()); + assert_eq!(found.as_deref(), Some(program.as_path()), "the absolute entry with the program wins, relative entries are skipped"); + let none = path_search_in([PathBuf::from("relative/dir")].into_iter()); + assert_eq!(none, None, "only relative entries: nothing is found"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// "HELLO" as blocky 5×7 glyphs, 8 pixels per dot with wide tracking, at roughly 300 dpi + /// scale (each dot ~10 px): big enough for Tesseract's word detector. + fn hello_world_image() -> (u32, u32, Vec) { + const GLYPHS: &[&str] = &[ + "X...X", "X...X", "X...X", "XXXXX", "X...X", "X...X", "X...X", // H + ]; + // Draw the five glyphs of HELLO with per-letter shapes for E, L, L, O. + const E: &[&str] = &["XXXXX", "X....", "X....", "XXXX.", "X....", "X....", "XXXXX"]; + const L: &[&str] = &["X....", "X....", "X....", "X....", "X....", "X....", "XXXXX"]; + const O: &[&str] = &[".XXX.", "X...X", "X...X", "X...X", "X...X", "X...X", ".XXX."]; + let letters: [&[&str]; 5] = [GLYPHS, E, L, L, O]; + let (dot, x0, y0, advance) = (8u32, 40u32, 40u32, 7u32 * 8); + let w = x0 * 2 + advance * 5; + let h = y0 * 2 + 7 * dot; + let mut rgba = vec![255u8; (w * h * 4) as usize]; + for (i, rows) in letters.iter().enumerate() { + for (ry, row) in rows.iter().enumerate() { + for (rx, b) in row.bytes().enumerate() { + if b != b'X' { + continue; + } + for dy in 0..dot { + for dx in 0..dot { + let x = x0 + i as u32 * advance + rx as u32 * dot + dx; + let y = y0 + ry as u32 * dot + dy; + let o = ((y * w + x) * 4) as usize; + rgba[o..o + 3].copy_from_slice(&[0, 0, 0]); + } + } + } + } + } + (w, h, rgba) + } +} + +#[cfg(all(test, unix))] +mod process_tests { + use super::*; + + /// A stderr flood is killed at the cap while the child still runs, not accepted until it + /// exits (an endless writer would otherwise run out the whole timeout and fill the disk). + /// The fixture needs a Unix shell, like the other process fixtures. + #[test] + fn a_stderr_flood_is_killed_while_the_child_runs() { + let dir = std::env::temp_dir().join(format!("pdfcraft-tesseract-flood-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let program = dir.join("flood-tesseract"); + std::fs::write(&program, "#!/bin/sh\nwhile true; do echo aaaaaaaaaaaaaaaaaaaaaaaa >&2; done\n").unwrap(); + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&program).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&program, perms).unwrap(); + let t = TesseractCli::new(program); + let started = std::time::Instant::now(); + let err = t.run_capture(&["--list-langs"], Duration::from_secs(60), 1024 * 1024).unwrap_err(); + assert!(matches!(err, OcrError::OutputTooLarge(_)), "{err}"); + assert!(started.elapsed() < Duration::from_secs(30), "killed at the cap, not at the timeout"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index d463303fcd..78cff889d3 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3953,6 +3953,41 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// An invisible OCR-style text layer drawn in a Type3 font (what the OCR output writes for +/// characters beyond WinAnsi) is redacted like any other text: its glyphs are hit-tested by +/// their real advances, removed, and the verifier passes. +#[test] +fn a_type3_text_layer_is_redacted_like_any_other() { + // One glyph (code 1): a filled box, 500 units wide, with a ToUnicode CMap that reads it + // back as U+3042. + let t3 = b"<< /Type /Font /Subtype /Type3 /Name /PCOcr /FontBBox [0 0 700 700] /FontMatrix [0.001 0 0 0.001 0 0] \ + /FirstChar 1 /LastChar 1 /Widths [500] /Encoding << /Type /Encoding /Differences [1 /g01] >> \ + /CharProcs << /g01 8 0 R >> /ToUnicode 9 0 R >>"; + let charproc = b"500 0 0 0 0 700 700 d1\n0 0 700 700 re f\n"; + let cmap = b"/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapType 2 def\n1 begincodespacerange\n<01> <01>\nendcodespacerange\n1 beginbfchar\n<01> <3042>\nendbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"; + let layer = b"BT 3 Tr /T3 10 Tf 100 100 Td (\x01) Tj ET"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /T3 5 0 R >> >> >>".to_vec(), + stream("", layer), + t3.to_vec(), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + stream("", charproc), + stream("", cmap), + ]); + // The glyph box at size 10: 5 pt wide, from the font's descent to its ascent. + assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 1, "the Type3 glyph is found under the mark"); + assert_eq!(under(&mut doc, 0, &[[200.0, 200.0, 250.0, 250.0]]), 0, "elsewhere the page is clean"); + mark(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.glyphs, 1, "{report:?}"); + // The verified apply left no text-showing operator: the re-read page is empty of them. + assert!(!content(&doc, 0).contains("Tj"), "{}", content(&doc, 0)); + assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 0, "nothing is left under the mark"); +} + #[test] fn a_rewritten_form_xobject_drops_the_source_text_of_added_content() { // A form drawing invisible text whose dictionary repeats it in /PCAdded. diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index c859b93cf6..22e7323476 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -23,6 +23,7 @@ pdfcraft-fonts = { workspace = true } pdfcraft-engine = { workspace = true } pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } +pdfcraft-ocr = { workspace = true } pdfcraft-render = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-platform = { workspace = true } diff --git a/crates/ui-egui/examples/shot.rs b/crates/ui-egui/examples/shot.rs index 01e48e03d6..8058df40c8 100644 --- a/crates/ui-egui/examples/shot.rs +++ b/crates/ui-egui/examples/shot.rs @@ -6,18 +6,64 @@ //! `--width N` downscales the image to N pixels wide (README screenshots). Any other //! `--key value` pair is passed to `PdfCraftApp::set_option` //! (the same verbs as the desktop app's command-line flags and the future control channel). +//! `--ocr-verify on` ignores any file and instead builds a synthetic scanned page, recognizes +//! it with the real engines (models required: `cargo xtask models`) and opens the OCR Verify +//! screen on the result, as a reviewer sees it. use std::time::{Duration, Instant}; use egui_kittest::Harness; +use pdfcraft_engine::ocr::{MergeStrategy, OcrSettings}; +use pdfcraft_engine::{Session, export}; use pdfcraft_ui_egui::PdfCraftApp; +/// Build a one-page PDF that is only a picture of a sentence (as a scanner would produce), +/// recognize it with the engines named in `settings` and open the OCR Verify screen on the +/// result — the same path the review tests use, with a real recognition behind it. The toolbar +/// dropdowns are seeded to the ensemble the shot ran. +fn load_ocr_verify(app: &mut PdfCraftApp) -> Result<(), String> { + let mut s = Session::new(); + let text = s + .create_from_text( + "t", + "Scanned pages are pictures of words.\nThe Verify screen reads them back before anything is applied.\nReview the uncertain words, then Accept.", + ) + .map_err(|e| e.to_string())?; + let id = s.open("text.pdf", None, text, None).map_err(|e| e.to_string())?; + let png = export::Exporter::new(s.get(id).ok_or("the document is gone")?).png(0, 150.0).map_err(|e| e.to_string())?; + // Keep the text block of the rendered page, so the scan is a short note-sized picture: + // the whole page and the confidence legend fit in the shot's source-page pane. + let mut page = image::load_from_memory(&png).map_err(|e| e.to_string())?; + let mut scan_png = std::io::Cursor::new(Vec::new()); + page.crop(0, 0, page.width(), (page.height() as f32 * 0.34) as u32) + .write_to(&mut scan_png, image::ImageFormat::Png) + .map_err(|e| e.to_string())?; + let scan = s.create_from_images(&[("scan.png".into(), scan_png.into_inner())]).map_err(|e| e.to_string())?.to_vec(); + let settings = + OcrSettings { engine: pdfcraft_engine::ocr::EngineChoice::Auto, strategy: MergeStrategy::ConfidenceWeighted, ..OcrSettings::default() }; + app.ocr_draft.engine = settings.engine; + app.ocr_draft.strategy = settings.strategy; + app.open_bytes("scan.pdf", None, scan).map_err(|e| e.to_string())?; + app.run_inline = true; + if !app.execute("ocr.correct") { + return Err("ocr.correct did not run".into()); + } + let Some((_, id)) = app.active_ids() else { return Err("no open document".into()) }; + let job = app.session.ocr_job(id, &[], settings.clone()).ok_or("the OCR job could not be captured")?; + let recognizers = pdfcraft_engine::ocr::recognizers(&settings)?; + let guard = job.guard.clone(); + let pages = job.run(&recognizers, |_, _| true); + app.ocr_verify.load(id, guard, pages); + Ok(()) +} + fn main() -> Result<(), String> { let mut args = std::env::args().skip(1); let out = args.next().ok_or("usage: shot [file.pdf] [--option value …]")?; let (mut file, mut opts) = (None, Vec::new()); let (mut size, mut scale) = (egui::vec2(1440.0, 900.0), 2.0f32); let mut width: Option = None; + let mut ocr_verify = false; while let Some(a) = args.next() { match a.strip_prefix("--") { Some("size") => { @@ -27,13 +73,20 @@ fn main() -> Result<(), String> { } Some("scale") => scale = args.next().and_then(|v| v.parse().ok()).ok_or("bad --scale")?, Some("width") => width = Some(args.next().and_then(|v| v.parse().ok()).ok_or("bad --width")?), + Some("ocr-verify") => ocr_verify = args.next().is_some_and(|v| v == "on"), Some(k) => opts.push((k.to_string(), args.next().unwrap_or_default())), None => file = Some(a), } } let mut harness = Harness::builder().with_size(size).with_pixels_per_point(scale).build_eframe(move |_cc| { let mut app = PdfCraftApp::new(); - if let Some(f) = &file { + if ocr_verify { + if let Err(e) = load_ocr_verify(&mut app) { + eprintln!("shot: --ocr-verify: {e}"); + // an empty Verify screen must not pass for the documentation's screenshot + std::process::exit(1); + } + } else if let Some(f) = &file { let bytes = std::fs::read(f).unwrap_or_default(); let name = std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); if let Err(e) = app.open_bytes(&name, Some(f.clone()), bytes) { diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 61b85cd7f1..3b3d5dd036 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -202,6 +202,7 @@ impl PdfCraftApp { ocr_draft: ocr_ui::OcrDraft::default(), ocr_run: None, ocr_batch: None, + ocr_verify: ocr_verify::OcrVerify::default(), run_inline: false, custom_actions: Vec::new(), wizard: Default::default(), @@ -624,6 +625,14 @@ impl eframe::App for PdfCraftApp { self.poll_optimize(); self.poll_scan_clean(); self.poll_action(); + // The OCR Verify screen's worker may have finished while no frame was drawn. + if self.ocr_verify.phase == workflows::ocr_verify::Phase::Running { + workflows::ocr_verify::with_state(self, workflows::ocr_verify::OcrVerify::poll); + } + // So may a page or region re-read's worker. + if self.ocr_verify.reread.is_some() { + workflows::ocr_verify::with_state(self, workflows::ocr_verify::OcrVerify::poll_reread); + } self.process_file_requests(); #[cfg(not(target_arch = "wasm32"))] self.process_picked(); @@ -680,6 +689,20 @@ impl eframe::App for PdfCraftApp { if self.draws_window_frame() { window_controls::resize_edges(&ctx); } + if self.ocr_verify.open { + // The OCR Verify screen owns the window (its four panes); dialogs and notices + // still show above it. + workflows::ocr_verify::show(self, ui); + dialogs::show(self, &ctx); + self.show_progress(&ctx); + widgets::toast(self, &ctx); + self.finish_render_frame(&ctx); + if let Some(mut control) = self.control.take() { + control.finish_frame(&ctx, self); + self.control = Some(control); + } + return; + } chrome::tab_strip(self, ui); chrome::mode_bar(self, ui); // Whichever way another tab came to the front, it shows instead of the expanded view. diff --git a/crates/ui-egui/src/app/prefs.rs b/crates/ui-egui/src/app/prefs.rs index 89789d8a4c..1a8c2d269e 100644 --- a/crates/ui-egui/src/app/prefs.rs +++ b/crates/ui-egui/src/app/prefs.rs @@ -144,6 +144,7 @@ impl PdfCraftApp { "combine_zoom": self.combine_zoom, "comments_panel_closed": self.comments_panel_closed, "right_panel_closed": self.right_panel_closed, + "ocr": self.ocr_draft.to_pref(), }); // Set apart: one more key inside the macro above passes serde_json's recursion limit. v["default_page_gaps"] = serde_json::Value::Bool(self.view_defaults.show_page_gaps); @@ -295,6 +296,9 @@ impl PdfCraftApp { if let Some(closed) = v["right_panel_closed"].as_bool() { self.right_panel_closed = closed; } + // Scan & OCR choices (language, engine, strategy, output mode, preprocessing and skip + // flags); unknown or malformed values keep the defaults. + self.ocr_draft.apply_pref(&v["ocr"]); if let Some(defaults) = v["default_zoom"].as_str().and_then(|zoom| self.view_defaults.with_zoom(zoom)) { self.view_defaults = defaults; } diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index 4601986e9a..7c0df87782 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -527,6 +527,7 @@ impl PdfCraftApp { "ocr.recognize" => self.dialog = Some(Dialog::RecognizeText), "ocr.enhance" => self.dialog = Some(Dialog::EnhanceScan), "ocr.recognize_batch" => self.ocr_files_dialog(), + "ocr.correct" => self.ocr_verify.open = true, "edit.edit_text" => { self.quick_tool = crate::QuickTool::EditText; self.left = crate::LeftPanel::Tool("edit"); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 94e83f451a..2f393c61d1 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -69,8 +69,9 @@ use pdfcraft_engine::{DocId, Session}; pub use canvas::{DocView, RasterMemory}; pub use editing::{CloseRequest, SaveTarget}; pub use files::{ExtractDraft, FilePurpose, FileRequest, RotateDraft, SplitDraft, SplitMode, SplitPlan}; -pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore}; +pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore, default_dictionary_dir}; use theme::{ThemeKind, ThemePreference}; +pub use workflows::ocr_verify::{OcrVerify, Phase}; mod app; mod hooks; @@ -181,6 +182,7 @@ use workflows::export_ui; use workflows::files; use workflows::js_ui; use workflows::ocr_ui; +use workflows::ocr_verify; use workflows::optimize_ui; use workflows::print_ui; use workflows::scan_clean_ui; @@ -510,6 +512,8 @@ pub struct PdfCraftApp { pub ocr_draft: ocr_ui::OcrDraft, pub ocr_run: Option, pub ocr_batch: Option>>, + /// Scan & OCR ▸ Correct recognized text: the OCR Verify screen (its state, never persisted). + pub ocr_verify: ocr_verify::OcrVerify, /// Background jobs (OCR, actions, listing pinned folders) run inline instead (tests). pub run_inline: bool, /// Action Wizard: the user's actions, the dialog state, the running action and (tests) the diff --git a/crates/ui-egui/src/persist/recovery.rs b/crates/ui-egui/src/persist/recovery.rs index 71759c4599..f217781c07 100644 --- a/crates/ui-egui/src/persist/recovery.rs +++ b/crates/ui-egui/src/persist/recovery.rs @@ -184,7 +184,27 @@ impl RecoveryStore { pub fn dir(&self) -> &Path { &self.dir } +} +/// The OCR Verify screen's per-language user dictionary folder, next to the recovery store: +/// the portable data folder's `Dictionary` in portable mode ([`crate::portable`]), otherwise +/// `…/PdfCraft/Dictionary` in the platform's per-user data folder. `None` where files are not +/// kept (the web build). +pub fn default_dictionary_dir() -> Option { + if let Some(dir) = crate::portable::data_dir() { + return Some(dir.join("Dictionary")); + } + let env = |k: &str| std::env::var_os(k).filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(target_os = "macos") { + env("HOME").map(|h| h.join("Library/Application Support/PdfCraft/Dictionary")) + } else if cfg!(windows) { + env("LOCALAPPDATA").map(|d| d.join("PdfCraft").join("Dictionary")) + } else { + env("XDG_DATA_HOME").or_else(|| env("HOME").map(|h| h.join(".local/share"))).map(|d| d.join("pdfcraft/dictionary")) + } +} + +impl RecoveryStore { /// Write (or replace) an entry. The PDF is written before its metadata, both atomically, so /// a listed entry always has complete bytes. pub fn write(&self, meta: &RecoveryMeta, bytes: &[u8]) -> std::io::Result<()> { diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index c4ca254948..f621c7832b 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -477,6 +477,9 @@ pub fn toast(app: &mut PdfCraftApp, ctx: &egui::Context) { // Above the progress notice while a job runs. let card = if app.progress_notice.is_some() { ctx.data(|d| d.get_temp::(egui::Id::unique(PROGRESS_HEIGHT))).unwrap_or(0.0) } else { 0.0 }; let lift = if card > 0.0 { 28.0 + card + 10.0 } else { 28.0 }; + // A running batch keeps its toast up (poll_ocr re-notifies), so it can carry the one + // control a long job needs: Cancel — the file in flight finishes, the rest are not run. + let batch_running = app.ocr_batch.is_some(); egui::Area::new(egui::Id::unique("toast")) .order(egui::Order::Tooltip) .pivot(Align2::CENTER_BOTTOM) @@ -494,7 +497,12 @@ pub fn toast(app: &mut PdfCraftApp, ctx: &egui::Context) { ui.set_max_width((screen.width() - 64.0).clamp(1.0, 560.0)); let ink = if t.dark() { Color32::from_rgb(0x22, 0x22, 0x26) } else { Color32::WHITE }; let text = |ui: &mut egui::Ui| { - ui.add(egui::Label::new(egui::RichText::new(msg).color(ink)).wrap()); + ui.horizontal_wrapped(|ui| { + ui.add(egui::Label::new(egui::RichText::new(msg).color(ink)).wrap()); + if batch_running && ui.small_button(crate::i18n::t("Cancel")).clicked() { + app.cancel_ocr_batch(); + } + }); }; match script { // A form script's message: its icon, then its title (when it gave one) diff --git a/crates/ui-egui/src/workflows.rs b/crates/ui-egui/src/workflows.rs index 9314914ec4..8701eac119 100644 --- a/crates/ui-egui/src/workflows.rs +++ b/crates/ui-egui/src/workflows.rs @@ -9,6 +9,7 @@ pub(super) mod export_ui; pub(super) mod files; pub(super) mod js_ui; pub(super) mod ocr_ui; +pub(super) mod ocr_verify; pub(super) mod optimize_ui; pub(super) mod print_ui; pub(super) mod scan_clean_ui; diff --git a/crates/ui-egui/src/workflows/ocr_ui.rs b/crates/ui-egui/src/workflows/ocr_ui.rs index 04e257d52d..7534915d10 100644 --- a/crates/ui-egui/src/workflows/ocr_ui.rs +++ b/crates/ui-egui/src/workflows/ocr_ui.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use egui::{Align, Layout}; use pdfcraft_engine::DocId; -use pdfcraft_engine::ocr::{LANGUAGES, OcrPage, OcrSettings}; +use pdfcraft_engine::ocr::{EngineChoice, LANGUAGES, MergeStrategy, OcrPage, OcrSettings, OutputMode}; use crate::theme::{self, Tokens}; use crate::{PdfCraftApp, widgets}; @@ -19,7 +19,10 @@ pub enum OcrPages { Range, } -/// The dialog's choices (kept between runs, as in Acrobat). +/// The dialog's choices (kept between runs, as in Acrobat). The Recognize Text dialog, the OCR +/// Verify screen and the batch options all read and write this one draft, and it is what the +/// preferences persist: language, output mode, the four preprocessing flags, engine, strategy +/// and the three skip flags — never review edits, verified flags or the selection. #[derive(Clone, Debug, PartialEq)] pub struct OcrDraft { pub pages: OcrPages, @@ -30,18 +33,135 @@ pub struct OcrDraft { pub dpi: u32, /// Also recognize pages that already have text (pictures on them; their text isn't added again). pub text_pages: bool, + /// Which engine reads the pages. + pub engine: EngineChoice, + /// How two engines' readings combine. + pub strategy: MergeStrategy, + /// Preprocessing flags, all off by default. + pub auto_rotate: bool, + pub deskew: bool, + pub denoise: bool, + pub binarize: bool, + /// What Accept produces. + pub output_mode: OutputMode, + /// Batch: leave files that already contain text alone. + pub skip_text_files: bool, + /// Batch: leave pages that already contain text alone. + pub skip_text_pages: bool, + /// Batch: read everything, overriding both skips. + pub force_ocr: bool, + /// Batch: where the searchable copies go (None: ask each time). Remembered from the last + /// batch's folder pick, so a repeated export into the same place does not re-ask. + pub output_folder: Option, } impl Default for OcrDraft { fn default() -> Self { - OcrDraft { pages: OcrPages::All, from: 1, to: 1, language: "en".into(), dpi: 300, text_pages: false } + OcrDraft { + pages: OcrPages::All, + from: 1, + to: 1, + language: "en".into(), + dpi: 300, + text_pages: false, + engine: EngineChoice::default(), + strategy: MergeStrategy::default(), + auto_rotate: false, + deskew: false, + denoise: false, + binarize: false, + output_mode: OutputMode::default(), + skip_text_files: false, + skip_text_pages: true, + force_ocr: false, + output_folder: None, + } } } impl OcrDraft { /// The recognition settings these choices make. pub fn settings(&self) -> OcrSettings { - OcrSettings { dpi: self.dpi as f32, language: self.language.clone(), skip_text_pages: !self.text_pages, ..Default::default() } + OcrSettings { + dpi: self.dpi as f32, + language: self.language.clone(), + engine: self.engine, + strategy: self.strategy, + auto_rotate: self.auto_rotate, + deskew: self.deskew, + denoise: self.denoise, + binarize: self.binarize, + output_mode: self.output_mode, + skip_text_files: self.skip_text_files, + skip_text_pages: self.skip_text_pages && !self.text_pages, + force_ocr: self.force_ocr, + ..Default::default() + } + } + + /// Restore from the preferences JSON (`ocr` object); unknown or malformed values keep the + /// defaults, like every other restored setting. + pub fn apply_pref(&mut self, v: &serde_json::Value) { + if let Some(l) = v["language"].as_str().filter(|l| LANGUAGES.iter().any(|x| x.0 == *l)) { + self.language = l.to_string(); + } + if let Some(e) = v["engine"].as_str() { + self.engine = EngineChoice::parse(e); + } + if let Some(s) = v["strategy"].as_str() { + self.strategy = match s.trim().to_ascii_lowercase().as_str() { + "confidence" | "confidence-weighted" | "confidence_weighted" => MergeStrategy::ConfidenceWeighted, + "rover" | "rover-vote" | "vote" => MergeStrategy::RoverVote, + "primary" | "primary-only" => MergeStrategy::PrimaryOnly, + _ => MergeStrategy::PrimaryOnly, + }; + } else if self.engine == EngineChoice::Auto { + // No strategy was saved with "Automatic": it reads the way Automatic reads by + // default, not as "Primary only" (which would never bring in the second engine). + self.strategy = self.engine.default_strategy(); + } + if let Some(m) = v["output_mode"].as_str() { + self.output_mode = OutputMode::parse(m); + } + for (key, flag) in + [("auto_rotate", &mut self.auto_rotate), ("deskew", &mut self.deskew), ("denoise", &mut self.denoise), ("binarize", &mut self.binarize)] + { + if let Some(on) = v[key].as_bool() { + *flag = on; + } + } + for (key, flag) in + [("skip_text_files", &mut self.skip_text_files), ("skip_text_pages", &mut self.skip_text_pages), ("force_ocr", &mut self.force_ocr)] + { + if let Some(on) = v[key].as_bool() { + *flag = on; + } + } + if let Some(f) = v["output_folder"].as_str().filter(|f| !f.is_empty()) { + self.output_folder = Some(f.to_string()); + } + } + + /// The preferences JSON for `persist`. + pub fn to_pref(&self) -> serde_json::Value { + serde_json::json!({ + "language": self.language, + "engine": self.engine.name(), + "strategy": match self.strategy { + MergeStrategy::PrimaryOnly => "primary", + MergeStrategy::ConfidenceWeighted => "confidence", + MergeStrategy::RoverVote => "rover", + }, + "output_mode": self.output_mode.name(), + "auto_rotate": self.auto_rotate, + "deskew": self.deskew, + "denoise": self.denoise, + "binarize": self.binarize, + "skip_text_files": self.skip_text_files, + "skip_text_pages": self.skip_text_pages, + "force_ocr": self.force_ocr, + "output_folder": self.output_folder, + }) } } @@ -54,12 +174,14 @@ pub struct OcrProgress { pub cancel: bool, } -/// Recognize text in multiple files: files done, total, and the summary once finished. +/// Recognize text in multiple files: files done, total, and the summary once finished. Cancel +/// is asked for between files: the file in flight finishes, the rest are counted as not run. #[derive(Default)] pub struct BatchProgress { pub done: usize, pub total: usize, pub message: Option, + pub cancel: bool, } /// A page Edit should read now (see [`PdfCraftApp::scan_to_read`]). @@ -74,10 +196,16 @@ struct ScanToRead { pub struct OcrRun { pub doc: DocId, pub progress: Arc>, + /// What the document looked like when the job started: a result is applied only while it + /// still does (the dialog closes on Go, so the user can edit, or redact, while this runs). + pub guard: pdfcraft_engine::ocr::OcrGuard, /// Set for a run Edit started on a scanned page, not asked for: it shows no progress, says nothing /// when it found no words (a page with text of its own is skipped, and Edit must not make noise), /// and its result waits for Read mode and dialogs and is dropped if the document changed. pub automatic: Option, + /// The dialog promised an editable document: the result goes to the OCR Verify screen + /// (whose Accept produces the editable document), not to an invisible layer. + pub editable: bool, } /// What an Edit-started run needs to know to refuse a result that no longer fits its document. @@ -90,9 +218,16 @@ pub struct AutoOcr { pub generation: u64, } +/// The Tesseract probe runs in the background (the interface never waits on a process): while +/// it runs, an engine choice reads as "still checking", so look again soon. +pub(crate) fn keep_asking_while_probing(ctx: &egui::Context) { + if pdfcraft_engine::ocr::engine_probe_running() { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } +} + pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { let pages = app.active_ids().and_then(|(_, id)| app.session.get(id)).map_or(1, |d| d.info.pages.len().max(1)); - let available = pdfcraft_engine::ocr::available(); let d = &mut app.ocr_draft; d.to = d.to.clamp(1, pages); d.from = d.from.clamp(1, d.to); @@ -127,11 +262,55 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo } }); ui.end_row(); + ui.label(tl!("Engine")); + let engine_name = match d.engine { + EngineChoice::Auto => tl!("Automatic"), + EngineChoice::Ocrs => tl!("Built-in"), + EngineChoice::Tesseract => tl!("Tesseract"), + }; + craft_ui::ComboBox::from_id_salt("ocr-engine").selected_text(engine_name).width(220.0).show_ui(ui, |ui| { + let choices: [(EngineChoice, &str); 3] = + [(EngineChoice::Auto, "Automatic"), (EngineChoice::Ocrs, "Built-in"), (EngineChoice::Tesseract, "Tesseract")]; + for (choice, name) in choices { + ui.selectable_value(&mut d.engine, choice, tl!(name)); + } + }); + ui.end_row(); + ui.label(tl!("Combine engines")); + let strategy_name = match d.strategy { + MergeStrategy::PrimaryOnly => tl!("Primary only"), + MergeStrategy::ConfidenceWeighted => tl!("Confidence weighted"), + MergeStrategy::RoverVote => tl!("ROVER vote"), + }; + craft_ui::ComboBox::from_id_salt("ocr-strategy").selected_text(strategy_name).width(220.0).show_ui(ui, |ui| { + let choices: [(MergeStrategy, &str); 3] = [ + (MergeStrategy::PrimaryOnly, "Primary only"), + (MergeStrategy::ConfidenceWeighted, "Confidence weighted"), + (MergeStrategy::RoverVote, "ROVER vote"), + ]; + for (choice, name) in choices { + ui.selectable_value(&mut d.strategy, choice, tl!(name)); + } + }); + ui.end_row(); ui.label(tl!("Output")); - craft_ui::ComboBox::from_id_salt("ocr-output").selected_text(tl!("Searchable Image (Exact)")).width(220.0).show_ui(ui, |ui| { - let _ = ui - .selectable_label(true, tl!("Searchable Image (Exact)")) + let mode_name = match d.output_mode { + OutputMode::Searchable => tl!("Searchable Image (Exact)"), + OutputMode::EditableText => tl!("Editable text"), + }; + craft_ui::ComboBox::from_id_salt("ocr-output").selected_text(mode_name).width(220.0).show_ui(ui, |ui| { + let s = ui + .selectable_label(d.output_mode == OutputMode::Searchable, tl!("Searchable Image (Exact)")) .on_hover_text(tl!("Adds invisible text over each word; the page image is not changed")); + if s.clicked() { + d.output_mode = OutputMode::Searchable; + } + let e = ui + .selectable_label(d.output_mode == OutputMode::EditableText, tl!("Editable text")) + .on_hover_text(tl!("Writes a new document of visible text; the scan is not changed")); + if e.clicked() { + d.output_mode = OutputMode::EditableText; + } }); ui.end_row(); ui.label(tl!("Downsample to")); @@ -145,20 +324,50 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.add_space(4.0); ui.add(crate::widgets::checkbox(&mut d.text_pages, tl!("Also recognize pages that already have text"))) .on_hover_text(tl!("Pictures on those pages are recognized too; words already in the page's text are not added again")); + ui.add(crate::widgets::checkbox(&mut d.auto_rotate, tl!("Straighten pages scanned sideways or upside down"))); + ui.add(crate::widgets::checkbox(&mut d.deskew, tl!("Straighten skewed text lines"))); + ui.add(crate::widgets::checkbox(&mut d.denoise, tl!("Remove scan speckle"))); + ui.add(crate::widgets::checkbox(&mut d.binarize, tl!("Black and white (binarize)"))); }); - if !available { - ui.label( - egui::RichText::new(tl!("Text recognition isn't installed: its model files are missing. Reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them.")) - .small() - .color(t.text_muted), - ); + group(ui, "Skip", &mut |ui| { + ui.add(crate::widgets::checkbox(&mut d.skip_text_pages, tl!("Skip pages that already have text"))) + .on_hover_text(tl!("Pages with renderable text are left alone and reported as skipped")); + ui.add(crate::widgets::checkbox(&mut d.skip_text_files, tl!("Skip files that already have text"))) + .on_hover_text(tl!("In multiple files: a file with any text is its own skipped bucket, and nothing is written for it")); + ui.add(crate::widgets::checkbox(&mut d.force_ocr, tl!("Force OCR (override skip options)"))) + .on_hover_text(tl!("Reads pages and files even where the skip options would leave them alone")); + }); + group(ui, "Output folder", &mut |ui| { + ui.horizontal(|ui| match &d.output_folder { + Some(f) => { + ui.monospace(f).on_hover_text(tl!("The searchable copies are written here under the source names")); + if ui.small_button(tl!("Clear")).clicked() { + d.output_folder = None; + } + } + None => { + ui.label(tl!("Automatic")).on_hover_text(tl!("A folder is asked for when a multiple-file run starts")); + } + }); + }); + keep_asking_while_probing(ui.ctx()); + if let Some(why) = pdfcraft_engine::ocr::engine_unavailable_reason(d.engine) { + // Said in the interface's language: the built-in engine's missing models get the + // notice users know; Tesseract's reasons are catalog keys of their own. + let text = match d.engine { + pdfcraft_engine::ocr::EngineChoice::Tesseract => crate::i18n::t(&why).to_string(), + _ => tl!("Text recognition isn't installed: its model files are missing. Reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them.") + .to_string(), + }; + ui.label(egui::RichText::new(text).small().color(t.text_muted)); ui.add_space(6.0); } ui.add_space(6.0); let (mut go, mut cancel) = (false, false); + let can_run = pdfcraft_engine::ocr::engine_unavailable_reason(d.engine).is_none(); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(available, |ui| widgets::pill_button(ui, tl!("Recognize text"), true)).inner.clicked() { + if ui.add_enabled_ui(can_run, |ui| widgets::pill_button(ui, tl!("Recognize text"), true)).inner.clicked() { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -201,7 +410,9 @@ impl PdfCraftApp { /// is one undoable step, "Recognize text", and is not run again for that page after Undo. pub(crate) fn auto_ocr_scanned_page(&mut self) { let Some(scan) = self.scan_to_read() else { return }; - if !pdfcraft_engine::ocr::available() { + // the engine this run will use, not always the built-in one + let mut settings = self.ocr_draft.settings(); + if pdfcraft_engine::ocr::engine_unavailable_reason(settings.engine).is_some() { self.views[scan.view].auto_ocr_skipped.insert(scan.page, scan.generation); return; } @@ -209,7 +420,6 @@ impl PdfCraftApp { // Only a page with no text of its own is read. Comments and text boxes over the picture do // not count as its text (a scan with a few comments is still a scan, as in Acrobat), but // a page with real text and a background picture is left alone. - let mut settings = self.ocr_draft.settings(); settings.skip_text_pages = true; settings.native_text_only = true; self.start_ocr_pages(scan.doc, &[scan.page], settings, true); @@ -267,7 +477,9 @@ impl PdfCraftApp { } else { None }; - let Some(job) = self.session.ocr_job(id, pages, settings) else { return }; + let Some(job) = self.session.ocr_job(id, pages, settings.clone()) else { return }; + let guard = job.guard.clone(); + let editable = settings.output_mode == OutputMode::EditableText; let progress = Arc::new(Mutex::new(OcrProgress { total: job.pages.len(), ..Default::default() })); let p = progress.clone(); // A failure is written on the worker thread: in the UI's language. @@ -275,8 +487,8 @@ impl PdfCraftApp { let work = move || { crate::i18n::set_current(lang); let result = recognize_guarded(|| { - pdfcraft_engine::ocr::engine().map(|ocr| { - job.run(&ocr, |done, total| { + pdfcraft_engine::ocr::recognizers(&settings).map(|r| { + job.run(&r, |done, total| { let Ok(mut s) = p.lock() else { return false }; s.done = done; s.total = total; @@ -296,7 +508,7 @@ impl PdfCraftApp { } #[cfg(target_arch = "wasm32")] work(); - self.ocr_run = Some(OcrRun { doc: id, progress, automatic: auto }); + self.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: auto, editable }); self.poll_ocr(); } @@ -307,10 +519,11 @@ impl PdfCraftApp { self.notify_tr("Text recognition is already running"); return; } - // The settings showing now, not when the folder arrives. + // The settings showing now, not when the folder arrives. A remembered folder (the last + // batch's pick, persisted with the preferences) is reused; otherwise the user is asked. let settings = self.ocr_draft.settings(); #[cfg(not(target_arch = "wasm32"))] - match self.export_dir_override.clone() { + match self.export_dir_override.clone().or_else(|| self.ocr_draft.output_folder.clone()) { Some(d) => self.ocr_files_into(files, settings, d.into()), None => { let dialog = rfd::AsyncFileDialog::new().set_title(tl!("Choose a folder for the searchable files").to_string()); @@ -328,21 +541,66 @@ impl PdfCraftApp { self.notify_tr("Text recognition is already running"); return; } + // The folder is remembered (the next batch reuses it; the dialog's Clear un-remembers). + #[cfg(not(target_arch = "wasm32"))] + { + self.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + } + // Two inputs of one file name (from different folders) would write the same output + // (or download under the same name): refused before anything is read. + let twice = pdfcraft_engine::ocr::duplicate_output_names(files.iter().map(|(name, _)| name.as_str())); + if !twice.is_empty() { + self.notify_error(crate::i18n::fmt( + tl!("Several files have the same name, so their results would overwrite each other; nothing was written: {list}"), + &[("list", &twice.join(", "))], + )); + return; + } + // Collision pre-flight: a searchable copy never silently overwrites. Every target is + // checked before the first page is read; a collision stops the batch here. + #[cfg(not(target_arch = "wasm32"))] + { + let hits: Vec = files + .iter() + .filter_map(|(name, _)| { + let target = dir.join(name); + target.exists().then(|| target.to_string_lossy().into_owned()) + }) + .take(5) + .collect(); + if !hits.is_empty() { + self.notify_error(crate::i18n::fmt( + tl!("The output folder already holds files of these names; nothing was written: {list}"), + &[("list", &hits.join(", "))], + )); + return; + } + } + // The engines are built (and named if they cannot be) before anything starts. + let recognizers = match pdfcraft_engine::ocr::recognizers(&settings) { + Ok(r) => r, + Err(why) => { + self.notify_error(why); + return; + } + }; let progress = Arc::new(Mutex::new(BatchProgress { total: files.len(), ..Default::default() })); let p = progress.clone(); // The summary is written on the worker thread: draw it in the UI's language. let lang = crate::i18n::current(); let work = move || { crate::i18n::set_current(lang); - // A panic in the recognizer still ends the batch with a summary (#3919). #[cfg(not(target_arch = "wasm32"))] - let summary = pdfcraft_engine::guard(|| recognize_files(files, &settings, &p, &dir)); + let write = move |name: &str, r: &pdfcraft_engine::ocr::FileResult| { + crate::editing::write_atomically(&dir.join(name).to_string_lossy(), &r.bytes).map_err(|e| e.to_string()) + }; #[cfg(target_arch = "wasm32")] - let summary = pdfcraft_engine::guard(|| recognize_files(files, &settings, &p)); - let summary = summary.unwrap_or_else(|panic| failed(&panic)); + let write = move |name: &str, r: &pdfcraft_engine::ocr::FileResult| crate::editing::download(name, &r.bytes); + // A panic in the recognizer still ends the batch with a summary (#3919). + let msg = pdfcraft_engine::guard(|| run_batch_files(files, settings, &recognizers, &p, write)).unwrap_or_else(|panic| failed(&panic)); if let Ok(mut s) = p.lock() { s.done = s.total; - s.message = Some(summary); + s.message = Some(msg); } }; #[cfg(not(target_arch = "wasm32"))] @@ -356,54 +614,17 @@ impl PdfCraftApp { self.ocr_batch = Some(progress); self.poll_ocr(); } -} -/// Recognize each file and write its searchable copy into `dir`; returns the summary to show. -fn recognize_files( - files: Vec<(String, Vec)>, - settings: &OcrSettings, - p: &Mutex, - #[cfg(not(target_arch = "wasm32"))] dir: &std::path::Path, -) -> String { - let (mut ok, mut words, mut failed) = (0, 0, Vec::new()); - match pdfcraft_engine::ocr::engine() { - Err(e) => failed.push(e), - Ok(ocr) => { - for (i, (name, bytes)) in files.into_iter().enumerate() { - if let Ok(mut s) = p.lock() { - s.done = i; - } - let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), &ocr, |_, _| true); - let saved = r.and_then(|r| { - #[cfg(not(target_arch = "wasm32"))] - crate::editing::write_atomically(&dir.join(&name).to_string_lossy(), &r.bytes).map_err(|e| e.to_string())?; - #[cfg(target_arch = "wasm32")] - crate::editing::download(&name, &r.bytes)?; - Ok(r.words()) - }); - match saved { - Ok(n) => { - ok += 1; - words += n; - } - Err(e) => failed.push(format!("{name}: {}", e)), - } - } + /// Ask the running batch to stop: the file in flight finishes, the rest are counted as not + /// run. + pub fn cancel_ocr_batch(&mut self) { + if let Some(b) = &self.ocr_batch + && let Ok(mut s) = b.lock() + { + s.cancel = true; } } - let done = if ok == 1 { - crate::i18n::fmt(tl!("Recognized {w} words in 1 file"), &[("w", &words.to_string())]) - } else { - crate::i18n::fmt(tl!("Recognized {w} words in {n} files"), &[("w", &words.to_string()), ("n", &ok.to_string())]) - }; - let mut msg = done; - if !failed.is_empty() { - msg.push_str(&crate::i18n::fmt(tl!("; failed: {list}"), &[("list", &failed.join("; "))])); - } - msg -} -impl PdfCraftApp { /// Stop a running recognition (the pages read so far are kept). pub fn cancel_ocr(&mut self) { if let Some(r) = &self.ocr_run @@ -442,7 +663,7 @@ impl PdfCraftApp { } } let Some(run) = self.ocr_run.as_ref() else { return }; - let (doc, progress, auto) = (run.doc, run.progress.clone(), run.automatic); + let (doc, progress, auto, guard, editable) = (run.doc, run.progress.clone(), run.automatic, run.guard.clone(), run.editable); let automatic = auto.is_some(); let mut s = match progress.lock() { Ok(s) => s, @@ -498,9 +719,20 @@ impl PdfCraftApp { return; } }; + // The dialog's "Editable text" choice is honored: the recognition is not written as an + // invisible layer here. It opens in the OCR Verify screen, whose Accept produces the + // editable document (a NEW file) the choice promises. + if editable && !automatic { + self.ocr_verify.load(doc, guard, found); + self.ocr_verify.open = true; + self.notify_tr("Review the recognition, then Accept to write the editable document"); + return; + } let read = found.iter().filter(|p| p.skipped.is_none()).count(); let skipped = found.len() - read; - match self.session.apply_ocr(doc, &found) { + // The guard refuses a result that no longer fits its document (the user edited, or + // redacted, while the worker was out); its reason says so. + match self.session.apply_ocr_checked(doc, &found, &guard) { Ok(words) => { if let Some(info) = self.session.get(doc).map(|d| d.info.clone()) && let Some(view) = self.views.iter_mut().find(|v| v.id == doc) @@ -525,6 +757,12 @@ impl PdfCraftApp { crate::i18n::fmt(tl!("; {n} pages already had text"), &[("n", &skipped.to_string())]) }); } + // The pages' own notes (the editable output's unrenderable-character warning, + // for one) are part of the result: said here, not dropped. + for note in found.iter().flat_map(|p| p.notes.iter()) { + msg.push_str("; "); + msg.push_str(note); + } self.notify(msg); } Err(e) => self.notify_error(e), @@ -532,6 +770,77 @@ impl PdfCraftApp { } } +/// The batch's worker: read each file, hand the searchable copy to `write` (the output folder, +/// or the web's download), count the buckets, and compose the summary. A file the skip-files +/// option passes over is its own bucket and `write` is never called for it. A cancel is asked +/// for between files: the file in flight finishes, the rest are counted as not run. +pub(crate) fn run_batch_files( + files: Vec<(String, Vec)>, + settings: OcrSettings, + recognizers: &pdfcraft_engine::ocr::Recognizers, + progress: &Mutex, + mut write: impl FnMut(&str, &pdfcraft_engine::ocr::FileResult) -> Result<(), String>, +) -> String { + let total = files.len(); + let (mut ok, mut skipped, mut words, mut failed, mut notes) = (0usize, 0usize, 0usize, Vec::new(), Vec::new()); + let mut not_run: Option = None; + for (i, (name, bytes)) in files.into_iter().enumerate() { + if progress.lock().map(|s| s.cancel).unwrap_or(false) { + not_run = Some(total - i); + break; + } + if let Ok(mut s) = progress.lock() { + s.done = i; + } + let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), recognizers, |_, _| true); + match r { + Ok(res) if res.skipped() => skipped += 1, + Ok(res) => match write(&name, &res) { + Ok(()) => { + ok += 1; + words += res.words(); + // The batch's low-confidence note: at or under the settings' threshold (60 by + // default), on 1-based sorted pages. + if let Some((count, pages)) = pdfcraft_engine::ocr::low_confidence_pages(&res.pages, settings.low_confidence_threshold) { + let list = pages.iter().map(|p| p.to_string()).collect::>().join(", "); + notes.push(crate::i18n::fmt( + tl!("{name}: {n} low-confidence word(s) on pages {pages}"), + &[("name", &name), ("n", &count.to_string()), ("pages", &list)], + )); + } + } + Err(e) => failed.push(format!("{name}: {e}")), + }, + Err(e) => failed.push(format!("{name}: {e}")), + } + } + batch_summary(ok, words, skipped, &failed, ¬es, not_run) +} + +/// The batch summary: what was read, the skipped files as their own bucket, the failures, the +/// low-confidence notes and (on a cancel) how many files were not run. +fn batch_summary(ok: usize, words: usize, skipped: usize, failed: &[String], notes: &[String], not_run: Option) -> String { + let mut msg = if ok == 1 { + crate::i18n::fmt(tl!("Recognized {w} words in 1 file"), &[("w", &words.to_string())]) + } else { + crate::i18n::fmt(tl!("Recognized {w} words in {n} files"), &[("w", &words.to_string()), ("n", &ok.to_string())]) + }; + if skipped > 0 { + msg.push_str(&crate::i18n::fmt(tl!("; {n} files already had text"), &[("n", &skipped.to_string())])); + } + if !failed.is_empty() { + msg.push_str(&crate::i18n::fmt(tl!("; failed: {list}"), &[("list", &failed.join("; "))])); + } + if let Some(rest) = not_run { + msg.push_str(&crate::i18n::fmt(tl!("; cancelled — {n} files were not run"), &[("n", &rest.to_string())])); + } + for note in notes { + msg.push_str("; "); + msg.push_str(note); + } + msg +} + /// "Recognize text failed: {e}", in the current language. fn failed(e: &str) -> String { crate::i18n::fmt(tl!("{label} failed: {e}"), &[("label", tl!("Recognize text")), ("e", e)]) @@ -547,6 +856,7 @@ fn recognize_guarded(recognize: impl FnOnce() -> Result) -> Result mod tests { use super::*; use crate::{Dialog, LeftPanel, Mode, QuickTool}; + use pdfcraft_engine::ocr::{Recognizer, Recognizers}; use pdfcraft_engine::{Edit, Session}; /// A one-page PDF that is one picture covering the page. @@ -629,12 +939,26 @@ mod tests { /// asked for is applied at once. The result is made up, so no OCR models are needed. #[test] fn the_result_of_an_automatic_run_waits_for_read_mode_and_dialogs() { - let word = || pdfcraft_engine::ocr::PlacedWord { text: "scanned".into(), origin: [72.0, 700.0], across: [60.0, 0.0], up: [0.0, 12.0] }; + let word = || pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: None, + source: "ocrs".into(), + }; let finished = |app: &PdfCraftApp, automatic: bool| { let generation = app.session.get(app.active_ids().unwrap().1).unwrap().edit_generation(); - let page = pdfcraft_engine::ocr::OcrPage { page: 0, words: vec![word()], already_on_page: 0, skipped: None }; + let page = pdfcraft_engine::ocr::OcrPage { page: 0, words: vec![word()], already_on_page: 0, skipped: None, notes: Vec::new() }; let progress = Arc::new(Mutex::new(OcrProgress { done: 1, total: 1, result: Some(Ok(vec![page])), cancel: false })); - OcrRun { doc: app.active_ids().unwrap().1, progress, automatic: automatic.then_some(AutoOcr { page: 0, generation }) } + let guard = app.session.ocr_guard(app.active_ids().unwrap().1).unwrap(); + OcrRun { + doc: app.active_ids().unwrap().1, + progress, + guard, + automatic: automatic.then_some(AutoOcr { page: 0, generation }), + editable: false, + } }; let label = |app: &PdfCraftApp| app.session.get(app.active_ids().unwrap().1).unwrap().can_undo().map(String::from); @@ -669,12 +993,21 @@ mod tests { let generation = app.session.get(id).unwrap().edit_generation(); let page = pdfcraft_engine::ocr::OcrPage { page: 0, - words: vec![pdfcraft_engine::ocr::PlacedWord { text: "scanned".into(), origin: [72.0, 700.0], across: [60.0, 0.0], up: [0.0, 12.0] }], + words: vec![pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: None, + source: "ocrs".into(), + }], already_on_page: 0, skipped: None, + notes: Vec::new(), }; let progress = Arc::new(Mutex::new(OcrProgress { done: 1, total: 1, result: Some(Ok(vec![page])), cancel: false })); - app.ocr_run = Some(OcrRun { doc: id, progress, automatic: Some(AutoOcr { page: 0, generation }) }); + let guard = app.session.ocr_guard(id).unwrap(); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: Some(AutoOcr { page: 0, generation }), editable: false }); app.views[0].auto_ocr_tried.insert(0); // The document changes while the run is out. let mut png = std::io::Cursor::new(Vec::new()); @@ -712,9 +1045,13 @@ mod tests { fn a_run_whose_worker_died_ends_instead_of_waiting_forever() { let mut app = PdfCraftApp::new(); app.set_option("language", "en").unwrap(); + let pdf = pdfcraft_engine::Session::new().create_from_text("t", "guard fixture").unwrap(); + app.open_bytes("a.pdf", None, pdf.to_vec()).unwrap(); + let id = app.active_ids().unwrap().1; + let guard = app.session.ocr_guard(id).unwrap(); let progress = Arc::new(Mutex::new(OcrProgress { total: 3, ..Default::default() })); poison(&progress); - app.ocr_run = Some(OcrRun { doc: DocId(1), progress, automatic: None }); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); app.poll_ocr(); assert!(app.ocr_run.is_none(), "the next run isn't refused as already running"); let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); @@ -728,4 +1065,267 @@ mod tests { let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); assert!(toast.starts_with("Recognize text failed"), "{toast}"); } + + /// A fake engine: one confident word per call, no filesystem, no models needed. + struct Fake { + confidence: f32, + } + + impl Recognizer for Fake { + fn id(&self) -> &'static str { + "fake" + } + fn available(&self) -> bool { + true + } + fn languages(&self) -> Vec<(String, String)> { + vec![("en".into(), "English".into())] + } + fn recognize( + &self, + _image: &pdfcraft_ocr::OcrImage, + _options: &pdfcraft_ocr::RecognizeOptions, + ) -> Result, pdfcraft_engine::ocr::OcrError> { + Ok(vec![pdfcraft_ocr::Line { + words: vec![pdfcraft_ocr::Word::new("word", [1.0, 1.0, 9.0, 9.0], "fake").with_confidence(self.confidence)], + }]) + } + } + + /// A batch over two text files (the engine never runs on them: they are skipped files): + /// the summary counts them in their own bucket, nothing is written, and a cancel before + /// the first file leaves the rest not run. + #[test] + fn batch_counts_skipped_files_and_honours_a_cancel() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "Already typed here").expect("fixture"); + let files = vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), cancel: true, ..Default::default() }); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 95.0 }), secondary: None }; + let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let w = writes.clone(); + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + assert_eq!(writes.load(std::sync::atomic::Ordering::SeqCst), 0, "nothing is written for skipped files, and the cancel came first"); + assert!(msg.contains("cancelled — 2 files were not run"), "{msg}"); + // Without the cancel, the skipped bucket is named and still nothing is written. + let files = vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let w = writes.clone(); + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + assert_eq!(writes.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(msg.contains("; 2 files already had text"), "{msg}"); + assert!(msg.contains("Recognized 0 words in 0 files"), "{msg}"); + } + + /// A batch over scanned files: the words are counted, and a low-confidence reading lands + /// in the summary as a note with its 1-based sorted pages. + #[test] + fn batch_notes_low_confidence_pages() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "fake scan").expect("fixture"); + let files = vec![("scan.pdf".into(), pdf.to_vec()), ("clean.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 40.0 }), secondary: None }; + // The fixtures are text PDFs standing in for scans; the default skip-text-pages option + // would (correctly) leave them alone, so the batch here reads everything. + let settings = OcrSettings { skip_text_pages: false, ..Default::default() }; + let msg = run_batch_files(files, settings.clone(), &recognizers, &progress, |_, _| Ok(())); + assert!(msg.contains("Recognized 2 words in 2 files"), "{msg}"); + assert!(msg.contains("scan.pdf: 1 low-confidence word(s) on pages 1"), "{msg}"); + // The write failing is a per-file failure, named in the summary. + let files = vec![("scan.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let msg = run_batch_files(files, settings, &recognizers, &progress, |_, _| Err("disk full".into())); + assert!(msg.contains("; failed: scan.pdf: disk full"), "{msg}"); + } + + /// The batch note follows the settings' low-confidence threshold (60 by default), not a + /// number of its own: a 40% word is flagged under 60 and under 40, and not under 30. + #[test] + fn the_batch_note_uses_the_settings_threshold() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "fake scan").expect("fixture"); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 40.0 }), secondary: None }; + let run = |threshold: f32| { + let files = vec![("scan.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let settings = OcrSettings { skip_text_pages: false, low_confidence_threshold: threshold, ..Default::default() }; + run_batch_files(files, settings, &recognizers, &progress, |_, _| Ok(())) + }; + assert!(run(OcrSettings::default().low_confidence_threshold).contains("1 low-confidence word(s)"), "the default flags a 40% word"); + assert!(run(40.0).contains("1 low-confidence word(s)"), "at the threshold counts"); + assert!(!run(30.0).contains("low-confidence"), "a lower threshold leaves the word alone: {}", run(30.0)); + } + + /// The remembered output folder survives the preferences round trip; an absent or empty + /// value keeps the default (ask each time). + #[test] + fn the_output_folder_is_persisted_with_the_draft() { + let mut d = OcrDraft::default(); + assert_eq!(d.output_folder, None, "by default the folder is asked for"); + d.output_folder = Some("out".into()); + let mut r = OcrDraft::default(); + r.apply_pref(&d.to_pref()); + assert_eq!(r.output_folder.as_deref(), Some("out")); + let mut e = OcrDraft::default(); + e.apply_pref(&serde_json::json!({})); + assert_eq!(e.output_folder, None, "an absent value keeps the default"); + e.output_folder = Some("kept".into()); + e.apply_pref(&serde_json::json!({ "output_folder": "" })); + assert_eq!(e.output_folder.as_deref(), Some("kept"), "an empty value is not a folder"); + } + + /// "Automatic" saved without a strategy reads the way Automatic reads by default (the + /// second engine joins when the first is unsure); an explicit choice always stands. + #[test] + fn an_automatic_engine_without_a_saved_strategy_is_confidence_weighted() { + let mut d = OcrDraft::default(); + d.apply_pref(&serde_json::json!({ "engine": "auto" })); + assert_eq!((d.engine, d.strategy), (EngineChoice::Auto, MergeStrategy::ConfidenceWeighted)); + let mut named = OcrDraft::default(); + named.apply_pref(&serde_json::json!({ "engine": "tesseract" })); + assert_eq!(named.strategy, MergeStrategy::PrimaryOnly, "a named engine reads alone"); + let mut explicit = OcrDraft::default(); + explicit.apply_pref(&serde_json::json!({ "engine": "auto", "strategy": "primary" })); + assert_eq!((explicit.engine, explicit.strategy), (EngineChoice::Auto, MergeStrategy::PrimaryOnly)); + } + + /// Two inputs of one file name (from different folders, or differing only in case) would + /// write the same output: the batch stops before anything is read or written. + #[test] + fn inputs_that_share_a_file_name_are_refused_before_anything_is_read() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-same-name-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + app.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "scanned words here").expect("fixture"); + app.ocr_files(vec![("scan.pdf".into(), pdf.to_vec()), ("other.pdf".into(), pdf.to_vec()), ("Scan.PDF".into(), pdf.to_vec())]); + assert!(app.ocr_batch.is_none(), "the batch never starts"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("same name") && toast.contains("Scan.PDF"), "{toast}"); + assert_eq!(std::fs::read_dir(&dir).expect("fixture").count(), 0, "nothing was written"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A remembered output folder is used without asking, is remembered again after the run, + /// and its collision pre-flight still stops the batch before the first page is read. + #[test] + fn a_remembered_output_folder_is_used_and_still_preflights_collisions() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-folder-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + std::fs::write(dir.join("a.pdf"), b"would be overwritten").expect("fixture"); + app.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "scanned words here").expect("fixture"); + app.ocr_files(vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]); + assert!(app.ocr_batch.is_none(), "the collision stops the batch before anything is read"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("already holds files of these names"), "{toast}"); + assert!(toast.contains("a.pdf"), "the colliding names are in the error: {toast}"); + assert_eq!(app.ocr_draft.output_folder.as_deref(), Some(dir.to_string_lossy().as_ref()), "the folder stays remembered for the next batch"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A result that arrives after the document changed (here: a page turned while the worker was + /// reading; the same applies to a redaction applied meanwhile) is refused with the reason and + /// writes nothing, so words read before a redaction cannot come back as invisible text. + #[test] + fn a_result_that_arrives_after_the_document_changed_is_refused() { + let mut app = PdfCraftApp::new(); + let pdf = pdfcraft_engine::Session::new().create_from_text("t", "Typed text").expect("fixture"); + app.open_bytes("a.pdf", None, pdf.to_vec()).expect("opens"); + let id = app.active_ids().expect("a document").1; + let guard = app.session.ocr_guard(id).expect("a guard"); + app.session.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an edit while the worker runs"); + let after_edit = app.session.get(id).and_then(|d| d.can_undo().map(str::to_owned)); + let word = pdfcraft_engine::ocr::PlacedWord { + text: "123-45-6789".into(), + origin: [40.0, 40.0], + across: [50.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "fake".into(), + }; + let page = OcrPage { page: 0, words: vec![word], already_on_page: 0, skipped: None, notes: Vec::new() }; + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); + app.poll_ocr(); + assert!(app.ocr_run.is_none(), "the run is over"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("changed since recognition started"), "{toast}"); + assert_eq!( + app.session.get(id).and_then(|d| d.can_undo().map(str::to_owned)), + after_edit, + "nothing was written: the last undoable step is still the edit" + ); + } + + /// A page's notes survive to the result the user sees: the editable output's + /// unrenderable-character warning ("N characters had no font … appear as ?") is said in + /// the run's summary, not dropped. The result is fabricated, so no models are needed. + #[test] + fn the_result_summary_names_the_substituted_characters() { + let mut app = app_with(scan()); + let id = app.active_ids().unwrap().1; + let word = pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + let page = pdfcraft_engine::ocr::OcrPage { + page: 0, + words: vec![word], + already_on_page: 0, + skipped: None, + notes: vec!["3 characters had no font that could show them and appear as ? in the output".into()], + }; + let guard = app.session.ocr_guard(id).unwrap(); + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); + app.poll_ocr(); + let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("3 characters had no font"), "{toast}"); + } + + /// The dialog's "Editable text" output is honored: a run that promised an editable document + /// does not quietly write the invisible searchable layer — its result opens in the OCR + /// Verify screen, whose Accept produces the editable document. The result is fabricated, + /// so no models are needed. + #[test] + fn an_editable_output_run_opens_the_review_instead_of_applying() { + let mut app = app_with(scan()); + app.run_inline = true; + let id = app.active_ids().unwrap().1; + let word = pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + let page = OcrPage { page: 0, words: vec![word], already_on_page: 0, skipped: None, notes: Vec::new() }; + let guard = app.session.ocr_guard(id).unwrap(); + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_draft.output_mode = pdfcraft_engine::ocr::OutputMode::EditableText; + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: true }); + app.poll_ocr(); + assert!(app.ocr_run.is_none(), "the run is over"); + assert_eq!(app.session.get(id).unwrap().can_undo(), None, "no invisible layer was written"); + assert!(app.ocr_verify.open, "the review opens, to accept as an editable document"); + assert_eq!(app.ocr_verify.phase, crate::workflows::ocr_verify::Phase::ReviewReady); + } } diff --git a/crates/ui-egui/src/workflows/ocr_verify.rs b/crates/ui-egui/src/workflows/ocr_verify.rs new file mode 100644 index 0000000000..4054339de4 --- /dev/null +++ b/crates/ui-egui/src/workflows/ocr_verify.rs @@ -0,0 +1,2830 @@ +//! Scan & OCR ▸ Correct recognized text: the OCR Verify screen. A full-workspace surface with +//! four panes (page list, source page, recognized text, zoom + word inspector) where a +//! recognition is reviewed before it is applied: words are colored by confidence, corrected, +//! removed (and restored), sent to a per-language user dictionary, and navigated by their +//! uncertainty. Accept applies the review — as one undoable step on the open document +//! (searchable), or as a new `_ocr.pdf` of visible text (editable). Reject throws the +//! reading away. +//! +//! The state machine is [`Phase`]: recognition runs on a worker thread ([`Phase::Running`], a +//! real Cancel keeps the pages already read), results land in [`Phase::ReviewReady`], failures +//! land in [`Phase::RecoverableError`] with the cause named and Run enabled again, and +//! Accept/Reject/Re-OCR are ignored outside ReviewReady instead of acting on nothing. + +use std::collections::BTreeSet; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use pdfcraft_engine::ocr::{ + ConfidenceBand, EngineChoice, MergeStrategy, OcrGuard, OcrPage, OcrSettings, OutputMode, PlacedWord, band_for, clamp_region, +}; +use pdfcraft_engine::{DocId, Session}; + +use super::ocr_ui; +use crate::theme::{self, Tokens}; +use crate::{PdfCraftApp, widgets}; + +/// Where the screen stands. Run is enabled in Idle, ReviewReady and RecoverableError; the +/// review tools (Accept, Reject, navigation, the inspector, re-OCR) only in ReviewReady. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Phase { + #[default] + Idle, + Running, + ReviewReady, + Saving, + RecoverableError, +} + +/// The four panes; Ctrl+Tab moves the focus ring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Pane { + #[default] + Pages, + Source, + Text, + Inspector, +} + +impl Pane { + fn next(self) -> Pane { + match self { + Pane::Pages => Pane::Source, + Pane::Source => Pane::Text, + Pane::Text => Pane::Inspector, + Pane::Inspector => Pane::Pages, + } + } +} + +/// One review change, so it can be undone within the panel (review edits are not document +/// edits; they live until the review is applied or rejected). +#[derive(Clone, Debug)] +pub enum Op { + /// The correction text field committed (an empty commit removes instead). + Corrected { + key: (usize, usize), + before: String, + after: String, + }, + Removed { + key: (usize, usize), + }, + Restored { + key: (usize, usize), + }, + WordVerified { + key: (usize, usize), + on: bool, + }, + Dictionary { + word: String, + }, + /// A re-read (a region or the whole page) replaced words on the page. Review marks are + /// keyed by word position, so the page's marks as they were are kept for the undo. + Region { + page: usize, + before: Vec, + keys: PageKeys, + }, + PageVerified { + page: usize, + on: bool, + }, + /// "Reject entire page": the page's words were cleared (and its review marks with them). + PageCleared { + page: usize, + before: Vec, + keys: PageKeys, + }, +} + +/// One page's positional review marks (word indices), saved when its word list is replaced. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PageKeys { + pub removed: Vec, + pub verified: Vec, +} + +/// The review state: which words are struck out, which are vouched for, which are in the +/// per-language dictionary, and the undo history. Never persisted. +#[derive(Clone, Debug, Default)] +pub struct Review { + pub removed: BTreeSet<(usize, usize)>, + pub verified: BTreeSet<(usize, usize)>, + pub verified_pages: BTreeSet, + pub dictionary: BTreeSet, + /// Pages whose words were already applied to the document ("Accept entire page"); a later + /// Accept of the whole review never writes them twice. + pub applied_pages: BTreeSet, + pub history: Vec, +} + +/// A word that asks for review: its confidence is in the Low band and the reviewer has not +/// dealt with it — it is not removed, not corrected into the dictionary, not marked verified, +/// and not on a page marked verified. A corrected word stays uncertain by its ORIGINAL +/// confidence: the reviewer changed the text, not the engine's confidence in it. +pub fn is_uncertain(page: &OcrPage, key: (usize, usize), review: &Review) -> bool { + let (_, wi) = key; + if review.removed.contains(&key) || review.verified.contains(&key) || review.verified_pages.contains(&key.0) { + return false; + } + let Some(w) = page.words.get(wi) else { return false }; + if w.text.trim().is_empty() || review.dictionary.contains(&w.text.to_lowercase()) { + return false; + } + pdfcraft_engine::ocr::is_suspect(w.confidence) +} + +/// Candidate corrections for `text`: dictionary words and the page's own words within edit +/// distance 2, nearest first, at most 5, never the word itself. +pub fn suggestions(text: &str, dictionary: &BTreeSet, page_words: &[PlacedWord]) -> Vec { + let lower = text.to_lowercase(); + let mut out: Vec<(usize, String)> = Vec::new(); + let consider = |candidate: &str, out: &mut Vec<(usize, String)>| { + let c = candidate.to_lowercase(); + if c.is_empty() || c == lower { + return; + } + if let Some(d) = edit_distance_within_2(&lower, &c, 2) { + out.push((d, candidate.to_owned())); + } + }; + for w in dictionary { + consider(w, &mut out); + } + for w in page_words { + consider(&w.text, &mut out); + } + // Nearest first; ties break alphabetically; at most five. + out.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + out.truncate(5); + out.into_iter().map(|(_, s)| s).collect() +} + +/// Levenshtein distance between `a` and `b`, abandoned as soon as it is sure to exceed `max` +/// (small bounded DP over char boundaries; `None` means "farther than `max`"). +pub fn edit_distance_within_2(a: &str, b: &str, max: usize) -> Option { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + if a.len().abs_diff(b.len()) > max { + return None; + } + let mut prev: Vec = (0..=b.len()).collect(); + let mut cur = vec![0usize; b.len() + 1]; + for i in 1..=a.len() { + cur[0] = i; + let mut row_min = cur[0]; + for j in 1..=b.len() { + let cost = usize::from(a[i - 1] != b[j - 1]); + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + cost); + row_min = row_min.min(cur[j]); + } + if row_min > max { + return None; + } + std::mem::swap(&mut prev, &mut cur); + } + (prev[b.len()] <= max).then_some(prev[b.len()]) +} + +/// The page preview render: cached until the document or page changes. +pub struct Preview { + pub doc: DocId, + pub page: usize, + /// Pixels per point the page was rendered at. + pub scale: f32, + pub width: u32, + pub height: u32, + pub(crate) texture: Option, +} + +/// A page or region re-read in flight ([`OcrVerify::start_reread`]): the page being read +/// again (and the drawn region, in view-space points, for a region re-read), with the +/// worker's progress. +pub struct Reread { + pub page: usize, + pub region: Option<[f32; 4]>, + pub(crate) progress: Arc>, +} + +/// A re-read's progress: the replacement words once the worker is done, and a cancel request. +/// A cancelled re-read is discarded when it lands: the page's words stay as they were. +#[derive(Default)] +pub struct RereadProgress { + pub cancel: bool, + pub(crate) result: Option, String>>, +} + +/// The OCR Verify screen. One instance per app; lives across frames. +pub struct OcrVerify { + pub open: bool, + pub phase: Phase, + /// The document being reviewed; `None` until a run starts. + pub doc: Option, + /// The recognition results, in page order, as read (review edits mutate them). + pub pages: Vec, + /// What applying the results still expects of the document. + pub guard: Option, + pub review: Review, + /// The selected word (page, index) and the page the source pane shows. + pub selected: Option<(usize, usize)>, + pub current: usize, + /// A region drawn on the source pane, in view-space points on the shown page, snapped + /// outward to whole pixels when it is used ([`Self::start_reread`]). + pub region: Option<[f32; 4]>, + /// Where a source-pane drag started (the live region rectangle is drawn from here). + pub(crate) drag: Option, + pub focus: Pane, + /// The status line: where the run stands, why something was refused. + pub message: String, + cancelled: bool, + pub progress: Option>>, + /// A page or region re-read running on a worker thread: the screen stays live (the review + /// edits and navigation work) and the reading is applied by [`Self::poll_reread`]. + pub reread: Option, + /// The correction text field's current contents. + pub correction: String, + /// Where the per-language dictionary files live (`None`: off, e.g. the web build). + pub dictionary_dir: Option, + /// Language of the captured recognition, independent of the interface locale. + pub dictionary_language: String, + pub(crate) preview: Option, +} + +fn dictionary_path(dir: &std::path::Path, language: &str) -> Result { + if language.is_empty() || language.len() > 32 || !language.bytes().all(|b| b.is_ascii_alphabetic() || b == b'-') { + return Err("invalid OCR dictionary language".into()); + } + Ok(dir.join(format!("ocr-dictionary-{language}.txt"))) +} + +fn read_dictionary(path: &std::path::Path) -> Result { + use std::io::Read as _; + let mut text = String::new(); + match std::fs::File::open(path) { + Ok(file) => { + file.take((pdfcraft_engine::ocr::dictionary::MAX_BYTES + 1) as u64).read_to_string(&mut text).map_err(|e| e.to_string())?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.to_string()), + } + Ok(text) +} + +impl Default for OcrVerify { + fn default() -> Self { + OcrVerify { + open: false, + phase: Phase::Idle, + doc: None, + pages: Vec::new(), + guard: None, + review: Review::default(), + selected: None, + current: 0, + region: None, + drag: None, + focus: Pane::Pages, + message: String::new(), + cancelled: false, + progress: None, + reread: None, + correction: String::new(), + dictionary_dir: None, + dictionary_language: "en".into(), + preview: None, + } + } +} + +impl OcrVerify { + /// Begin a review of `found` for `doc`: ReviewReady even when nothing was found (the + /// status then says so), with the guard remembered for Accept. + pub fn load(&mut self, doc: DocId, guard: OcrGuard, found: Vec) { + self.doc = Some(doc); + self.guard = Some(guard); + self.pages = found; + self.review = Review::default(); + self.selected = None; + self.region = None; + self.current = self.current.min(self.pages.len().saturating_sub(1)); + self.phase = Phase::ReviewReady; + self.load_dictionary_pref(); + } + + /// The count of live (not removed, not empty) words across the review. + pub fn word_count(&self) -> usize { + self.pages + .iter() + .enumerate() + .map(|(p, pg)| pg.words.iter().enumerate().filter(|(i, w)| !w.text.trim().is_empty() && !self.review.removed.contains(&(p, *i))).count()) + .sum() + } + + /// The words that ask for review, in reading order across pages. + pub fn uncertain(&self) -> Vec<(usize, usize)> { + self.pages + .iter() + .flat_map(|pg| pg.suspects().into_iter().map(move |i| (pg.page, i))) + .filter(|k| self.page(k.0).is_some_and(|p| is_uncertain(p, *k, &self.review))) + .collect() + } + + /// The count of Low-band live words (the info strip's LOW-CONFIDENCE WORDS). + pub fn suspect_count(&self) -> usize { + self.pages + .iter() + .enumerate() + .map(|(p, pg)| { + pg.words + .iter() + .enumerate() + .filter(|(i, w)| { + !self.review.removed.contains(&(p, *i)) + && !w.text.trim().is_empty() + && !self.review.dictionary.contains(&w.text.to_lowercase()) + && pdfcraft_engine::ocr::is_suspect(w.confidence) + }) + .count() + }) + .sum() + } + + /// The share of words the reviewer vouched for: marked verified, or on a page marked + /// verified (0.0 with no words). + pub fn verified_fraction(&self) -> f32 { + let (mut yes, mut all) = (0usize, 0usize); + for (p, pg) in self.pages.iter().enumerate() { + let page_done = self.review.verified_pages.contains(&pg.page); + for (i, w) in pg.words.iter().enumerate() { + if w.text.trim().is_empty() || self.review.removed.contains(&(p, i)) { + continue; + } + all += 1; + if page_done || self.review.verified.contains(&(p, i)) { + yes += 1; + } + } + } + if all == 0 { 0.0 } else { yes as f32 / all as f32 } + } + + pub fn page(&self, page: usize) -> Option<&OcrPage> { + self.pages.iter().find(|p| p.page == page) + } + + pub fn word(&self, key: (usize, usize)) -> Option<&PlacedWord> { + self.page(key.0).and_then(|p| p.words.get(key.1)) + } + + pub fn is_removed(&self, key: (usize, usize)) -> bool { + self.review.removed.contains(&key) + } + + pub fn is_verified(&self, key: (usize, usize)) -> bool { + self.review.verified.contains(&key) || self.review.verified_pages.contains(&key.0) + } + + /// The word's text with any correction applied. + pub fn text_of(&self, key: (usize, usize)) -> Option<&str> { + self.word(key).map(|w| w.text.as_str()) + } + + fn in_review(&self) -> bool { + self.phase == Phase::ReviewReady + } + + /// Commit the correction field for `key`. An empty (after trim) commit removes the word; + /// the box never moves. Only in ReviewReady. + pub fn correct(&mut self, key: (usize, usize), text: &str) -> bool { + if !self.in_review() || self.word(key).is_none() { + return false; + } + if text.trim().is_empty() { + return self.set_removed(key, true); + } + let Some(w) = self.page_mut(key.0).and_then(|p| p.words.get_mut(key.1)) else { return false }; + let before = w.text.clone(); + if before == text { + return false; + } + w.text = text.to_string(); + self.review.history.push(Op::Corrected { key, before, after: text.to_string() }); + true + } + + /// Strike a word out (or restore it). Removed words stay in place — reviewable, restorable, + /// and never written by Accept. + pub fn set_removed(&mut self, key: (usize, usize), removed: bool) -> bool { + if !self.in_review() || self.word(key).is_none() || self.is_removed(key) == removed { + return false; + } + if removed { + self.review.removed.insert(key); + self.review.history.push(Op::Removed { key }); + } else { + self.review.removed.remove(&key); + self.review.history.push(Op::Restored { key }); + } + true + } + + /// Vouch for one word. + pub fn set_word_verified(&mut self, key: (usize, usize), on: bool) -> bool { + if !self.in_review() || self.word(key).is_none() || self.review.verified.contains(&key) == on { + return false; + } + if on { + self.review.verified.insert(key); + } else { + self.review.verified.remove(&key); + } + self.review.history.push(Op::WordVerified { key, on }); + true + } + + /// Mark the page reviewed (the toolbar's Verified toggle, Ctrl+T). + pub fn set_page_verified(&mut self, page: usize, on: bool) -> bool { + if !self.in_review() || self.page(page).is_none() || self.review.verified_pages.contains(&page) == on { + return false; + } + if on { + self.review.verified_pages.insert(page); + } else { + self.review.verified_pages.remove(&page); + } + self.review.history.push(Op::PageVerified { page, on }); + true + } + + /// Add a word to the per-language dictionary (case-insensitive, duplicate-free) and + /// remember it in the dictionary file when one can be written. Privacy: that file + /// (`ocr-dictionary-.txt` in the preferences folder) is plain text outside any + /// document, so a word added from a document stays on disk after the document is closed, + /// until it is removed (the undo removes it again). + pub fn add_to_dictionary(&mut self, word: &str, language: &str) -> bool { + if !self.in_review() { + return false; + } + if language != self.dictionary_language { + self.message = "The dictionary language differs from the captured recognition".into(); + return false; + } + let lower = word.trim().to_lowercase(); + if self.review.dictionary.contains(&lower) { + return false; + } + match self.updated_dictionary(&[word], &[]) { + Ok((entries, changed)) => { + self.review.dictionary = entries; + if !changed { + return false; + } + self.review.history.push(Op::Dictionary { word: lower }); + true + } + Err(why) => { + self.message = format!("OCR dictionary: {why}"); + false + } + } + } + + /// Validate the complete file and persist atomically before changing review state. + fn updated_dictionary(&self, add: &[&str], remove: &[&str]) -> Result<(BTreeSet, bool), String> { + use pdfcraft_engine::ocr::dictionary; + let path = self.dictionary_dir.as_ref().map(|dir| dictionary_path(dir, &self.dictionary_language)).transpose()?; + let current = + if let Some(path) = &path { read_dictionary(path)? } else { self.review.dictionary.iter().cloned().collect::>().join("\n") }; + let before = dictionary::words(¤t)?; + let text = dictionary::update(¤t, add, remove)?; + let entries = dictionary::words(&text)?; + let changed = before != entries; + if let Some(path) = path.filter(|_| changed) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + crate::editing::write_atomically(&path.to_string_lossy(), text.as_bytes()).map_err(|e| e.to_string())?; + } + Ok((entries, changed)) + } + + /// Read the captured recognition language's dictionary; errors preserve its live state. + pub fn load_dictionary_pref(&mut self) { + let Some(dir) = &self.dictionary_dir else { return }; + let path = match dictionary_path(dir, &self.dictionary_language) { + Ok(path) => path, + Err(why) => { + self.message = why; + return; + } + }; + match read_dictionary(&path).and_then(|text| pdfcraft_engine::ocr::dictionary::words(&text)) { + Ok(words) => self.review.dictionary = words, + Err(why) => self.message = format!("OCR dictionary: {why}"), + } + } + + /// Undo the last review change (Ctrl+Z / the inspector's Undo). + pub fn undo(&mut self) -> bool { + if !self.in_review() { + return false; + } + let Some(op) = self.review.history.pop() else { return false }; + match op { + Op::Corrected { key, before, .. } => { + if let Some(w) = self.page_mut(key.0).and_then(|p| p.words.get_mut(key.1)) { + w.text = before; + } + } + Op::Removed { key } => { + self.review.removed.remove(&key); + } + Op::Restored { key } => { + self.review.removed.insert(key); + } + Op::WordVerified { key, on } => { + if on { + self.review.verified.remove(&key); + } else { + self.review.verified.insert(key); + } + } + Op::Dictionary { word } => match self.updated_dictionary(&[], &[&word]) { + Ok((entries, _)) => self.review.dictionary = entries, + Err(why) => { + self.message = format!("OCR dictionary: {why}"); + self.review.history.push(Op::Dictionary { word }); + return false; + } + }, + Op::Region { page, before, keys } => { + if let Some(p) = self.page_mut(page) { + p.words = before; + } + self.restore_page_keys(page, keys); + } + Op::PageVerified { page, on } => { + if on { + self.review.verified_pages.remove(&page); + } else { + self.review.verified_pages.insert(page); + } + } + Op::PageCleared { page, before, keys } => { + if let Some(p) = self.page_mut(page) { + p.words = before; + } + self.restore_page_keys(page, keys); + } + } + true + } + + fn reread_settings(&self, draft: &super::ocr_ui::OcrDraft) -> OcrSettings { + let mut settings = draft.settings(); + settings.language = self.dictionary_language.clone(); + settings + } + + /// Whether this word's text was corrected in the review (the inspector says so, and the + /// source pane outlines it). + pub fn is_corrected(&self, key: (usize, usize)) -> bool { + // Newest first: a correction recorded before the page's words were replaced (a re-read, + // a cleared page) names a position in the OLD word list, not this word. + for op in self.review.history.iter().rev() { + match op { + Op::Corrected { key: k, .. } if *k == key => return true, + Op::Region { page, .. } | Op::PageCleared { page, .. } if *page == key.0 => return false, + _ => {} + } + } + false + } + + /// Take `page`'s positional review marks out of the review (its word list is about to be + /// replaced), returning them for the undo. + fn take_page_keys(&mut self, page: usize) -> PageKeys { + let take = |set: &mut BTreeSet<(usize, usize)>| { + let on_page: Vec = set.iter().filter(|k| k.0 == page).map(|k| k.1).collect(); + set.retain(|k| k.0 != page); + on_page + }; + PageKeys { removed: take(&mut self.review.removed), verified: take(&mut self.review.verified) } + } + + /// Put back `page`'s marks exactly as `keys` holds them (the undo of a replacement). + fn restore_page_keys(&mut self, page: usize, keys: PageKeys) { + self.review.removed.retain(|k| k.0 != page); + self.review.verified.retain(|k| k.0 != page); + self.review.removed.extend(keys.removed.into_iter().map(|i| (page, i))); + self.review.verified.extend(keys.verified.into_iter().map(|i| (page, i))); + } + + /// Accept ONE page ("Accept entire page"): the page's live words become one undoable step + /// on the open document, the page is marked applied, and a later whole-review Accept (or a + /// second click on the same page) never writes it again. Ignored outside ReviewReady; zero + /// live words on the page is a typed failure; a stale session is refused with its reason + /// (edits kept). + pub fn accept_page(&mut self, session: &mut Session, page: usize) -> Result { + if !self.in_review() { + return Ok(0); + } + let Some(guard) = self.guard.clone() else { return Ok(0) }; + let Some(doc) = self.doc else { return Ok(0) }; + let Some(pg) = self.live_pages().into_iter().find(|p| p.page == page) else { return Ok(0) }; + if pg.words.is_empty() { + return Err("no words are left to apply on this page".into()); + } + // The page's words are already in the document: never append the OCR layer twice. + // (A page accepted and then rejected still counts: its words were written, and the + // empty-words check above has already spoken for it.) + if self.review.applied_pages.contains(&page) { + return Ok(0); + } + match session.apply_ocr_checked(doc, &[pg], &guard) { + Ok(n) => { + self.review.applied_pages.insert(page); + // This apply is now the document's newest edit: the next page (or the final + // Accept) checks against the document as it stands after it, so only an edit + // made elsewhere in between makes it stale. + self.guard = session.ocr_guard(doc); + Ok(n) + } + Err(why) => { + self.phase = Phase::RecoverableError; + self.message = why.clone(); + Err(why) + } + } + } + + /// Reject ONE page ("Reject entire page"): clear its words (undoable within the panel), + /// drop the selection if it was there. + pub fn reject_page(&mut self, page: usize) -> bool { + if !self.in_review() { + return false; + } + let Some(pg) = self.page_mut(page) else { return false }; + if pg.words.is_empty() { + return false; + } + let before = std::mem::take(&mut pg.words); + let keys = self.take_page_keys(page); + self.review.history.push(Op::PageCleared { page, before, keys }); + if self.selected.is_some_and(|k| k.0 == page) { + self.selected = None; + } + true + } + + /// Replace a page's words wholesale ("Re-OCR entire page"), undoable within the panel. The + /// page's review marks belonged to the old words (they are kept by position), so they are + /// cleared with them and come back with the undo. + pub fn replace_page_words(&mut self, page: usize, words: Vec) -> bool { + if !self.in_review() || self.page(page).is_none() { + return false; + } + let Some(pg) = self.page_mut(page) else { return false }; + let before = std::mem::replace(&mut pg.words, words); + let keys = self.take_page_keys(page); + self.review.history.push(Op::Region { page, before, keys }); + if self.selected.is_some_and(|k| k.0 == page) { + self.selected = None; + } + true + } + + fn page_mut(&mut self, page: usize) -> Option<&mut OcrPage> { + self.pages.iter_mut().find(|p| p.page == page) + } + + /// Walk the uncertain words. Wraps around, skips removed words; with none to show it is a + /// no-op. A corrected word that is still Low-band stays in the walk. + pub fn goto_uncertain(&mut self, forward: bool) -> bool { + let list = self.uncertain(); + let Some(target) = (match self.selected { + None if forward => list.first().copied(), + None => list.last().copied(), + Some(sel) => { + let pos = list.iter().position(|k| *k == sel); + match pos { + Some(i) => { + let next = if forward { i + 1 } else { i + list.len() - 1 }; + list.get(next % list.len()).copied() + } + None if forward => list.first().copied(), + None => list.last().copied(), + } + } + }) else { + return false; + }; + self.select(target); + true + } + + /// Select a word and show its page. + pub fn select(&mut self, key: (usize, usize)) { + self.selected = Some(key); + self.current = key.0; + self.correction = self.text_of(key).unwrap_or_default().to_string(); + } + + /// The results Accept would write: the read pages with removed words stripped. + pub fn live_pages(&self) -> Vec { + self.pages + .iter() + .map(|p| OcrPage { + already_on_page: 0, + page: p.page, + skipped: p.skipped.clone(), + notes: p.notes.clone(), + words: p + .words + .iter() + .enumerate() + .filter(|(i, w)| !self.review.removed.contains(&(p.page, *i)) && !w.text.trim().is_empty()) + .map(|(_, w)| w.clone()) + .collect(), + }) + .collect() + } + + /// Accept in searchable mode: apply the reviewed words as ONE undoable step on the open + /// document, but only while the session is still the one that was read. Ignored outside + /// ReviewReady; zero words and a stale session are typed failures (edits kept). + /// Returns the number of words written. + pub fn accept(&mut self, session: &mut Session) -> Result { + if !self.in_review() { + return Ok(0); // ignored, not an error + } + let Some(guard) = self.guard.clone() else { return Ok(0) }; + let Some(doc) = self.doc else { return Ok(0) }; + // Pages accepted page-by-page are already in the document; never write them twice. + let applied = &self.review.applied_pages; + let pages: Vec = self.live_pages().into_iter().filter(|p| !applied.contains(&p.page)).collect(); + let words = pages.iter().map(|p| p.words.len()).sum::(); + if words == 0 { + if !applied.is_empty() { + self.finish(); + return Ok(0); + } + return Err("no words are left to apply: every word was removed or empty".into()); + } + match session.apply_ocr_checked(doc, &pages, &guard) { + Ok(n) => { + self.finish(); + Ok(n) + } + Err(why) => { + self.phase = Phase::RecoverableError; + self.message = why.clone(); + Err(why) + } + } + } + + /// Accept in editable-text mode: the NEW document's bytes and its default file name + /// (`_ocr.pdf`). The open document is not touched; refuses zero words. + pub fn editable_output(&self, session: &Session) -> Result<(Arc>, String), String> { + if !self.in_review() { + return Err("the review is not ready".into()); + } + let Some(doc) = self.doc else { return Err("the review is not ready".into()) }; + // The review can stay open while the document is edited (Exit keeps it): words read before a + // redaction must not be written out as visible text afterwards. + if let Some(why) = self.guard.as_ref().and_then(|g| session.stale_reason(g)) { + return Err(why); + } + let pages = self.live_pages(); + if pages.iter().map(|p| p.words.len()).sum::() == 0 { + return Err("no words are left to write: every word was removed or empty".into()); + } + let bytes = session.editable_ocr_bytes(doc, &pages)?; + let d = session.get(doc).ok_or("the document is no longer open")?; + let base = d.name.strip_suffix(".pdf").unwrap_or(&d.name).to_string(); + Ok((bytes, format!("{base}_ocr.pdf"))) + } + + /// Reject: clear the words, the session, the selection and the flags; back to Idle. + /// Ignored outside ReviewReady. + pub fn reject(&mut self) -> bool { + if !self.in_review() { + return false; + } + *self = OcrVerify { open: self.open, dictionary_dir: self.dictionary_dir.take(), ..Default::default() }; + true + } + + /// The state after a successful accept (or a saved editable copy): the review is done. + pub(crate) fn finish(&mut self) { + *self = OcrVerify { open: self.open, phase: Phase::Idle, doc: self.doc, dictionary_dir: self.dictionary_dir.take(), ..Default::default() }; + } + + /// Ask the running recognition to stop. The pages already read are kept; the rest of the + /// document is not read. + pub fn cancel(&mut self) { + if self.phase == Phase::Running { + self.cancelled = true; + if let Some(p) = &self.progress + && let Ok(mut s) = p.lock() + { + s.cancel = true; + } + } + } + + pub fn can_run(&self) -> bool { + matches!(self.phase, Phase::Idle | Phase::ReviewReady | Phase::RecoverableError) + } + + /// Start re-reading a whole page (`region` `None`) or a drawn region on a WORKER THREAD: + /// the UI thread only builds the job and refuses an empty or out-of-page region BEFORE + /// anything runs — the render, the recognition and the engine probe (a `--list-langs` + /// process for Tesseract) happen off it, with the status line and a Cancel while they run. + /// The reading is applied by [`Self::poll_reread`] when it lands. + pub fn start_reread(&mut self, app: &mut PdfCraftApp, page: usize, region: Option<[f32; 4]>) { + if self.phase != Phase::ReviewReady || self.reread.is_some() { + return; // one re-read at a time, and never under a run + } + let Some(doc) = self.doc else { return }; + let settings = self.reread_settings(&app.ocr_draft); + if let Some(region) = region { + let Some(info) = app.session.get(doc).and_then(|d| d.info.pages.get(page)) else { + app.notify_error("the page is gone"); + return; + }; + // Refuse before running: an empty, inverted or out-of-page rectangle never + // reaches the engine (the words are untouched). + let scale = app.session.ocr_job(doc, &[page], settings.clone()).map_or(1.0, |job| job.render_scale(page)); + let (w, h) = (info.width * scale, info.height * scale); + let px = [region[0] * scale, region[1] * scale, region[2] * scale, region[3] * scale]; + if clamp_region(px, w.max(1.0) as u32, h.max(1.0) as u32).is_none() { + app.notify_error("the selected region is empty or outside the page"); + return; + } + } + let Some(mut job) = app.session.ocr_job(doc, &[page], settings) else { + app.notify_error("the document is no longer open"); + return; + }; + if let Some(region) = region { + let scale = job.render_scale(page); + job.settings.region = Some([region[0] * scale, region[1] * scale, region[2] * scale, region[3] * scale]); + } + let is_region = region.is_some(); + let progress = Arc::new(Mutex::new(RereadProgress::default())); + self.message = if is_region { tl!("Re-reading the region…").to_string() } else { tl!("Re-reading the page…").to_string() }; + let p = progress.clone(); + let work = move || { + let result = pdfcraft_engine::ocr::recognizers(&job.settings).and_then(|r| { + let found = job.run(&r, |_, _| !p.lock().map(|s| s.cancel).unwrap_or(false)); + let words: Vec = found.into_iter().flat_map(|pg| pg.words).collect(); + if words.is_empty() { + Err(if is_region { + "the region re-read found no text; the existing words are unchanged".into() + } else { + "the re-read found no text; the existing words are unchanged".into() + }) + } else { + Ok(words) + } + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if app.run_inline { + work(); + } else { + std::thread::Builder::new().name("pdfcraft-ocr-reread".into()).spawn(work).ok(); + } + #[cfg(target_arch = "wasm32")] + work(); + self.reread = Some(Reread { page, region, progress }); + } + + /// Ask a running re-read to stop and be discarded: when the worker lands, its reading is + /// thrown away and the page's words stay as they were. + pub fn cancel_reread(&mut self) { + if let Some(r) = &self.reread + && let Ok(mut s) = r.progress.lock() + { + s.cancel = true; + } + } + + /// Apply a finished re-read, once the worker is done: a page re-read replaces the page's + /// words, a region re-read the words inside the drawn rectangle. A cancel discards the + /// reading, and so does a refusal — a document changed since the review began no longer + /// matches the page the words were read from. Called every frame while a re-read is out. + pub(crate) fn poll_reread(&mut self, app: &mut PdfCraftApp) { + if self.phase != Phase::ReviewReady { + return; + } + let Some(reread) = &self.reread else { return }; + let (cancelled, result) = match reread.progress.lock() { + Ok(mut s) => (s.cancel, s.result.take()), + Err(_) => return, + }; + let Some(result) = result else { + // Still reading: keep the status line live and ask for another frame. + if let Some(ctx) = &app.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } + return; + }; + let (page, region) = (reread.page, reread.region); + self.reread = None; + if cancelled { + self.message = tl!("The re-read was cancelled; the words are unchanged").to_string(); + return; + } + // The document changed since the review began: the reading may no longer fit the page + // it was read from, so it is refused with the reason and the words stay. + if let Some(why) = self.guard.as_ref().and_then(|g| app.session.stale_reason(g)) { + self.message = why.clone(); + app.notify_error(why); + return; + } + match result { + Err(why) => { + self.message = why.clone(); + app.notify_error(why); + } + Ok(words) => { + let applied = if let Some(region) = region { + self.replace_region_words(&app.session, page, region, words).map(|_| true) + } else if self.replace_page_words(page, words) { + Ok(false) + } else { + Err("the page is gone".to_string()) + }; + match applied { + Ok(is_region) => { + if is_region { + self.message = tl!("The region was read again; its words were replaced (undo in the panel)").to_string(); + app.notify_tr("The region was read again; its words were replaced (undo in the panel)"); + } else { + self.message = tl!("The page was read again; its words were replaced (undo in the panel)").to_string(); + app.notify_tr("The page was read again; its words were replaced (undo in the panel)"); + }; + } + Err(why) => app.notify_error(why), + } + } + } + } + + /// Put `words` (the worker's reading of the drawn rectangle) in place of the words inside + /// it: the rectangle's bbox in user space decides which old words go; the kept words' + /// review marks follow them to their new positions, the replacement is undoable within + /// the panel, and the first NEW word is selected. An empty reading was refused before the + /// run, so `words` is never empty here. + fn replace_region_words(&mut self, session: &Session, page: usize, region_view: [f32; 4], words: Vec) -> Result<(), String> { + let Some(doc) = self.doc else { return Ok(()) }; + let Some(info) = session.get(doc).and_then(|d| d.info.pages.get(page)) else { return Err("the page is gone".into()) }; + // The rectangle in user space (a bbox of its mapped corners) decides which old words + // the new reading replaces. + let (a, b, c, d) = ( + info.view_to_user(region_view[0], region_view[1]), + info.view_to_user(region_view[2], region_view[3]), + info.view_to_user(region_view[0], region_view[3]), + info.view_to_user(region_view[2], region_view[1]), + ); + let xs = [a[0] as f64, b[0] as f64, c[0] as f64, d[0] as f64]; + let ys = [a[1] as f64, b[1] as f64, c[1] as f64, d[1] as f64]; + let bbox = [ + xs.iter().cloned().fold(f64::INFINITY, f64::min), + ys.iter().cloned().fold(f64::INFINITY, f64::min), + xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max), + ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max), + ]; + let Some(pg) = self.page_mut(page) else { return Err("the page is gone".into()) }; + let before = pg.words.clone(); + let center = |w: &PlacedWord| [w.origin[0] + w.across[0] / 2.0 + w.up[0] / 2.0, w.origin[1] + w.across[1] / 2.0 + w.up[1] / 2.0]; + // The old words outside the rectangle stay, in order; `moved` maps each kept word's old + // position to its new one, so its review marks follow it. + let mut moved = std::collections::BTreeMap::new(); + let mut after: Vec = Vec::with_capacity(before.len().saturating_add(words.len())); + for (i, w) in before.iter().enumerate() { + let c = center(w); + if !(c[0] >= bbox[0] && c[0] <= bbox[2] && c[1] >= bbox[1] && c[1] <= bbox[3]) { + moved.insert(i, after.len()); + after.push(w.clone()); + } + } + // The new words follow the kept ones, so the first of them is at the kept words' count; + // selecting it shows the reviewer what the re-read produced. + let first_new = after.len(); + after.extend(words.iter().cloned()); + pg.words = after; + let keys = self.take_page_keys(page); + let remap = |old: &[usize]| old.iter().filter_map(|i| moved.get(i).map(|&j| (page, j))).collect::>(); + self.review.removed.extend(remap(&keys.removed)); + self.review.verified.extend(remap(&keys.verified)); + self.review.history.push(Op::Region { page, before, keys }); + self.select((page, first_new)); + Ok(()) + } + + /// Store a finished run's pages (the worker's result) and move to ReviewReady. A cancel is + /// honest: the pages already read are kept, and the status says the rest was not read. + pub fn finished(&mut self, found: Vec, cancelled: bool) { + let read = found.iter().filter(|p| p.skipped.is_none()).count(); + let found_words: usize = found.iter().map(|p| p.words.len()).sum(); + self.pages = found; + self.review = Review::default(); + self.selected = None; + self.region = None; + self.current = self.current.min(self.pages.len().saturating_sub(1)); + let total = self.pages.len(); + self.message = if cancelled { + crate::i18n::fmt( + tl!("Cancelled — {n} words were read on {m} of {t} pages; the rest were not read"), + &[("n", &found_words.to_string()), ("m", &read.to_string()), ("t", &total.to_string())], + ) + } else if found_words == 0 { + tl!("OCR complete — no text recognized on this page.").to_string() + } else { + crate::i18n::fmt( + tl!("Read {n} words on {m} of {t} pages"), + &[("n", &found_words.to_string()), ("m", &read.to_string()), ("t", &total.to_string())], + ) + }; + self.phase = Phase::ReviewReady; + self.load_dictionary_pref(); + } + + /// A run that failed: the cause is named and Run is enabled again. + pub fn failed(&mut self, why: &str) { + self.phase = Phase::RecoverableError; + self.message = why.to_string(); + self.progress = None; + } + + pub(crate) fn start_run(&mut self, app: &mut PdfCraftApp) { + let Some((_, id)) = app.active_ids() else { return }; + if !self.can_run() || self.phase == Phase::Running { + return; + } + if self.reread.is_some() { + return; // the page re-read in flight replaces these words; one worker at a time + } + if let Some(why) = app.session.get(id).and_then(|d| d.read_only_reason.clone()) { + self.message = why; + self.phase = Phase::RecoverableError; + return; + } + let d = &app.ocr_draft; + let pages: Vec = match d.pages { + super::ocr_ui::OcrPages::All => Vec::new(), + super::ocr_ui::OcrPages::Current => vec![app.active.map_or(0, |i| app.views.get(i).map_or(0, |v| v.current))], + super::ocr_ui::OcrPages::Range => (d.from.saturating_sub(1)..d.to).collect(), + }; + let settings = d.settings(); + let Some(job) = app.session.ocr_job(id, &pages, settings.clone()) else { return }; + self.dictionary_language = job.settings.language.clone(); + let progress = Arc::new(Mutex::new(ocr_ui::OcrProgress { total: job.pages.len(), ..Default::default() })); + self.doc = Some(id); + self.guard = Some(job.guard.clone()); + self.current = 0; + self.cancelled = false; + self.phase = Phase::Running; + self.message = String::new(); + let p = progress.clone(); + let work = move || { + let result = pdfcraft_engine::ocr::recognizers(&settings).map(|r| { + job.run(&r, |done, total| { + let Ok(mut s) = p.lock() else { return false }; + s.done = done; + s.total = total; + !s.cancel + }) + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if app.run_inline { + work(); + } else { + std::thread::Builder::new().name("pdfcraft-ocr-verify".into()).spawn(work).ok(); + } + #[cfg(target_arch = "wasm32")] + work(); + self.progress = Some(progress); + self.poll(app); + } + + /// Take the worker's result, once it is done. + pub(crate) fn poll(&mut self, app: &mut PdfCraftApp) { + if self.phase != Phase::Running { + return; + } + let Some(progress) = self.progress.clone() else { return }; + let result = progress.lock().ok().and_then(|mut s| s.result.take()); + let Some(result) = result else { + // Still running: refresh the status line and ask for another frame. + if let Ok(s) = progress.lock() { + self.message = crate::i18n::fmt( + tl!("Recognizing text… page {d} of {t}"), + &[("d", &(s.done + 1).min(s.total.max(1)).to_string()), ("t", &s.total.max(1).to_string())], + ); + } + if let Some(ctx) = &app.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } + return; + }; + self.progress = None; + match result { + Ok(found) => self.finished(found, self.cancelled), + Err(why) => self.failed(&why), + } + } +} + +/// What a frame's controls asked for, applied after the drawing (the panes read `app` while +/// the controls write it, so nothing mutates through an open borrow). +#[derive(Default)] +struct Intents { + run: bool, + cancel: bool, + cancel_reread: bool, + accept: bool, + reject: bool, + exit: bool, + undo: bool, + next_uncertain: bool, + prev_uncertain: bool, + toggle_page_verified: bool, + reocr_region: bool, + reocr_page: bool, + accept_page: bool, + reject_page: bool, + cycle_focus: bool, + select_page: Option, + select_word: Option<(usize, usize)>, + correct: Option, + set_removed: Option, + set_word_verified: Option, + dictionary_add: bool, +} + +/// Band colours: High green, Medium yellow, Low red (the fill at ~20% opacity, the outline +/// at ~60%; the legend carries the same thresholds as text). +fn band_color(band: ConfidenceBand) -> egui::Color32 { + match band { + ConfidenceBand::High => egui::Color32::from_rgb(34, 139, 74), + ConfidenceBand::Medium => egui::Color32::from_rgb(196, 148, 20), + ConfidenceBand::Low => egui::Color32::from_rgb(204, 61, 47), + } +} + +/// The whole-window OCR Verify screen (Scan & OCR ▸ Correct recognized text). +pub(crate) fn show(app: &mut PdfCraftApp, ui: &mut egui::Ui) { + use egui::{Key, Modifiers}; + let ctx = ui.ctx().clone(); + let t = Tokens::get(&ctx); + let wants_text = ctx.egui_wants_keyboard_input(); + let (next_u, prev_u, toggle_verified, cycle, undo_key) = ctx.input_mut(|i| { + ( + i.consume_key(Modifiers::NONE, Key::F8), + i.consume_key(Modifiers::SHIFT, Key::F8), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::T)), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::Tab)), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::Z)), + ) + }); + + let mut n = Intents { + next_uncertain: next_u, + prev_uncertain: prev_u, + toggle_page_verified: toggle_verified, + cycle_focus: cycle, + undo: undo_key, + ..Default::default() + }; + + egui::Panel::left("ocr-verify-pages").resizable(false).default_size(190.0).frame(egui::Frame::new().fill(t.panel).inner_margin(8.0)).show( + ui, + |ui| { + pane_title(ui, &t, tl!("Pages"), app.ocr_verify.focus == Pane::Pages); + pages_pane(app, ui, &mut n); + }, + ); + + egui::Panel::right("ocr-verify-right").resizable(false).default_size(340.0).frame(egui::Frame::new().fill(t.panel).inner_margin(8.0)).show( + ui, + |ui| { + pane_title(ui, &t, tl!("Recognized text"), app.ocr_verify.focus == Pane::Text); + text_pane(app, ui, &t, &mut n); + ui.add_space(10.0); + pane_title(ui, &t, tl!("Word"), app.ocr_verify.focus == Pane::Inspector); + inspector(app, ui, &t, &mut n); + }, + ); + + egui::CentralPanel::default().frame(egui::Frame::new().fill(t.pasteboard).inner_margin(egui::Margin::same(8))).show(ui, |ui| { + toolbar(app, ui, &mut n); + ui.add_space(4.0); + info_strip(app, ui, &t); + ui.add_space(6.0); + pane_title(ui, &t, tl!("Source page"), app.ocr_verify.focus == Pane::Source); + source_pane(app, ui, &t, &mut n); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.label(egui::RichText::new(&app.ocr_verify.message).small().color(if app.ocr_verify.phase == Phase::RecoverableError { + t.accent + } else { + t.text_muted + })); + }); + }); + + if n.cycle_focus { + app.ocr_verify.focus = app.ocr_verify.focus.next(); + } + apply(app, n); +} + +/// A pane's caption; the focused pane's name is underlined (Ctrl+Tab moves the focus). +fn pane_title(ui: &mut egui::Ui, t: &Tokens, name: &str, focused: bool) { + ui.add_space(2.0); + let font = theme::semibold(11.5); + let w = ui.fonts_mut(|f| f.layout_no_wrap(name.to_owned(), font.clone(), t.text).size().x); + let (rect, _) = ui.allocate_exact_size(egui::vec2(w, 14.0), egui::Sense::hover()); + ui.painter().text(rect.left_center(), egui::Align2::LEFT_CENTER, name.to_uppercase(), font, t.text_muted); + if focused { + let r = egui::Rect::from_min_max(rect.left_bottom() + egui::vec2(0.0, 1.0), rect.right_bottom() + egui::vec2(0.0, 2.0)); + ui.painter().rect_filled(r, egui::CornerRadius::same(1), t.accent); + } + ui.add_space(4.0); +} + +/// Toolbar: every control is live — the choices feed the next run (and are persisted), Run +/// starts a recognition, Accept/Reject/navigate/verify only act in ReviewReady. +fn toolbar(app: &mut PdfCraftApp, ui: &mut egui::Ui, n: &mut Intents) { + let v = &app.ocr_verify; + let d = &mut app.ocr_draft; + let review = v.phase == Phase::ReviewReady; + let run_enabled = v.can_run() && v.reread.is_none(); + let accept_enabled = review && v.word_count() + v.review.applied_pages.len() > 0; + + // Row 1: the settings (each written back into the persisted draft). + ui.horizontal_wrapped(|ui| { + craft_ui::ComboBox::from_id_salt("ocr-verify-language").width(150.0).selected_text(language_name(&d.language)).show_ui(ui, |ui| { + for (code, name) in pdfcraft_engine::ocr::LANGUAGES { + ui.selectable_value(&mut d.language, (*code).to_string(), *name); + } + }); + engine_combo(ui, &mut d.engine, &mut d.strategy); + strategy_combo(ui, &mut d.strategy); + output_combo(ui, &mut d.output_mode); + for (flag, label, why) in [ + (&mut d.auto_rotate, tl!("Auto-rotate"), tl!("Turns pages scanned sideways or upside down upright before reading")), + (&mut d.deskew, tl!("Deskew"), tl!("Straightens skewed text lines before reading")), + (&mut d.denoise, tl!("Denoise"), tl!("A 3×3 median filter against scan speckle")), + (&mut d.binarize, tl!("Binarize"), tl!("Sauvola binarization: ink to black, paper to white")), + ] { + ui.add(crate::widgets::checkbox(flag, label)).on_hover_text(format!("{label} \u{2014} {why}")); + } + }); + + // Row 2: scope and actions. + ui.add_space(4.0); + ui.horizontal_wrapped(|ui| { + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::All, tl!("All pages")); + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::Current, tl!("Current page")); + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::Range, tl!("From")); + let on = d.pages == super::ocr_ui::OcrPages::Range; + ui.add_enabled(on, egui::DragValue::new(&mut d.from).range(1..=9999)); + ui.label(tl!("to")); + ui.add_enabled(on, egui::DragValue::new(&mut d.to).range(1..=9999)); + + ui.separator(); + if ui.add_enabled_ui(run_enabled, |ui| widgets::pill_button(ui, tl!("Run OCR"), true)).inner.clicked() { + n.run = true; + } + if v.phase == Phase::Running && widgets::pill_button(ui, tl!("Cancel"), false).clicked() { + n.cancel = true; + } + if v.reread.is_some() && widgets::pill_button(ui, tl!("Cancel re-read"), false).clicked() { + n.cancel_reread = true; + } + if ui.add_enabled_ui(accept_enabled, |ui| widgets::pill_button(ui, tl!("Accept"), false)).inner.clicked() { + n.accept = true; + } + if ui.add_enabled_ui(review, |ui| widgets::pill_button(ui, tl!("Reject"), false)).inner.clicked() { + n.reject = true; + } + ui.separator(); + let page_done = review && v.review.verified_pages.contains(&v.current); + let mut verified = page_done; + if ui + .add_enabled(review, crate::widgets::checkbox(&mut verified, tl!("Page verified"))) + .on_hover_text(tl!("Mark this page as reviewed (Ctrl+T)")) + .changed() + { + n.toggle_page_verified = true; + } + if ui + .add_enabled_ui(review, |ui| widgets::ghost_button(ui, "chevron-right", tl!("Next uncertain"))) + .inner + .on_hover_text(tl!("F8: next word to review; Shift+F8 goes back")) + .clicked() + { + n.next_uncertain = true; + } + if ui + .add_enabled_ui(review, |ui| widgets::ghost_button(ui, "chevron-left", tl!("Previous uncertain"))) + .inner + .on_hover_text(tl!("Shift+F8: the previous word to review")) + .clicked() + { + n.prev_uncertain = true; + } + if widgets::ghost_button(ui, "x", tl!("Exit")).clicked() { + n.exit = true; + } + }); +} + +/// A language code's dropdown text: its name and its code ("English (en)"). +fn language_name(code: &str) -> String { + pdfcraft_engine::ocr::LANGUAGES.iter().find(|l| l.0 == code).map_or_else(|| code.to_owned(), |l| format!("{} ({})", l.1, l.0)) +} + +/// The Engine dropdown's text. +fn engine_label(engine: EngineChoice) -> &'static str { + match engine { + EngineChoice::Auto => tl!("Automatic"), + EngineChoice::Ocrs => "ocrs", + EngineChoice::Tesseract => "Tesseract", + } +} + +/// The Strategy dropdown's text. +fn strategy_label(strategy: MergeStrategy) -> &'static str { + match strategy { + MergeStrategy::PrimaryOnly => tl!("Primary only"), + MergeStrategy::ConfidenceWeighted => tl!("Confidence weighted"), + MergeStrategy::RoverVote => tl!("ROVER vote"), + } +} + +/// The Engine dropdown: Automatic, ocrs, Tesseract. An engine that cannot run is disabled and +/// says why (a tooltip names the missing models or program). The choice feeds the next run. +/// Choosing Automatic while the strategy is "Primary only" (which would never run the second +/// engine) moves the strategy to Automatic's own default, Confidence weighted. +fn engine_combo(ui: &mut egui::Ui, engine: &mut EngineChoice, strategy: &mut MergeStrategy) { + ocr_ui::keep_asking_while_probing(ui.ctx()); + craft_ui::ComboBox::from_id_salt("ocr-verify-engine").width(120.0).selected_text(engine_label(*engine)).show_ui(ui, |ui| { + for (choice, label) in [(EngineChoice::Auto, tl!("Automatic")), (EngineChoice::Ocrs, "ocrs"), (EngineChoice::Tesseract, "Tesseract")] { + let why = pdfcraft_engine::ocr::engine_unavailable_reason(choice); + let resp = ui.add_enabled(why.is_none(), egui::Button::selectable(*engine == choice, label)); + let clicked = resp.clicked(); + match why { + Some(reason) => { + resp.on_disabled_hover_text(crate::i18n::t(&reason)); + } + None => { + resp.on_hover_text(match choice { + EngineChoice::Auto => tl!("Both engines, combined as the Strategy says"), + EngineChoice::Ocrs => tl!("The built-in engine (pure Rust)"), + EngineChoice::Tesseract => tl!("The tesseract program, run as an external process"), + }); + } + } + if clicked { + *engine = choice; + if choice == EngineChoice::Auto && *strategy == MergeStrategy::PrimaryOnly { + *strategy = choice.default_strategy(); + } + } + } + }); +} + +/// The Strategy dropdown: how two engines' readings combine. With a single named engine the +/// ensemble strategies still run — they just degrade to the primary reading (the tooltip says +/// so), because only one engine was asked to read. +fn strategy_combo(ui: &mut egui::Ui, strategy: &mut MergeStrategy) { + craft_ui::ComboBox::from_id_salt("ocr-verify-strategy").width(150.0).selected_text(strategy_label(*strategy)).show_ui(ui, |ui| { + for (s, label, hint) in [ + (MergeStrategy::PrimaryOnly, tl!("Primary only"), tl!("Use the chosen engine's reading alone")), + (MergeStrategy::ConfidenceWeighted, tl!("Confidence weighted"), tl!("Add a second engine's reading when the first is unsure")), + (MergeStrategy::RoverVote, tl!("ROVER vote"), tl!("Always read with both engines and merge the words")), + ] { + let resp = ui.add(egui::Button::selectable(*strategy == s, label)).on_hover_text(hint); + if resp.clicked() { + *strategy = s; + } + } + }); +} + +/// The Output mode dropdown: what Accept produces, with tooltips that say what each mode +/// changes (searchable: the open page gains an invisible text layer; editable: a new file). +fn output_combo(ui: &mut egui::Ui, mode: &mut OutputMode) { + let name = match *mode { + OutputMode::Searchable => tl!("Searchable Image (Exact)"), + OutputMode::EditableText => tl!("Editable text"), + }; + craft_ui::ComboBox::from_id_salt("ocr-verify-output").width(180.0).selected_text(name).show_ui(ui, |ui| { + let s = ui + .selectable_label(*mode == OutputMode::Searchable, tl!("Searchable Image (Exact)")) + .on_hover_text(tl!("Adds invisible text over each word; the page image is not changed")); + if s.clicked() { + *mode = OutputMode::Searchable; + } + let e = ui + .selectable_label(*mode == OutputMode::EditableText, tl!("Editable text")) + .on_hover_text(tl!("Writes a new document of visible text; the scan is not changed")); + if e.clicked() { + *mode = OutputMode::EditableText; + } + }); +} + +/// The info strip: where the review stands, in facts — page, average confidence, how many +/// words ask for review, the share verified, and the engine · strategy that read it. +fn info_strip(app: &PdfCraftApp, ui: &mut egui::Ui, t: &Tokens) { + let v = &app.ocr_verify; + let read = v.pages.iter().filter(|p| p.skipped.is_none()).count(); + let avg = v.page(v.current).and_then(pdfcraft_engine::ocr::OcrPage::mean_confidence); + let cells = [ + (tl!("PAGE"), if v.pages.is_empty() { "—".into() } else { format!("{} / {}", v.current + 1, v.pages.len()) }), + (tl!("AVG CONFIDENCE"), avg.map_or_else(|| "—".into(), |c| format!("{c:.0}%"))), + (tl!("LOW-CONFIDENCE WORDS"), v.suspect_count().to_string()), + (tl!("VERIFIED %"), format!("{}%", (v.verified_fraction() * 100.0).round() as i64)), + ( + tl!("ENGINE · STRATEGY"), + if read == 0 { "—".into() } else { format!("{} · {}", engine_label(app.ocr_draft.engine), strategy_label(app.ocr_draft.strategy)) }, + ), + ]; + ui.horizontal(|ui| { + for (k, val) in cells { + ui.label(egui::RichText::new(k.to_string()).small().color(t.text_faint)); + ui.label(egui::RichText::new(val).small()); + ui.separator(); + } + }); +} + +/// The page list: one row per read page, with its live word count and a flag for the words +/// that still ask for review. Clicking shows the page. +fn pages_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, n: &mut Intents) { + let v = &app.ocr_verify; + egui::ScrollArea::vertical().show(ui, |ui| { + for pg in &v.pages { + let p = pg.page; + let (live, suspects) = ( + pg.words.iter().filter(|w| !w.text.trim().is_empty()).count(), + pg.words.iter().filter(|w| !w.text.trim().is_empty() && pdfcraft_engine::ocr::is_suspect(w.confidence)).count(), + ); + let marked = if pg.skipped.is_some() { + format!(" · {}", tl!("skipped")) + } else if v.review.verified_pages.contains(&p) { + format!(" · {}", tl!("verified")) + } else { + String::new() + }; + let text = format!("{} {}{} · {live}", tl!("Page"), p + 1, marked); + let label = ui.selectable_label(p == v.current, text); + let label = if suspects > 0 { + label.on_hover_text(crate::i18n::fmt(tl!("{n} word(s) to review"), &[("n", &suspects.to_string())])) + } else { + label + }; + if label.clicked() { + n.select_page = Some(p); + } + } + }); +} + +/// The SOURCE PAGE pane: the rendered page with one box per recognized word. A click selects +/// the word under the pointer; a drag draws the re-OCR region (in view-space points, clamped +/// to the page). Words are coloured by their confidence band; removed words are struck out and +/// dimmed, corrected words outlined. The context menu re-reads a region or the page, and +/// accepts or rejects the page. +fn source_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let page = app.ocr_verify.current; + let review = app.ocr_verify.phase == Phase::ReviewReady; + ensure_preview(app, page); + let Some(doc) = app.ocr_verify.doc else { return }; + let Some(info) = app.session.get(doc).and_then(|d| d.info.pages.get(page)).cloned() else { return }; + // Reserve the room the confidence legend (at the end of this pane) and the message row + // (after it) need, so the page cannot push them off the panel. + let size = (ui.available_size() - egui::vec2(0.0, 40.0)).max(egui::vec2(120.0, 160.0)); + let (rect, resp) = ui.allocate_exact_size(size, egui::Sense::click_and_drag()); + let painter = ui.painter_at(rect); + // The page fits the pane; the pasteboard shows around it. + let s = (rect.width() / info.width.max(1.0)).min(rect.height() / info.height.max(1.0)).max(0.01); + let size = egui::vec2(info.width * s, info.height * s); + let img_rect = egui::Rect::from_min_size(rect.min + (rect.size() - size) * 0.5, size); + painter.rect_filled(img_rect.expand(2.0), egui::CornerRadius::same(2), t.hover); + if let Some(preview) = &app.ocr_verify.preview + && let Some(tex) = &preview.texture + { + painter.image(tex.id(), img_rect, egui::Rect::from_min_max(egui::pos2(0.0, 0.0), egui::pos2(1.0, 1.0)), egui::Color32::WHITE); + } + // A word's box: its user-space corners, through the page's rotation, into pane pixels. + let to_pane = |p: [f64; 2]| { + let v = info.user_to_view(p[0] as f32, p[1] as f32); + egui::pos2(img_rect.left() + v[0] * s, img_rect.top() + v[1] * s) + }; + let corners = |w: &PlacedWord| { + [ + to_pane(w.origin), + to_pane([w.origin[0] + w.across[0], w.origin[1] + w.across[1]]), + to_pane([w.origin[0] + w.across[0] + w.up[0], w.origin[1] + w.across[1] + w.up[1]]), + to_pane([w.origin[0] + w.up[0], w.origin[1] + w.up[1]]), + ] + }; + if let Some(pg) = app.ocr_verify.page(page) { + for (wi, w) in pg.words.iter().enumerate() { + let key = (pg.page, wi); + let quad = corners(w); + if app.ocr_verify.is_removed(key) { + // Removed: dimmed, struck through — still there, still restorable. + painter.add(egui::Shape::convex_polygon(quad.to_vec(), egui::Color32::from_black_alpha(32), egui::Stroke::NONE)); + painter.line_segment([quad[0], quad[2]], egui::Stroke::new(1.0, t.text_faint)); + continue; + } + if let Some(band) = w.confidence.map(band_for) { + let (fill, _) = band_colors(band); + painter.add(egui::Shape::convex_polygon(quad.to_vec(), fill, egui::Stroke::NONE)); + } + let outline = if app.ocr_verify.selected == Some(key) { + egui::Stroke::new(2.0, egui::Color32::from_rgb(43, 122, 236)) + } else if app.ocr_verify.is_corrected(key) { + egui::Stroke::new(1.5, t.text) + } else { + // The band's outline; a word without a confidence reads as unmeasured. + w.confidence.map(|c| band_colors(band_for(c)).1).unwrap_or_else(|| egui::Stroke::new(0.5, t.text_faint)) + }; + painter.add(egui::Shape::convex_polygon(quad.to_vec(), egui::Color32::TRANSPARENT, outline)); + } + } + // The chosen region and the one being drawn right now. + let drag_now = resp.interact_pointer_pos().zip(app.ocr_verify.drag).map(|(p, start)| egui::Rect::from_two_pos(start, p).intersect(img_rect)); + let chosen = app.ocr_verify.region.and_then(|region| pane_region(region, &info, img_rect)); + for r in [chosen, drag_now].into_iter().flatten() { + painter.rect_stroke(r, 0.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Middle); + } + // A click selects the word under the pointer; dragging away from a click draws the region. + if resp.drag_started() { + app.ocr_verify.drag = resp.interact_pointer_pos(); + } + if resp.drag_stopped() { + let (start, end) = (app.ocr_verify.drag.take(), resp.interact_pointer_pos()); + if let (Some(a), Some(b)) = (start, end) + && egui::Rect::from_two_pos(a, b).intersect(img_rect).area() > 36.0 + { + // A region: normalize it into view-space points, clamped to the page. + let r = egui::Rect::from_two_pos(a, b).intersect(img_rect); + let region = [ + (r.left() - img_rect.left()) / s, + (r.top() - img_rect.top()) / s, + (r.right() - img_rect.left()) / s, + (r.bottom() - img_rect.top()) / s, + ]; + app.ocr_verify.region = (region[2] > region[0] && region[3] > region[1]).then_some(region); + } + } + if resp.clicked() + && let Some(at) = resp.interact_pointer_pos() + && let Some(pg) = app.ocr_verify.page(page) + { + n.select_word = pg + .words + .iter() + .enumerate() + .find(|(wi, w)| !app.ocr_verify.is_removed((pg.page, *wi)) && point_in_quad(at, &corners(w))) + .map(|(wi, _)| (pg.page, wi)); + } + resp.context_menu(|ui| { + let rereading = app.ocr_verify.reread.is_some(); + if ui.add_enabled(review && !rereading && app.ocr_verify.region.is_some(), egui::Button::new(tl!("Re-OCR this region"))).clicked() { + n.reocr_region = true; + ui.close(); + } + if ui.add_enabled(review && !rereading, egui::Button::new(tl!("Re-OCR entire page"))).clicked() { + n.reocr_page = true; + ui.close(); + } + ui.separator(); + if ui.add_enabled(review, egui::Button::new(tl!("Accept entire page"))).clicked() { + n.accept_page = true; + ui.close(); + } + if ui.add_enabled(review, egui::Button::new(tl!("Reject entire page"))).clicked() { + n.reject_page = true; + ui.close(); + } + ui.separator(); + ui.label(egui::RichText::new(tl!("Click a word to inspect it; drag a rectangle to re-read part of the page")).small().color(t.text_faint)); + }); + legend(ui, t); +} + +/// A region given in view-space points, back into pane pixels on `img_rect` (None when it +/// does not lie on the page). The region is already in view space (the drag stored it so), so +/// it is only scaled: mapping it through `user_to_view` again would mirror it vertically (and +/// misplace it on rotated or cropped pages). +fn pane_region(region: [f32; 4], info: &pdfcraft_render::PageInfo, img_rect: egui::Rect) -> Option { + let scale = img_rect.width() / info.width.max(1.0); + let to_pane = |v: [f32; 2]| egui::pos2(img_rect.left() + v[0] * scale, img_rect.top() + v[1] * scale); + let a = to_pane([region[0], region[1]]); + let b = to_pane([region[2], region[3]]); + let r = egui::Rect::from_two_pos(a, b).intersect(img_rect); + (r.area() > 1.0).then_some(r) +} + +/// Point in a convex quad (corners in order): the pointer is on the same side of all four +/// edges. +fn point_in_quad(p: egui::Pos2, quad: &[egui::Pos2; 4]) -> bool { + (0..4).all(|i| { + let (a, b) = (quad[i], quad[(i + 1) % 4]); + (b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x) <= 0.0 + }) +} + +/// Band colours: the fill at ~20% opacity, the outline at ~60% — the legend names the bands +/// as text, so meaning never rests on colour alone. +fn band_colors(band: ConfidenceBand) -> (egui::Color32, egui::Stroke) { + let c = band_color(band); + (c.gamma_multiply(0.2), egui::Stroke::new(1.0, c.gamma_multiply(0.6))) +} + +/// The overlay legend: the classifier's bands, with their thresholds spelled out. +fn legend(ui: &mut egui::Ui, t: &Tokens) { + ui.add_space(2.0); + ui.horizontal_wrapped(|ui| { + ui.label(egui::RichText::new(tl!("CONFIDENCE")).small().strong()); + for (band, text) in legend_rows() { + let (fill, stroke) = band_colors(band); + let (rect, _) = ui.allocate_exact_size(egui::vec2(12.0, 12.0), egui::Sense::hover()); + ui.painter().rect_filled(rect, 2.0, fill); + ui.painter().rect_stroke(rect, 2.0, stroke, egui::StrokeKind::Inside); + ui.label(egui::RichText::new(text).small().color(t.text_muted)); + } + }); +} + +/// The legend's rows: the same [`ConfidenceBand`] values the overlay, the counts and the +/// navigation use, with the classifier's thresholds as text (a test holds the drawing to +/// these numbers). +pub fn legend_rows() -> [(ConfidenceBand, String); 3] { + [ + (ConfidenceBand::High, tl!("HIGH (>= 90%)").to_string()), + (ConfidenceBand::Medium, tl!("MEDIUM (70-89%)").to_string()), + (ConfidenceBand::Low, tl!("LOW (< 70%)").to_string()), + ] +} + +/// Render (or re-render) the source pane's page preview when the shown page changed. One +/// synchronous render per page; the texture is cached until then. +fn ensure_preview(app: &mut PdfCraftApp, page: usize) { + let Some(doc) = app.ocr_verify.doc else { return }; + if app.ocr_verify.preview.as_ref().is_some_and(|p| p.doc == doc && p.page == page && p.texture.is_some()) { + return; + } + let Some(d) = app.session.get(doc) else { return }; + let Some(info) = d.info.pages.get(page) else { return }; + // Screen scale (the boxes are drawn on top of it); the recognition dpi is irrelevant here. + let scale = 2.0_f32.min((900.0 / info.width.max(1.0)).min(1200.0 / info.height.max(1.0))).max(0.2); + let config = pdfcraft_render::RenderConfig { password: d.password.as_deref().map(std::sync::Arc::from), ..Default::default() }; + let mut r = pdfcraft_render::PageRenderer::new(d.bytes.clone(), config); + let shot = r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Pixels, scale, ..Default::default() }); + if shot.error.is_some() || shot.width == 0 || shot.height == 0 { + return; // keep any earlier preview; a render failure names itself elsewhere + } + let scale = shot.width as f32 / info.width.max(1.0); + let texture = app.ctx.as_ref().map(|ctx| { + ctx.load_texture( + "ocr-verify-page", + egui::ColorImage::from_rgba_premultiplied([shot.width as usize, shot.height as usize], &shot.rgba), + egui::TextureOptions::LINEAR, + ) + }); + app.ocr_verify.preview = Some(Preview { doc, page, scale, width: shot.width, height: shot.height, texture }); +} + +/// The RECOGNIZED pane: the page's words as the engine read them — a preview, not +/// authoritative (the source pane's boxes and the inspector are where corrections happen). +/// Removed words show struck out; clicking a word selects it in the inspector. +fn text_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let v = &app.ocr_verify; + let Some(pg) = v.page(v.current) else { return }; + if let Some(why) = &pg.skipped { + ui.label(egui::RichText::new(why.as_str()).small().color(t.text_muted)); + return; + } + if pg.words.is_empty() { + ui.label(egui::RichText::new(tl!("OCR complete — no text recognized on this page.")).small().color(t.text_muted)); + return; + } + egui::ScrollArea::vertical().max_height(180.0).show(ui, |ui| { + ui.horizontal_wrapped(|ui| { + for (wi, w) in pg.words.iter().enumerate() { + let key = (pg.page, wi); + let mut label = egui::RichText::new(w.text.as_str()).small(); + if v.is_removed(key) { + label = label.strikethrough().color(t.text_faint); + } else if v.is_corrected(key) { + label = label.underline(); + } + if ui.selectable_label(v.selected == Some(key), label).clicked() { + n.select_word = Some(key); + } + } + }); + }); +} + +/// The ZOOM + word inspector: a magnified crop of the selected word, its facts (id, +/// confidence, engine, review state), the correction field (Enter commits; clearing removes +/// the word; the original box never moves), Remove/Restore, + Dictionary, the per-word +/// Verified toggle, and suggestions within edit distance 2. +fn inspector(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let v = &app.ocr_verify; + let Some(key) = v.selected else { + ui.label(egui::RichText::new(tl!("Select a word on the page to inspect it")).small().color(t.text_muted)); + return; + }; + if v.word(key).is_none() { + ui.label(egui::RichText::new(tl!("The selected word is gone; choose another")).small().color(t.text_muted)); + return; + } + zoom_crop(app, ui, key, t); + ui.add_space(4.0); + let (id, conf, source) = { + let w = v.word(key); + ( + key.1 + 1, + w.and_then(|w| w.confidence).map_or_else(|| "—".to_owned(), |c| format!("{c:.0}%")), + w.map(|w| w.source.clone()).unwrap_or_default(), + ) + }; + let state = if v.is_removed(key) { + tl!("removed") + } else if v.is_corrected(key) { + tl!("corrected") + } else if v.is_verified(key) { + tl!("verified") + } else { + tl!("unreviewed") + }; + ui.label(egui::RichText::new(format!("{id} · {conf} · {source} · {state}")).small().color(t.text_muted)); + ui.add_space(4.0); + // The correction field: Enter commits, an empty commit removes, the box never moves. + let (mut correction, verified) = (app.ocr_verify.correction.clone(), app.ocr_verify.is_verified(key)); + let field = ui.add(egui::TextEdit::singleline(&mut correction).hint_text(tl!("Corrected text (Enter commits)"))); + let commit = field.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.horizontal(|ui| { + if ui.add(egui::Button::new(tl!("Remove")).small()).clicked() { + n.set_removed = Some(true); + } + if ui.add(egui::Button::new(tl!("Restore")).small()).clicked() { + n.set_removed = Some(false); + } + if ui.add(egui::Button::new(tl!("+ Dictionary")).small()).clicked() { + n.dictionary_add = true; + } + let mut on = verified; + if ui.add(crate::widgets::checkbox(&mut on, tl!("Verified"))).changed() { + n.set_word_verified = Some(on); + } + if ui.add(egui::Button::new(tl!("Undo")).small()).on_hover_text(tl!("Undo the last review change (Ctrl+Z)")).clicked() { + n.undo = true; + } + }); + app.ocr_verify.correction = correction; + if commit { + n.correct = Some(app.ocr_verify.correction.clone()); + } + // Suggestions: dictionary words and this page's own words within edit distance 2. + let (word_text, page_words) = { + let v = &app.ocr_verify; + (v.text_of(key).unwrap_or_default().to_owned(), v.page(key.0).map(|p| p.words.clone()).unwrap_or_default()) + }; + let sugg = suggestions(&word_text, &app.ocr_verify.review.dictionary, &page_words); + if !sugg.is_empty() { + ui.add_space(2.0); + ui.horizontal_wrapped(|ui| { + ui.label(egui::RichText::new(tl!("Suggestions")).small().color(t.text_muted)); + for s in sugg { + if ui.add(egui::Button::new(egui::RichText::new(s.clone()).small())).clicked() { + n.correct = Some(s); + } + } + }); + } +} + +/// The magnified crop of the selected word, drawn out of the page preview's texture. +fn zoom_crop(app: &PdfCraftApp, ui: &mut egui::Ui, key: (usize, usize), t: &Tokens) { + let Some(preview) = &app.ocr_verify.preview else { return }; + let Some(tex) = &preview.texture else { return }; + let Some(w) = app.ocr_verify.word(key) else { return }; + let Some(info) = app.ocr_verify.doc.and_then(|doc| app.session.get(doc)).and_then(|d| d.info.pages.get(preview.page)).cloned() else { + return; + }; + // The word's box in preview pixels (user → view → × preview scale), plus a margin. + let to_px = |p: [f64; 2]| { + let v = info.user_to_view(p[0] as f32, p[1] as f32); + [v[0] * preview.scale, v[1] * preview.scale] + }; + let a = to_px(w.origin); + let b = to_px([w.origin[0] + w.across[0] + w.up[0], w.origin[1] + w.across[1] + w.up[1]]); + let (x0, y0, x1, y1) = (a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])); + let (mw, mh) = ((x1 - x0).max(8.0) * 0.5, (y1 - y0).max(8.0) * 0.6); + let crop = egui::Rect::from_min_max( + egui::pos2((x0 - mw).max(0.0), (y0 - mh).max(0.0)), + egui::pos2((x1 + mw).min(preview.width as f32), (y1 + mh).min(preview.height as f32)), + ); + if crop.width() < 1.0 || crop.height() < 1.0 { + return; + } + // At most 300 × 90 points, at most 3× the preview's own resolution. + let k = (300.0 / crop.width()).min(90.0 / crop.height()).clamp(0.1, 3.0); + let (rect, _) = ui.allocate_exact_size(egui::vec2(crop.width() * k, crop.height() * k), egui::Sense::hover()); + let uv = egui::Rect::from_min_max( + egui::pos2(crop.left() / preview.width as f32, crop.top() / preview.height as f32), + egui::pos2(crop.right() / preview.width as f32, crop.bottom() / preview.height as f32), + ); + ui.painter().image(tex.id(), rect, uv, egui::Color32::WHITE); + ui.painter().rect_stroke(rect, 0.0, egui::Stroke::new(1.0, t.accent), egui::StrokeKind::Inside); +} + +/// Run `f` with the screen's state moved out of the app (a step that needs both the state and +/// the rest of the app would otherwise borrow it twice). The state is put back afterwards. +pub(crate) fn with_state(app: &mut PdfCraftApp, f: impl FnOnce(&mut OcrVerify, &mut PdfCraftApp)) { + let mut v = std::mem::take(&mut app.ocr_verify); + f(&mut v, app); + app.ocr_verify = v; +} + +/// Apply what the frame's controls asked for. The guards live in the [`OcrVerify`] methods: +/// outside ReviewReady, Accept/Reject/Re-OCR and edits are ignored, not acted on. +fn apply(app: &mut PdfCraftApp, n: Intents) { + with_state(app, |v, app| { + if n.exit { + v.open = false; + } + if let Some(p) = n.select_page { + v.current = p; + if v.selected.is_some_and(|k| k.0 != p) { + v.selected = None; + } + } + if let Some(k) = n.select_word { + v.select(k); + } + if n.toggle_page_verified { + let on = !v.review.verified_pages.contains(&v.current); + v.set_page_verified(v.current, on); + } + if n.next_uncertain { + v.goto_uncertain(true); + } + if n.prev_uncertain { + v.goto_uncertain(false); + } + if n.undo { + v.undo(); + } + if let Some(text) = n.correct + && let Some(k) = v.selected + { + v.correct(k, &text); + v.correction = v.text_of(k).unwrap_or_default().to_owned(); + } + if let Some(removed) = n.set_removed + && let Some(k) = v.selected + { + v.set_removed(k, removed); + } + if let Some(on) = n.set_word_verified + && let Some(k) = v.selected + { + v.set_word_verified(k, on); + } + if n.dictionary_add + && let Some(k) = v.selected + { + let word = v.text_of(k).unwrap_or_default().to_owned(); + let language = v.dictionary_language.clone(); + v.add_to_dictionary(&word, &language); + } + if n.reocr_page { + let page = v.current; + v.start_reread(app, page, None); + } + if n.reocr_region { + let (page, region) = (v.current, v.region); + if let Some(region) = region { + v.start_reread(app, page, Some(region)); + } + } + if n.cancel_reread { + v.cancel_reread(); + } + if n.accept_page { + let page = v.current; + match v.accept_page(&mut app.session, page) { + Ok(count) if count > 0 => { + if let Some(doc) = v.doc { + refresh_view(app, doc); + } + app.notify_fmt(tl!("The page's words were applied as one undoable step ({n} words)"), &[("n", &count.to_string())]); + } + Ok(_) => {} + Err(why) => app.notify_error(why), + } + } + if n.reject_page { + let page = v.current; + if v.reject_page(page) { + app.notify_tr("The page's words were cleared (undo in the panel)"); + } + } + if n.run { + v.start_run(app); + } + if n.cancel { + v.cancel(); + } + if n.accept { + do_accept(v, app); + } + if n.reject && v.reject() { + app.notify_tr("OCR results rejected."); + } + }); +} + +/// Accept: searchable mode applies the review to the open document as one undoable step; +/// editable mode asks where to write the new `_ocr.pdf`. Stale sessions and empty +/// reviews are typed failures the user sees (the edits are kept). +fn do_accept(v: &mut OcrVerify, app: &mut PdfCraftApp) { + let doc = v.doc; + match app.ocr_draft.output_mode { + OutputMode::Searchable => match v.accept(&mut app.session) { + Ok(count) => { + if let Some(doc) = doc { + refresh_view(app, doc); + } + if count > 0 { + app.notify_fmt(tl!("Recognized text applied ({n} words, one undoable step)"), &[("n", &count.to_string())]); + } + } + Err(why) => app.notify_error(why), + }, + OutputMode::EditableText => match v.editable_output(&app.session) { + Ok((bytes, name)) => save_editable(app, bytes, name, doc), + Err(why) => app.notify_error(why), + }, + } +} + +/// Write the editable-text output: into the tests' export folder when one is set, otherwise +/// ask where (the default name is `_ocr.pdf`). The source is never overwritten; a +/// cancelled dialog keeps the review. The write is the [`Phase::Saving`] state: it ends the +/// review when the file lands, and returns to ReviewReady when it does not, so a failed or +/// cancelled save can simply be tried again. +fn save_editable(app: &mut PdfCraftApp, bytes: Arc>, name: String, doc: Option) { + let source = doc.and_then(|id| app.session.get(id)).and_then(|d| d.path.clone()); + // The chosen file is the open source when both resolve to the same path (a new file does + // not exist yet, so fall back to its folder and name). + let is_source = |picked: &std::path::Path, source: &Option| { + let Some(src) = source else { return false }; + if let (Ok(a), Ok(b)) = (std::fs::canonicalize(picked), std::fs::canonicalize(src)) { + return a == b; + } + let same_name = picked.file_name().is_some_and(|n| n.to_string_lossy() == src.as_str()); + let same_folder = std::fs::canonicalize(src) + .ok() + .zip(picked.parent().map(std::path::Path::to_path_buf)) + .is_some_and(|(s, parent)| s.parent().is_some_and(|p| p == parent)); + same_name && same_folder + }; + #[cfg(not(target_arch = "wasm32"))] + if let Some(dir) = app.export_dir_override.clone() { + let target = std::path::Path::new(&dir).join(&name); + if is_source(&target, &source) { + app.notify_tr("That is the source file; choose another destination"); + return; + } + write_editable(app, &target, bytes); + return; + } + #[cfg(not(target_arch = "wasm32"))] + { + let dialog = rfd::AsyncFileDialog::new().add_filter("PDF", &["pdf"]).set_file_name(&name); + app.ask_one(crate::pickers::Ask::Save(dialog), None, move |app, picked: std::path::PathBuf| { + if is_source(&picked, &source) { + app.notify_tr("That is the source file; choose another destination"); + return; + } + write_editable(app, &picked, bytes); + }); + } + // The web build has no file system to write into: the new document downloads under its + // `_ocr.pdf` name, so it never lands on the source either. + #[cfg(target_arch = "wasm32")] + { + let _ = (&source, doc, is_source); + let saved = crate::editing::download(&name, &bytes); + with_state(app, |v, _| { + if saved.is_ok() { + v.finish(); + } else { + v.phase = Phase::ReviewReady; + } + }); + if let Err(e) = saved { + app.notify_error(e); + } + } +} + +/// The editable write itself: [`Phase::Saving`] while it runs, Idle once the file lands (the +/// review is done), back to ReviewReady when it fails — the reviewer sees the error and can +/// accept again without re-reading the document. Desktop only: the web build downloads instead. +#[cfg(not(target_arch = "wasm32"))] +fn write_editable(app: &mut PdfCraftApp, target: &std::path::Path, bytes: Arc>) { + with_state(app, |v, _| v.phase = Phase::Saving); + match crate::editing::write_atomically(&target.to_string_lossy(), &bytes) { + Ok(()) => { + with_state(app, |v, _| v.finish()); + app.notify(crate::i18n::fmt(tl!("Wrote {name}"), &[("name", &target.to_string_lossy())])); + } + Err(e) => { + with_state(app, |v, _| v.phase = Phase::ReviewReady); + app.notify_error(e); + } + } +} + +/// Pull the document's changed appearance into its view after an apply. +fn refresh_view(app: &mut PdfCraftApp, doc: DocId) { + if let Some(info) = app.session.get(doc).map(|d| d.info.clone()) + && let Some(view) = app.views.iter_mut().find(|view| view.id == doc) + { + view.document_changed(&info); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfcraft_engine::Session; + + /// A word box that sits where a real placed word would (high on the page, reading left + /// to right). + fn word(text: &str, conf: Option) -> PlacedWord { + PlacedWord { text: text.into(), origin: [72.0, 700.0], across: [40.0, 0.0], up: [0.0, 12.0], confidence: conf, source: "ocrs".into() } + } + + /// One page of words with the given confidences, and a review loaded with it (the guard + /// comes from a real job, as it does on screen). + fn loaded(session: &mut Session, confs: &[Option]) -> (DocId, OcrVerify) { + let text = session.create_from_text("t", "Recognize these scanned words").expect("fixture"); + let id = session.open("t.pdf", None, text, None).expect("fixture"); + let job = session.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + let page = OcrPage { already_on_page: 0, page: 0, words: confs.iter().map(|c| word("w", *c)).collect(), skipped: None, notes: Vec::new() }; + let mut v = OcrVerify::default(); + v.load(id, job.guard.clone(), vec![page]); + (id, v) + } + + /// The spec's status wording is fixed: a finished run that found nothing says so, and a + /// cancelled one never claims a complete run. + #[test] + fn finished_states_keep_the_spec_wording() { + let mut v = OcrVerify::default(); + v.finished(vec![OcrPage { already_on_page: 0, page: 0, words: vec![], skipped: None, notes: Vec::new() }], false); + assert_eq!(v.phase, Phase::ReviewReady); + assert_eq!(v.message, "OCR complete — no text recognized on this page."); + v.finished( + vec![OcrPage { + already_on_page: 0, + page: 0, + words: vec![word("Hi", Some(95.0)), word("there", Some(40.0))], + skipped: None, + notes: Vec::new(), + }], + true, + ); + assert!(v.message.starts_with("Cancelled — 2 words were read on 1 of 1 pages"), "{}", v.message); + assert_eq!(v.phase, Phase::ReviewReady, "a cancel keeps the pages already read, in review"); + assert_eq!(v.page(0).expect("the cancelled run's page is kept").words.len(), 2); + } + + /// The legend's text is the classifier's thresholds, spelled out — a drift between the + /// two would put the wrong words in front of the reviewer. + #[test] + fn legend_rows_carry_the_classifier_thresholds() { + let rows = legend_rows(); + assert_eq!(rows[0].0, ConfidenceBand::High); + assert_eq!(rows[1].0, ConfidenceBand::Medium); + assert_eq!(rows[2].0, ConfidenceBand::Low); + assert_eq!(rows[0].1, "HIGH (>= 90%)"); + assert_eq!(rows[1].1, "MEDIUM (70-89%)"); + assert_eq!(rows[2].1, "LOW (< 70%)"); + // The same classifier drives the overlay: 90 is High, 89 is not; 70 is Medium, 69 is + // not; the bands clamp out-of-range values. + assert_eq!(band_for(90.0), ConfidenceBand::High); + assert_eq!(band_for(89.0), ConfidenceBand::Medium); + assert_eq!(band_for(70.0), ConfidenceBand::Medium); + assert_eq!(band_for(69.0), ConfidenceBand::Low); + assert_eq!(band_for(-1.0), ConfidenceBand::Low); + assert_eq!(band_for(140.0), ConfidenceBand::High); + } + + /// F8/Shift+F8 walk the uncertain words: wrap around, skip removed words, and a page + /// marked verified drops out entirely. With nothing to show, the walk is a no-op. + #[test] + fn uncertain_walk_wraps_and_skips_removed_and_verified() { + let mut s = Session::new(); + // 95 is fine; 40 and 50 are Low; the third word is corrected into the dictionary. + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0), Some(50.0), Some(95.0)]); + assert_eq!(v.uncertain(), vec![(0, 1), (0, 2)], "the Low-band words, in order"); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 1))); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 2))); + assert!(v.goto_uncertain(true), "wraps around"); + assert_eq!(v.selected, Some((0, 1))); + // A removed word leaves the walk; a corrected word stays by its original confidence. + v.set_removed((0, 1), true); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 2))); + assert!(v.correct((0, 2), "fixed"), "the correction commits"); + assert!(v.uncertain().contains(&(0, 2)), "a corrected word stays uncertain by its original confidence"); + v.set_word_verified((0, 2), true); + assert!(v.uncertain().is_empty(), "a vouched-for word leaves the walk"); + assert!(!v.goto_uncertain(true), "nothing left: a no-op"); + // A page marked verified drops all its words from the walk. + v.set_word_verified((0, 2), false); + v.set_removed((0, 1), false); + v.set_page_verified(0, true); + assert!(v.uncertain().is_empty()); + } + + /// The review tools are guarded: outside ReviewReady they are ignored, not acted on. + #[test] + fn guards_ignore_review_actions_outside_reviewready() { + let mut v = OcrVerify::default(); + assert_eq!(v.phase, Phase::Idle); + assert!(v.can_run(), "Run is enabled in Idle"); + assert!(!v.reject(), "Reject is ignored in Idle"); + assert!(!v.set_removed((0, 0), true)); + assert!(!v.correct((0, 0), "x")); + assert!(!v.set_page_verified(0, true)); + assert!(!v.reject_page(0)); + let mut s = Session::new(); + assert_eq!(v.accept(&mut s).expect("ignored, not an error"), 0); + v.phase = Phase::Saving; + assert!(!v.can_run(), "Run is disabled while saving"); + v.phase = Phase::RecoverableError; + assert!(v.can_run(), "Run is enabled again after a recoverable error"); + } + + /// Accept: zero words left is a typed failure; a real review is applied as ONE undoable + /// step; a stale session is refused with its reason and the edits are kept. + #[test] + fn accept_zero_words_fails_and_a_live_review_applies_once() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0)]); + v.set_removed((0, 0), true); + v.set_removed((0, 1), true); + assert!(v.accept(&mut s).is_err(), "zero words: a typed failure"); + assert_eq!(v.phase, Phase::ReviewReady, "the failure keeps the review"); + let live = v.live_pages(); + assert!(live[0].words.is_empty(), "removed words are not written"); + // Restore one word: the accept applies the whole review in one step. + v.set_removed((0, 0), false); + let n = v.accept(&mut s).expect("the accept applies"); + assert_eq!(n, 1); + assert_eq!(s.get(id).expect("open").can_undo(), Some("Recognize text"), "one undoable step"); + assert_eq!(v.phase, Phase::Idle, "a successful accept ends the review"); + // A second accept of the same (now spent) review has nothing to write. + let mut v2 = OcrVerify::default(); + let job = s.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + v2.load( + id, + job.guard.clone(), + vec![OcrPage { already_on_page: 0, page: 0, words: vec![word("w", Some(95.0))], skipped: None, notes: Vec::new() }], + ); + v2.set_removed((0, 0), true); + assert!(v2.accept(&mut s).is_err()); + } + + /// Accept refuses a stale session: the document was edited since the job started, the + /// refusal names the reason, and the review edits survive. + #[test] + fn accept_refuses_a_stale_session_with_a_named_reason() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0)]); + assert!(v.correct((0, 0), "Corrected"), "the correction commits"); + // The document changed since the job started: the result no longer applies. + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("fixture"); + let why = v.accept(&mut s).expect_err("the stale result is refused"); + assert!(why.contains("changed"), "{why}"); + assert_eq!(v.phase, Phase::RecoverableError, "Run is enabled again"); + assert_eq!(v.text_of((0, 0)), Some("Corrected"), "the edits are kept"); + } + + /// "Accept entire page" applies just that page and a later whole-review Accept never + /// writes it twice; "Reject entire page" clears the page (undoable in the panel). + #[test] + fn accept_page_then_accept_does_not_double_write() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(95.0)]); + v.pages = vec![OcrPage { + already_on_page: 0, + page: 0, + words: vec![word("one", Some(95.0)), word("two", Some(95.0))], + skipped: None, + notes: Vec::new(), + }]; + let n = v.accept_page(&mut s, 0).expect("the page applies"); + assert_eq!(n, 2); + assert!(v.review.applied_pages.contains(&0)); + // Reject entire page: the words clear (undoable), then come back. + assert!(v.reject_page(0)); + assert!(v.page(0).expect("page").words.is_empty()); + assert!(v.undo(), "the clear is undoable"); + assert_eq!(v.page(0).expect("page").words.len(), 2); + // A zero-word page cannot be accepted. + v.reject_page(0); + assert!(v.accept_page(&mut s, 0).is_err(), "zero words on the page: a typed failure"); + } + + /// The text the document's page `page` carries now, as the renderer extracts it. + fn page_text(s: &Session, id: DocId, page: usize) -> String { + let doc = s.get(id).expect("open"); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), pdfcraft_render::RenderConfig::default()); + let request = pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; + r.render(request).text.map(|t| t.plain_text()).unwrap_or_default() + } + + /// A blank document of `pages` pages, open, with a review of one word list per page loaded + /// against a real job's guard. + fn blank_review(s: &mut Session, words: Vec>) -> (DocId, OcrVerify) { + let blank = s.create_blank(612.0, 792.0, words.len()).expect("fixture"); + let id = s.open("blank.pdf", None, blank, None).expect("fixture"); + let job = s.ocr_job(id, &[], OcrSettings::default()).expect("fixture"); + let pages = words + .into_iter() + .enumerate() + .map(|(page, words)| OcrPage { already_on_page: 0, page, words, skipped: None, notes: Vec::new() }) + .collect(); + let mut v = OcrVerify::default(); + v.load(id, job.guard.clone(), pages); + (id, v) + } + + /// Pages accepted one at a time, then the whole review: each page-accept is the document's + /// newest edit, so the next one (and the final Accept) still applies, and every page is + /// written exactly once. Before the guard was refreshed, the second page-accept was refused + /// as "the document changed since recognition started" and the review was stuck. + #[test] + fn accepting_pages_one_by_one_then_the_review_applies_every_page_once() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))], vec![word("three", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1)); + assert_eq!(v.accept_page(&mut s, 1), Ok(1), "the second page applies after the first"); + assert_eq!(v.phase, Phase::ReviewReady); + assert_eq!(v.accept(&mut s), Ok(1), "the final Accept writes only the page not yet applied"); + assert_eq!(v.phase, Phase::Idle, "the review ended"); + for (page, text) in ["one", "two", "three"].into_iter().enumerate() { + let on_page = page_text(&s, id, page); + assert_eq!(on_page.matches(text).count(), 1, "page {page} written exactly once: {on_page:?}"); + } + } + + /// A page already applied ("Accept entire page") is never applied again: a second + /// "Accept entire page" on it writes nothing (the OCR layer is not appended twice), and + /// corrections made after the page-accept are kept in the review. + #[test] + fn accepting_a_page_twice_writes_it_once_and_keeps_later_corrections() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1), "the first accept applies the page"); + // The reviewer corrects a word on the applied page afterwards. + assert!(v.correct((0, 0), "uno"), "the correction commits"); + // A second "Accept entire page" on the same page writes nothing. + assert_eq!(v.accept_page(&mut s, 0), Ok(0), "already applied: ignored, not applied twice"); + let text = page_text(&s, id, 0); + assert_eq!(text.matches("one").count(), 1, "the page was written exactly once: {text:?}"); + // The later correction is kept (not lost by the refusal), and the final Accept writes + // only the page not yet applied. + assert_eq!(v.text_of((0, 0)), Some("uno")); + assert_eq!(v.accept(&mut s), Ok(1), "the final Accept writes only page 2"); + let text = page_text(&s, id, 0); + assert_eq!(text.matches("one").count() + text.matches("uno").count(), 1, "still written once: {text:?}"); + assert_eq!(page_text(&s, id, 1).matches("two").count(), 1); + } + + /// The refreshed guard still catches an edit made elsewhere between two page-accepts: the + /// next page is refused with the reason, nothing is written, and the review is kept. + #[test] + fn an_outside_edit_between_page_accepts_is_still_refused() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1)); + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an unrelated edit"); + let why = v.accept_page(&mut s, 1).expect_err("stale after an outside edit"); + assert!(why.contains("changed since recognition started"), "{why}"); + assert_eq!(v.phase, Phase::RecoverableError); + assert_eq!(v.page(1).expect("page").words.len(), 1, "the review is kept"); + assert!(page_text(&s, id, 1).trim().is_empty(), "nothing was written to page 2"); + } + + /// A word at `x` on the first line of a US Letter page, with a Low confidence. + fn word_at(text: &str, x: f64) -> PlacedWord { + PlacedWord { text: text.into(), origin: [x, 700.0], across: [40.0, 0.0], up: [0.0, 12.0], confidence: Some(40.0), source: "ocrs".into() } + } + + /// Whole-page re-read: review marks are positional, so the page's marks go with the old + /// words (a struck-out slot must not strike the new word there), a correction of an old + /// word no longer shows on the new one, and the undo restores words and marks exactly. + #[test] + fn rereading_a_page_clears_its_marks_and_undo_restores_them() { + let mut s = Session::new(); + let (_, mut v) = blank_review(&mut s, vec![vec![word_at("keep", 72.0), word_at("gone", 300.0), word_at("struck", 400.0)]]); + assert!(v.set_removed((0, 2), true)); + assert!(v.set_word_verified((0, 0), true)); + assert!(v.correct((0, 1), "fixed")); + let marks = (v.review.removed.clone(), v.review.verified.clone()); + assert!(v.replace_page_words(0, vec![word_at("a", 72.0), word_at("b", 300.0), word_at("c", 400.0)])); + assert!(v.review.removed.is_empty() && v.review.verified.is_empty(), "the old marks are gone"); + assert!(!v.is_corrected((0, 1)), "the correction belonged to the old word list"); + let texts: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["a", "b", "c"], "no new word is struck out by an old mark"); + assert!(v.undo()); + assert_eq!((v.review.removed.clone(), v.review.verified.clone()), marks, "the undo restores the marks"); + assert!(v.is_corrected((0, 1))); + let texts: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "fixed"], "the struck word stays out"); + } + + /// A blank two-page-shaped review inside a real app (the document open, Run set to go + /// inline), one word list per page, against a real job's guard. + fn app_review(words: Vec>) -> (PdfCraftApp, DocId) { + let blank = Session::new().create_blank(612.0, 792.0, words.len()).expect("fixture").to_vec(); + let mut app = PdfCraftApp::new(); + app.run_inline = true; + app.open_bytes("blank.pdf", None, blank).expect("fixture"); + let id = app.active_ids().expect("open").1; + let job = app.session.ocr_job(id, &[], OcrSettings::default()).expect("fixture"); + let pages = words + .into_iter() + .enumerate() + .map(|(page, words)| OcrPage { already_on_page: 0, page, words, skipped: None, notes: Vec::new() }) + .collect(); + app.ocr_verify.load(id, job.guard.clone(), pages); + (app, id) + } + + /// The status the tests see, from the app's last toast. + fn toast(app: &mut PdfCraftApp) -> String { + app.toast.take().map(|t| t.0).unwrap_or_default() + } + + /// A region re-read whose worker has finished with `result`, already asked for. + fn finished_reread(region: Option<[f32; 4]>, result: Result, String>) -> Reread { + Reread { page: 0, region, progress: std::sync::Arc::new(Mutex::new(RereadProgress { cancel: false, result: Some(result) })) } + } + + /// A page re-read runs OFF the UI thread: start_reread returns with the reading in flight + /// (Run is refused while it is out), and the worker's result is applied by the next poll. + /// No OCR models are needed: the engine fails fast here, which still walks the whole + /// off-thread path — and a failed reading keeps the words. + #[test] + fn a_page_reread_runs_off_the_ui_thread_and_poll_applies_it() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_draft.engine = EngineChoice::Ocrs; // an ocrs-only job never probes tesseract + app.run_inline = false; // the point of the test: a real worker thread + with_state(&mut app, |v, app| v.start_reread(app, 0, None)); + assert!(app.ocr_verify.reread.is_some(), "the reading is in flight"); + assert!(app.ocr_verify.message.contains("Re-reading"), "{}", app.ocr_verify.message); + // A full run waits for the re-read. + with_state(&mut app, OcrVerify::start_run); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "Run is refused while a re-read is out"); + // Wait for the worker (it fails fast without models), then apply its result. + let progress = app.ocr_verify.reread.as_ref().expect("in flight").progress.clone(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while progress.lock().expect("progress").result.is_none() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(progress.lock().expect("progress").result.is_some(), "the worker finished"); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none(), "the re-read is over"); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "the review survives a failed re-read"); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the words survive"); + assert!(!toast(&mut app).is_empty(), "the failure is said"); + } + + /// A re-read that finds nothing is a typed failure: the words stay, the review stays. + #[test] + fn a_reread_that_finds_nothing_keeps_the_words() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_verify.reread = Some(finished_reread(None, Err("the re-read found no text; the existing words are unchanged".into()))); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none(), "the re-read is over"); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the words survive the failed re-read"); + assert!(toast(&mut app).contains("found no text")); + } + + /// Cancel asks the worker to stop and discards the reading when it lands anyway: the + /// page's words stay as they were. + #[test] + fn a_cancelled_reread_is_discarded_and_keeps_the_words() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_verify.reread = Some(Reread { page: 0, region: None, progress: std::sync::Arc::new(Mutex::new(RereadProgress::default())) }); + with_state(&mut app, |v, _| v.cancel_reread()); + assert!(app.ocr_verify.reread.as_ref().expect("in flight").progress.lock().expect("progress").cancel, "the cancel reached the worker"); + // The worker finishes anyway; its (late) reading is discarded. + app.ocr_verify.reread.as_ref().expect("in flight").progress.lock().expect("progress").result = Some(Ok(vec![word("late", Some(95.0))])); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none()); + let words: Vec<_> = app.ocr_verify.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(words, ["one"], "the late reading is discarded"); + assert!(app.ocr_verify.message.contains("cancelled"), "{}", app.ocr_verify.message); + } + + /// A re-read of a document that changed since the review began is refused: the reading may + /// no longer fit the page it was read from, so the words stay and the reason is said. + #[test] + fn a_reread_of_a_changed_document_is_refused() { + let (mut app, id) = app_review(vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + app.ocr_verify.reread = Some(finished_reread(None, Ok(vec![word("late", Some(95.0))]))); + app.ocr_verify.current = 1; + app.session.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![1], degrees: 90 }).expect("an edit"); + with_state(&mut app, OcrVerify::poll_reread); + let words: Vec<_> = app.ocr_verify.page(1).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(words, ["two"], "the late reading is not applied"); + assert!(toast(&mut app).contains("changed"), "the refusal names its reason"); + } + + /// Region re-read (the worker's reading applied by the poll): the words outside the + /// rectangle stay and their marks follow them to their new positions; the replaced word's + /// marks go; the first NEW word is the one selected (it follows the kept words); undo + /// restores everything. + #[test] + fn rereading_a_region_remaps_marks_and_selects_the_first_new_word() { + let (mut app, _) = app_review(vec![vec![word_at("keep", 72.0), word_at("gone", 300.0), word_at("struck", 400.0)]]); + // The drawn region and the worker's reading of it: one word, "new". + let region = [290.0, 70.0, 350.0, 100.0]; + let new_word = PlacedWord { + text: "new".into(), + origin: [300.0, 700.0], + across: [40.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + { + let v = &mut app.ocr_verify; + assert!(v.set_removed((0, 2), true)); + assert!(v.set_word_verified((0, 0), true)); + assert!(v.set_word_verified((0, 1), true)); + v.region = Some(region); + v.reread = Some(finished_reread(Some(region), Ok(vec![new_word]))); + } + let marks = { + let v = &app.ocr_verify; + (v.review.removed.clone(), v.review.verified.clone()) + }; + // "gone" spans x 300..340 and y 700..712 in user space: view y 80..92 on the 792 pt page. + with_state(&mut app, OcrVerify::poll_reread); + let v = &mut app.ocr_verify; + assert!(v.reread.is_none(), "the re-read is over"); + let texts: Vec<_> = v.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "struck", "new"]); + assert_eq!(v.selected, Some((0, 2)), "the first new word is selected"); + assert!(v.is_removed((0, 1)), "the struck word's mark moved with it"); + assert!(!v.is_removed((0, 2)), "the new word is not struck out"); + assert!(v.is_verified((0, 0)) && !v.is_verified((0, 1)) && !v.is_verified((0, 2)), "{:?}", v.review.verified); + let live: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(live, ["keep", "new"], "the struck word is never written, the new one is"); + assert!(v.undo()); + assert_eq!((v.review.removed.clone(), v.review.verified.clone()), marks); + let texts: Vec<_> = v.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "gone", "struck"]); + } + + /// The editable output: refuses zero words, keeps the open document untouched, and names + /// the new file `_ocr.pdf`. + #[test] + fn editable_output_refuses_zero_words_and_never_touches_the_source() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0)]); + let before = s.get(id).expect("open").bytes.clone(); + let (bytes, name) = v.editable_output(&s).expect("the output builds"); + assert!(name.ends_with("_ocr.pdf"), "{name}"); + assert!(std::sync::Arc::ptr_eq(&s.get(id).expect("open").bytes, &before), "the source is untouched"); + assert!(!bytes.is_empty()); + v.set_removed((0, 0), true); + assert!(v.editable_output(&s).is_err(), "zero words: refused"); + assert_eq!(v.phase, Phase::ReviewReady); + } + + /// The editable output is a new file made from words read earlier: when the document changed + /// since (a redaction, say), it is refused with the reason, as Accept in searchable mode is. + #[test] + fn editable_output_refuses_a_document_that_changed_since_the_review_started() { + let mut s = Session::new(); + let (id, v) = loaded(&mut s, &[Some(95.0)]); + assert!(v.editable_output(&s).is_ok(), "an untouched document is fine"); + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an edit"); + let err = v.editable_output(&s).expect_err("a changed document is refused"); + assert!(err.contains("changed since recognition started"), "{err}"); + } + + /// The editable save is the Saving state: the file lands and the review ends (Idle); a + /// failed write returns to ReviewReady with the edits kept, and a cancelled or unanswered + /// save dialog never touches the review either — Accept can simply be tried again. + #[test] + fn editable_save_lands_and_a_failed_or_cancelled_one_keeps_the_review() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let text = app.session.create_from_text("t", "Recognize these scanned words").expect("fixture"); + let id = app.session.open("t.pdf", None, text, None).expect("fixture"); + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + app.ocr_verify.load( + id, + job.guard.clone(), + vec![OcrPage { already_on_page: 0, page: 0, words: vec![word("w", Some(95.0))], skipped: None, notes: Vec::new() }], + ); + assert!(app.ocr_verify.correct((0, 0), "Edited"), "the correction commits"); + app.ocr_draft.output_mode = OutputMode::EditableText; + // A destination that cannot be written (the "folder" is a regular file): the write + // fails, the review stays in ReviewReady with the edit, Accept is enabled again. + let blocker = std::env::temp_dir().join(format!("pdfcraft-ocr-save-block-{}", std::process::id())); + std::fs::write(&blocker, b"not a folder").expect("fixture"); + app.export_dir_override = Some(blocker.to_string_lossy().into_owned()); + with_state(&mut app, do_accept); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "a failed write keeps the review"); + assert_eq!(app.ocr_verify.text_of((0, 0)), Some("Edited"), "the edits are kept"); + // A cancelled save dialog (the override answers with nothing): nothing is written, + // nothing changes. + app.export_dir_override = None; + app.pick_override = Some(Vec::new()); + with_state(&mut app, do_accept); + app.process_picked(); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "a cancelled save keeps the review"); + assert_eq!(app.ocr_verify.text_of((0, 0)), Some("Edited")); + assert_eq!(app.ocr_verify.doc, Some(id), "the review still belongs to its document"); + // A real destination: the file lands under `_ocr.pdf` and the review ends. + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-save-out-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + app.pick_override = Some(vec![dir.join("t_ocr.pdf").to_string_lossy().into_owned()]); + with_state(&mut app, do_accept); + app.process_picked(); + assert!(dir.join("t_ocr.pdf").exists(), "the editable copy is written"); + assert_eq!(app.ocr_verify.phase, Phase::Idle, "a successful save ends the review"); + let _ = std::fs::remove_file(&blocker); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Region re-OCR refuses an empty, inverted or out-of-page rectangle BEFORE running: no + /// job goes out, no engine is probed, and the existing words are intact. + #[test] + fn reread_refuses_an_empty_region_before_running() { + let (mut app, id) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_draft.dpi = 72; // one pixel per point, so the arithmetic below is exact + // An empty rectangle never reaches the engine. + with_state(&mut app, |v, app| v.start_reread(app, 0, Some([10.0, 10.0, 10.0, 60.0]))); + assert!(app.ocr_verify.reread.is_none(), "nothing runs"); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the existing words are intact"); + assert!(toast(&mut app).contains("empty or outside")); + // Nor does one wholly outside the page. + with_state(&mut app, |v, app| v.start_reread(app, 0, Some([-50.0, -50.0, -1.0, -1.0]))); + assert!(app.ocr_verify.reread.is_none()); + assert_eq!(id, app.ocr_verify.doc.expect("the review keeps its document")); + } + + /// Review edits are undoable within the panel, in order. + #[test] + fn review_edits_undo_in_order() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(40.0)]); + assert!(v.correct((0, 0), "Fixed"), "commits"); + assert_eq!(v.text_of((0, 0)), Some("Fixed")); + assert!(v.is_corrected((0, 0))); + assert!(v.add_to_dictionary("Fixed", "en")); + assert!(v.review.dictionary.contains("fixed")); + assert!(v.set_removed((0, 0), true)); + assert!(v.is_removed((0, 0))); + // Undo in order: the removal, then the dictionary add, then the correction. + assert!(v.undo()); + assert!(!v.is_removed((0, 0))); + assert!(v.undo()); + assert!(!v.review.dictionary.contains("fixed")); + assert!(v.undo()); + assert_eq!(v.text_of((0, 0)), Some("w")); + assert!(!v.undo(), "nothing left to undo"); + } + + /// Corrections commit on Enter; an empty (after trim) commit removes the word; the box + /// never moves. + #[test] + fn correction_commits_and_clearing_removes() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0)]); + let box_before = v.word((0, 0)).map(|w| (w.origin, w.across, w.up)); + assert!(v.correct((0, 0), " Fixed "), "commits"); + assert_eq!(v.text_of((0, 0)), Some(" Fixed "), "the reviewer's text is kept verbatim"); + assert_eq!(v.word((0, 0)).map(|w| (w.origin, w.across, w.up)), box_before, "the box never moves"); + assert!(v.correct((0, 0), " "), "an empty commit removes"); + assert!(v.is_removed((0, 0)), "clearing the text removes the word"); + } + + /// Suggestions: dictionary words and the page's own words within edit distance 2, nearest + /// first, at most 5, never the word itself. + #[test] + fn suggestions_are_near_dictionary_and_page_words() { + let mut dictionary = BTreeSet::new(); + dictionary.insert("recognize".into()); + dictionary.insert("these".into()); + let page_words = vec![word("recognise", None), word("scanned", None), word("words", None), word("completely", None)]; + let sugg = suggestions("recgnise", &dictionary, &page_words); + assert_eq!(sugg.first().map(String::as_str), Some("recognise"), "distance 1 beats distance 2"); + assert!(sugg.contains(&"recognize".to_string()), "distance 2 candidates are in"); + assert!(!sugg.contains(&"recgnise".to_string()), "never the word itself"); + assert!(sugg.len() <= 5); + assert!(suggestions("xyzzyx", &dictionary, &page_words).is_empty(), "nothing within distance 2"); + assert!(edit_distance_within_2("kitten", "sitting", 2).is_none(), "distance 3 is out"); + assert_eq!(edit_distance_within_2("kitten", "mitten", 2), Some(1)); + } + + /// The page list's data: a page marked verified, a skipped page, and the verified + /// fraction the info strip shows. + #[test] + fn verified_fraction_and_word_count_follow_the_review() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(95.0), Some(95.0)]); + assert!((v.verified_fraction() - 0.0).abs() < f32::EPSILON); + v.set_word_verified((0, 0), true); + assert!((v.verified_fraction() - 1.0 / 3.0).abs() < 0.01); + v.set_page_verified(0, true); + assert!((v.verified_fraction() - 1.0).abs() < f32::EPSILON, "a verified page vouches for all its words"); + assert_eq!(v.word_count(), 3); + v.set_removed((0, 1), true); + assert_eq!(v.word_count(), 2, "removed words are not counted"); + // Reject clears the words, the session, the selection and the flags; back to Idle. + assert!(v.reject()); + assert_eq!(v.phase, Phase::Idle); + assert!(v.pages.is_empty() && v.selected.is_none() && v.review.verified_pages.is_empty()); + assert!(!v.reject(), "Reject outside ReviewReady is ignored"); + } + + /// The suspects' count the info strip shows: live Low-band words that are not in the + /// dictionary. + #[test] + fn suspect_count_follows_bands_and_dictionary() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0), Some(69.0)]); + assert_eq!(v.suspect_count(), 2, "69 is Low, 40 is Low, 95 is High"); + v.add_to_dictionary("w", "en"); + assert_eq!(v.suspect_count(), 0, "dictionary words do not ask for review"); + } + + #[test] + fn dictionary_uses_recognition_language_instead_of_interface_locale() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let locale = crate::i18n::current().code(); + let language = if locale == "en" { "fr" } else { "en" }; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-language-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("ocr-dictionary-{locale}.txt")), "wrong\n").unwrap(); + let path = dir.join(format!("ocr-dictionary-{language}.txt")); + std::fs::write(&path, "recognized\n").unwrap(); + v.dictionary_dir = Some(dir.clone()); + v.dictionary_language = language.into(); + v.load_dictionary_pref(); + assert!(v.review.dictionary.contains("recognized")); + assert!(!v.review.dictionary.contains("wrong")); + assert!(v.add_to_dictionary("Added", language)); + assert!(std::fs::read_to_string(path).unwrap().contains("added\n")); + assert_eq!(std::fs::read_to_string(dir.join(format!("ocr-dictionary-{locale}.txt"))).unwrap(), "wrong\n"); + } + + #[test] + fn dictionary_limit_and_normalization_failures_preserve_file_and_review() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-limits-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ocr-dictionary-en.txt"); + let full: String = (0..65536).map(|i| format!("word{i}\n")).collect(); + std::fs::write(&path, &full).unwrap(); + v.dictionary_dir = Some(dir); + v.load_dictionary_pref(); + let before = v.review.dictionary.clone(); + assert!(!v.add_to_dictionary("extra", "en")); + assert!(v.message.contains("65536"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full); + assert!(!v.add_to_dictionary(&"İ".repeat(512), "en")); + assert!(v.message.contains("normalized"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full); + let full_bytes: String = (0..8192).map(|i| format!("{i:05}{}\n", "a".repeat(506))).collect(); + assert_eq!(full_bytes.len(), pdfcraft_engine::ocr::dictionary::MAX_BYTES); + std::fs::write(&path, &full_bytes).unwrap(); + v.load_dictionary_pref(); + let before = v.review.dictionary.clone(); + assert!(!v.add_to_dictionary("extra", "en")); + assert!(v.message.contains("4 MiB"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full_bytes); + // A corrupt file is not silently treated as a fresh, empty dictionary. + std::fs::write(&path, [0xff]).unwrap(); + v.load_dictionary_pref(); + assert_eq!(v.review.dictionary, before); + assert!(v.message.contains("OCR dictionary")); + } + + #[test] + fn dictionary_undo_persists_and_failed_undo_keeps_history() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-undo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ocr-dictionary-en.txt"); + std::fs::write(&path, "original\n").unwrap(); + v.dictionary_dir = Some(dir); + v.load_dictionary_pref(); + std::fs::write(&path, "original\nexternal\n").unwrap(); + assert!(!v.add_to_dictionary("external", "en"), "an externally added word creates no undo entry"); + assert!(v.review.dictionary.contains("external")); + assert!(v.add_to_dictionary("temporary", "en")); + assert!(v.undo()); + assert!(!std::fs::read_to_string(&path).unwrap().contains("temporary")); + assert!(v.add_to_dictionary("later", "en")); + assert!(!v.review.dictionary.contains("temporary")); + let before = v.review.dictionary.clone(); + std::fs::write(&path, [0xff]).unwrap(); + assert!(!v.undo()); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read(&path).unwrap(), [0xff]); + std::fs::write(&path, "original\nexternal\nlater\n").unwrap(); + assert!(v.undo(), "failed undo keeps its operation for retry"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "external\noriginal\n"); + assert_eq!(v.review.dictionary, std::collections::BTreeSet::from(["external".into(), "original".into()])); + } + + #[test] + fn rereading_keeps_captured_language_when_the_draft_changes() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + v.dictionary_language = "ja".into(); + let draft = super::ocr_ui::OcrDraft { language: "fr".into(), ..Default::default() }; + let settings = v.reread_settings(&draft); + assert_eq!(settings.language, "ja"); + assert_eq!(settings.dpi, draft.settings().dpi); + assert!(dictionary_path(std::path::Path::new("prefs"), "../outside").is_err()); + } + + #[test] + fn finishing_or_rejecting_a_review_keeps_dictionary_configuration() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = PathBuf::from("configured-dictionaries"); + v.dictionary_dir = Some(dir.clone()); + assert!(v.reject()); + assert_eq!(v.dictionary_dir.as_ref(), Some(&dir)); + v.finish(); + assert_eq!(v.dictionary_dir.as_ref(), Some(&dir)); + } + + /// The drawn region is stored in view space, so its outline is that rectangle scaled onto + /// the pane: not mirrored on an upright page, and in place on a rotated, cropped one. + #[test] + fn the_region_outline_is_where_it_was_drawn() { + let img = egui::Rect::from_min_size(egui::pos2(10.0, 20.0), egui::vec2(306.0, 396.0)); + let upright = pdfcraft_render::PageInfo { width: 612.0, height: 792.0, label: "1".into(), crop: [0.0, 0.0, 612.0, 792.0], rotation: 0 }; + let r = pane_region([100.0, 50.0, 200.0, 150.0], &upright, img).expect("on the page"); + assert_eq!((r.left(), r.top(), r.right(), r.bottom()), (60.0, 45.0, 110.0, 95.0), "top region stays at the top"); + let turned = pdfcraft_render::PageInfo { width: 792.0, height: 612.0, label: "1".into(), crop: [36.0, 36.0, 648.0, 828.0], rotation: 90 }; + let img = egui::Rect::from_min_size(egui::pos2(0.0, 0.0), egui::vec2(396.0, 306.0)); + let r = pane_region([100.0, 50.0, 200.0, 150.0], &turned, img).expect("on the page"); + assert_eq!((r.left(), r.top(), r.right(), r.bottom()), (50.0, 25.0, 100.0, 75.0)); + } +} diff --git a/crates/ui-egui/tests/it/ocr.rs b/crates/ui-egui/tests/it/ocr.rs index d7a5501987..c1a987fbb3 100644 --- a/crates/ui-egui/tests/it/ocr.rs +++ b/crates/ui-egui/tests/it/ocr.rs @@ -3,6 +3,7 @@ use egui_kittest::Harness; use egui_kittest::kittest::Queryable; +use pdfcraft_engine::ocr::{OcrPage, OcrSettings, PlacedWord}; use pdfcraft_engine::{AddedText, Edit, Session, export}; use pdfcraft_ui_egui::PdfCraftApp; @@ -127,7 +128,14 @@ fn an_ocr_word_opens_for_editing_and_stays_open() { app.open_bytes("scan.pdf", None, scan()).unwrap(); let id = app.views[0].id; let ph = app.session.get(id).unwrap().info.pages[0].height as f64; - let words = vec![PlacedWord { text: "Recognize".into(), origin: [72.0, ph - 100.0], across: [80.0, 0.0], up: [0.0, 14.0] }]; + let words = vec![PlacedWord { + text: "Recognize".into(), + origin: [72.0, ph - 100.0], + across: [80.0, 0.0], + up: [0.0, 14.0], + confidence: None, + source: "ocrs".into(), + }]; app.session.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); app.set_option("zoom", "100").unwrap(); assert!(app.execute("edit.edit_text")); @@ -348,3 +356,112 @@ fn only_the_edit_tool_recognizes_a_scanned_page() { h.run_steps(6); assert_eq!(h.state().session.get(id).unwrap().can_undo(), None); } + +/// A placed word the review can hold (its box sits where a real one would). +fn placed(text: &str, confidence: f32) -> PlacedWord { + PlacedWord { text: text.into(), origin: [40.0, 40.0], across: [50.0, 0.0], up: [0.0, 12.0], confidence: Some(confidence), source: "ocrs".into() } +} + +/// The OCR Verify screen (Scan & OCR ▸ Correct recognized text) in the real shell: every +/// control is live, the keyboard walks the uncertain words, the inspector removes and restores +/// through clicks, and Accept applies the review as one undoable step. The recognition is +/// loaded directly, so this runs without the OCR models. +#[test] +fn ocr_verify_screen_controls_keyboard_and_accept() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + // The screen owns the window and its controls are all present (no dead controls). + h.get_by_label("Run OCR"); + h.get_by_label("Reject"); + h.get_by_label("Next uncertain"); + h.get_by_label("Previous uncertain"); + h.get_by_label("Page verified"); + h.get_by_label("Exit"); + use pdfcraft_ui_egui::Phase; + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // The guards hold in the real UI: Reject outside ReviewReady is ignored, not acted on. + h.get_by_label("Reject").click(); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // A recognition result, as the worker would deliver it. + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = + OcrPage { already_on_page: 0, page: 0, words: vec![placed("Helo", 40.0), placed("wrold", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::ReviewReady); + // F8 walks to the Low-band word; the inspector shows it. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, Some((0, 0))); + h.get_by_label("Remove").click(); + h.run_steps(2); + assert!(h.state().ocr_verify.is_removed((0, 0)), "Remove strikes the word out"); + h.get_by_label("Restore").click(); + h.run_steps(2); + assert!(!h.state().ocr_verify.is_removed((0, 0)), "Restore brings it back"); + // Ctrl+T marks the page reviewed (the checkbox flips). + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(h.state().ocr_verify.review.verified_pages.contains(&0), "Ctrl+T marks the page verified"); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(!h.state().ocr_verify.review.verified_pages.contains(&0)); + // Accept applies the whole review as ONE undoable step on the open document. + h.get_by_label("Accept").click(); + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle, "a successful accept ends the review"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("one undoable step"), "{toast}"); +} + +/// Ctrl+Tab moves the pane focus ring, and F8 with nothing to review is a no-op: the walk +/// never invents a selection. The recognition is loaded directly (no models needed). +#[test] +fn ocr_verify_screen_focus_ring_and_empty_walk() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = OcrPage { already_on_page: 0, page: 0, words: vec![placed("Fine", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + // Nothing asks for review: F8 selects nothing instead of wrapping into an error. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, None); + // Ctrl+Tab cycles the four panes; four steps come back to the start. + let start = h.state().ocr_verify.focus; + let mut moved = false; + for _ in 0..4 { + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Tab); + h.run_steps(2); + moved |= h.state().ocr_verify.focus != start; + } + assert!(moved, "Ctrl+Tab moves the focus"); + assert_eq!(h.state().ocr_verify.focus, start, "the ring returns after the four panes"); +} diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs new file mode 100644 index 0000000000..a0af321a01 --- /dev/null +++ b/crates/ui-egui/tests/ocr.rs @@ -0,0 +1,281 @@ +//! Scan & OCR ▸ Recognize text in the real shell (egui_kittest): the dialog, the run and its +//! result (skipped without the OCR models: `cargo xtask models`). + +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_engine::ocr::{OcrPage, OcrSettings, PlacedWord}; +use pdfcraft_engine::{AddedText, Edit, Session, export}; +use pdfcraft_ui_egui::PdfCraftApp; + +/// A one-page PDF that is only a picture of a sentence. +fn scan() -> Vec { + let mut s = Session::new(); + let text = s.create_from_text("t", "Recognize these scanned words").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + s.create_from_images(&[("scan.png".into(), png)]).unwrap().to_vec() +} + +#[test] +fn recognize_text_dialog_adds_searchable_text() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.recognize")); + h.run_steps(2); + h.get_by_label("Document language"); + h.get_by_label("Output"); + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + h.get_by_label("Recognize text").click(); + h.run_steps(3); + assert!(h.state().ocr_run.is_none(), "finished"); + let toast = h.state().toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.contains("on 1 page"), "{toast}"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); +} + +#[test] +fn recognize_text_in_multiple_files_writes_searchable_copies() { + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-ui-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.run_inline = true; + app.export_dir_override = Some(dir.to_string_lossy().into_owned()); + app.use_files(pdfcraft_ui_egui::FilePurpose::Ocr, vec![("one.pdf".into(), scan()), ("two.pdf".into(), scan())]); + assert!(app.ocr_batch.is_none(), "finished"); + let toast = app.toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.ends_with("in 2 files"), "{toast}"); + for name in ["one.pdf", "two.pdf"] { + let bytes = std::fs::read(dir.join(name)).unwrap(); + assert!(bytes.len() > scan().len(), "an incremental update was appended"); + } +} + +/// A placed word the review can hold (its box sits where a real one would). +fn placed(text: &str, confidence: f32) -> PlacedWord { + PlacedWord { text: text.into(), origin: [40.0, 40.0], across: [50.0, 0.0], up: [0.0, 12.0], confidence: Some(confidence), source: "ocrs".into() } +} + +/// The OCR Verify screen (Scan & OCR ▸ Correct recognized text) in the real shell: every +/// control is live, the keyboard walks the uncertain words, the inspector removes and restores +/// through clicks, and Accept applies the review as one undoable step. The recognition is +/// loaded directly, so this runs without the OCR models. +#[test] +fn ocr_verify_screen_controls_keyboard_and_accept() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + // The screen owns the window and its controls are all present (no dead controls). + h.get_by_label("Run OCR"); + h.get_by_label("Reject"); + h.get_by_label("Next uncertain"); + h.get_by_label("Previous uncertain"); + h.get_by_label("Page verified"); + h.get_by_label("Exit"); + use pdfcraft_ui_egui::Phase; + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // The guards hold in the real UI: Reject outside ReviewReady is ignored, not acted on. + h.get_by_label("Reject").click(); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // A recognition result, as the worker would deliver it. + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = + OcrPage { already_on_page: 0, page: 0, words: vec![placed("Helo", 40.0), placed("wrold", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::ReviewReady); + // F8 walks to the Low-band word; the inspector shows it. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, Some((0, 0))); + h.get_by_label("Remove").click(); + h.run_steps(2); + assert!(h.state().ocr_verify.is_removed((0, 0)), "Remove strikes the word out"); + h.get_by_label("Restore").click(); + h.run_steps(2); + assert!(!h.state().ocr_verify.is_removed((0, 0)), "Restore brings it back"); + // Ctrl+T marks the page reviewed (the checkbox flips). + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(h.state().ocr_verify.review.verified_pages.contains(&0), "Ctrl+T marks the page verified"); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(!h.state().ocr_verify.review.verified_pages.contains(&0)); + // Accept applies the whole review as ONE undoable step on the open document. + h.get_by_label("Accept").click(); + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle, "a successful accept ends the review"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("one undoable step"), "{toast}"); +} + +/// Ctrl+Tab moves the pane focus ring, and F8 with nothing to review is a no-op: the walk +/// never invents a selection. The recognition is loaded directly (no models needed). +#[test] +fn ocr_verify_screen_focus_ring_and_empty_walk() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = OcrPage { already_on_page: 0, page: 0, words: vec![placed("Fine", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + // Nothing asks for review: F8 selects nothing instead of wrapping into an error. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, None); + // Ctrl+Tab cycles the four panes; four steps come back to the start. + let start = h.state().ocr_verify.focus; + let mut moved = false; + for _ in 0..4 { + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Tab); + h.run_steps(2); + moved |= h.state().ocr_verify.focus != start; + } + assert!(moved, "Ctrl+Tab moves the focus"); + assert_eq!(h.state().ocr_verify.focus, start, "the ring returns after the four panes"); +} + +/// #1028: "Also recognize pages that already have text" reads a picture on a page that has its own +/// text, without adding that text a second time. +#[test] +fn recognize_text_can_read_pictures_on_pages_with_text() { + let hybrid = { + let mut s = Session::new(); + let id = s.open("scan.pdf", None, std::sync::Arc::new(scan()), None).unwrap(); + let (w, h) = { + let p = &s.get(id).unwrap().info.pages[0]; + (p.width as f64, p.height as f64) + }; + let note = + AddedText { rect: [w * 0.1, h * 0.75, w * 0.9, h * 0.85], text: "Invoice reference 4711".into(), size: 18.0, ..Default::default() }; + s.apply(id, Edit::AddText { page: 0, text: note }).unwrap(); + s.save_bytes(id).unwrap().to_vec() + }; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("hybrid.pdf", None, hybrid).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.recognize")); + h.run_steps(2); + assert!(h.state().ocr_draft.settings().skip_text_pages, "pages with text are skipped by default"); + h.get_by_label("Also recognize pages that already have text").click(); + h.run_steps(2); + assert!(!h.state().ocr_draft.settings().skip_text_pages); + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + h.get_by_label("Recognize text").click(); + h.run_steps(3); + assert!(h.state().ocr_run.is_none(), "finished"); + let toast = h.state().toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.ends_with("on 1 page"), "{toast}"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + let doc = app.session.get(id).unwrap(); + assert_eq!(doc.can_undo(), Some("Recognize text")); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), Default::default()); + let text = r.render(pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); + let text = text.text.unwrap().plain_text().to_lowercase(); + assert_eq!((text.matches("invoice").count(), text.matches("scanned").count()), (1, 1), "{text}"); +} + +/// Edit text & images on a scanned page: an OCR word (invisible text over the scan) opens for +/// editing on a click, stays open when the pointer leaves it, and its editor hides the scanned +/// word instead of showing it through. +#[test] +fn an_ocr_word_opens_for_editing_and_stays_open() { + use pdfcraft_engine::ocr::PlacedWord; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + let id = app.views[0].id; + let ph = app.session.get(id).unwrap().info.pages[0].height as f64; + let words = vec![PlacedWord { + text: "Recognize".into(), + origin: [72.0, ph - 100.0], + across: [80.0, 0.0], + up: [0.0, 14.0], + confidence: None, + source: "ocrs".into(), + }]; + app.session.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); + app.set_option("zoom", "100").unwrap(); + assert!(app.execute("edit.edit_text")); + app + }); + for _ in 0..60 { + h.run_steps(2); + if !h.state().render_pending() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let (r, k) = { + let app = h.state(); + let d = app.session.get(app.views[0].id).unwrap(); + assert!(d.text_blocks(0).first().is_some_and(|b| b.invisible)); + let r = app.views[0].page_screen_rect(0).unwrap(); + (r, r.width() / d.info.pages[0].width) + }; + // The word, 100 pt from the top of the page. + let p = r.min + egui::vec2(100.0, 93.0) * k; + h.hover_at(p); + h.run_steps(2); + for pressed in [true, false] { + h.event(egui::Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(4); + let opened = |h: &Harness<'_, PdfCraftApp>| h.state().views[0].line_editor.as_ref().map(|e| e.text.clone()); + assert_eq!(opened(&h).as_deref(), Some("Recognize")); + assert!(h.state().views[0].image_selection.is_none(), "the scan under the word isn't selected"); + h.hover_at(r.min + egui::vec2(300.0, 400.0) * k); + h.run_steps(4); + assert_eq!(opened(&h).as_deref(), Some("Recognize"), "moving the pointer away keeps the word open"); +} diff --git a/docs/images/pdfcraft-ocr-verify.png b/docs/images/pdfcraft-ocr-verify.png new file mode 100644 index 0000000000000000000000000000000000000000..14a5b3dd5c98ecbd1b8b95b4aaf943673630eee1 GIT binary patch literal 224336 zcmd44cUV>Dwl%&G1REk?M*#sL#DanprD-H0SV;6l5>yaThyo%K1IVV>C_y@kh=M0Z zJzft2A&4lwij*BiDFJDl5`)4n0((=xF_*c|y}##v_xt?;q@RRybEPY$+yK9w# zY2U%9=e76G>~eYJ+4}hJ|2uId^_p!)s=k57pybEgljrV*nYzEp_`}G4GS}31P~)(D zT=_TlhtIzGAAd|bX+(J6jO{Fsa5+T(Jh?>u$zT8R@X^!%?T@jd%bowPe}DSd^x_Nu zkKf0Irw`8h->**}Jmt0O{|`6uPoZ9Bolio!bc^g(C2pP3XhQjI>Nfcs;{AfOq>iZA;79Ba4+k%}8eCFRB308nr!{e($&Vgk4r?CA zS-D5eU9w(3joZ@R-hE!N~2G zv-ojtu2g1}*8jE}rUN{(p9MsmxRJT1>Ff&2pn3Q`FRwJ`rB9q(f6d9M=XXs;A zSpWFr;|Do2aF08?`TX$SGu4fa+9Ju#@fK|j=82zoQg~-`)=Dj@Oh<)7Pk)HuRmHp< zHD}USFVSeplVuk9gS!<9Lg}ta55#HHRX7o5>mVT5>yq~jvLU`ww z$5mRbWTp?`{;f{`&xnF3GtI$X-7t?MSQ5P2_w89@6&01WNxM!xeYeA+XM1~0idSKT z*F>ck&ue-_t+}N|w)4+F!*hejbNkZU|El{o{Q#$9baF%RL{;#POL)`y4Ws ze!^L~XeoA?X!M#~ue7ryrSP4K5P$E(zRc|ilIs}p?5Ok1eY-9FYRajn=SKrOMmw(^ zJk_T+E%XxO32hw*avU$lO#P`avlENP`Dh)iG&Ei~R9&Vv^_9gS&*Cz7dVDcn(7g?={^$i~N?f%BGcafyr zZsG@zf92@Ub61`G(^@b!H0^lPBJd}iux~Tu@XLkqI360<%ipf;d;ei2f0ddr=-&s{ z6kQo; zGvgn8`B$AJXYmql_y+aomKh;p&TMUNzCG}{QnPx=rq$Ng))NO?T#Zr`6|p-OM{a(M z7_qZk5;4oj*w|!?!_k!B2+y&?`vY6%D=5U37hUso%}#YNWxx0zd5m(~wCT>0lR4eT zoks5ZXN;b+KUFk2zi@D%@bb{qpjxMa1utP~Ra}_G#}dwK>%duM8)8pHuWXg-Y%Q7$ zkkG!e=+?9dKX@uHnT}2GuA+|*ld=1{*A%Yk7!9kZh*2F(;NRag983tnc(Ts7~bLb-j!mypQ%oT zsY^*g?~=3(BrR!^Upfxc2*--DCtgNh(vrUJ~q;Hw}rj8Gq z*6hkJkMu5+Z=(MR)^!MTR;*kpwLtOE@CBQ0EcTE|z{uM4)$!kZc8dkd`L#9qo06-s zf5r;?-`NFkp*Sdxw7jNc{?6uy!&P<09krVbwNkfRWgc2Dc~~^IMpbmZYC}q0X5HNQ zWDgGyKHlk)iMoyn`JrLQp=y)po!wtvW&a_dXms3+w~X8Qw*AB5@6r$K><*sWdQ4kA z)sh0||1Bv`FQ2WSkf39CTG%stZ>m|zu7cN*B0A~go}Qj(>I~idZkrC}M|Mn4TJ{vQ z54`nzV;{JhX`g)Cv1~z2Z#)uP#^fJf!XLdxujNk-6d5sIl51WsKIC?ka3|aJ1hf+; zutyb_tUuI~Z8^R_Z}6$Rl=y<4cx@2!!bR^DS#^DV+PQ6Klr6;=EsvqKEv+q_q%NGx7l&@tW)H;=c!Yl?hDz=x$MDy9TR_{jE-GuCv7h=BEY68 zeCoBNU`f(yWX)4gy3R4tYL=lg#vL5Zy9*h8WLMfhezzN!>+D_8kEm=et{q>FMiEGW7bveaq84*QJwLO4JpGD zWtL{qT=5ldmC?ZP4TbScg`N_H!pphm9vbu9g8G(-@xtahzOPU%qs$`UhgmM9T-eVw z%5J&BmE7d=BJ4OC3c=#CPP9^J?b z3;Hum@~mh)An33AL07nA|K|4n;hw=$DH04k`Qe0uiq@)##^t#9L9akr3cJtkrg~yT zzlT{Q4xf|XYQ>*O+$a7kV`@2~yIfY*#{TXK!}#!7p8(va4?^k#%t1uY==sKJ_!~d| zMYY;<1v`|c8(QuSY(Y>>1`NIA866#+S@)kob2vD`li9lSkSieJp<%TeBXzCy*-}@u zri!xVLy+_tY4wQdA^hiyZvFoPm%-*hdwINQQ+gn|)QO(^0-a!CfBx9RqR*L}1F8H= zI)dahiR8Cy^U8|v2mYaRTF|{dxq7?W*b(O5gRgD*gF{n$8M9tpJC~)hs)Fz^sSWd_ z13N^|tu1cSUIem>!PZZ4&nO6`gvcKao;C?}?sQfNn3c1b!4vo)6zwm>uD?nls>icq zWL;*R$RJwtaFYOEygiivq^6%!f8V+M8)QN} z9~F!&r?Z!y+g@2yG*NS==)?_V?mhA; zcH@hd?0yQB8uv~dwMY|CNX4QSbc}DuZ8#vFZVnt<`0UxUi4Pr9(n$Au=RLKgP7f8h zN!o`dS2YeS3J8$njQR!}Fvt`6rEBt6q#NQ=0MQ|oR{P&LO@6}$_o)f)bUgMCx2Mc< z;-cS&rXYF!J(r&RC5fEHv`24PR?&!^Fa zL-;oS2sIqYA%u&_ccd&SuqA1qdoM6`IDz@v9rtwa9!Z&%m6r#Ey;vlfXOp<^+x1Ci zU%CGJ?OMN5<uR-k#oR9 z)bY<8*{!;lr}uayBq#3s&uVPC`st&6?spRznd`d^qLmvq34V?*?sKBZG_~e9N~cJI z_EVb8+8WpE&e_iXg(=?auq)JA%OI|Nu0{mv1%tRtT2tm)&MDKU4mZ6#dv%u7jNR_l zoA&J9&$}E~E}hfJgwLAN%iN=TxiG_OA?hNXhSy80B5K*1X)q}nS7vW!Y`WGcutU(J zV|jH5*sHhkU|LB{;rm@81TR(Qn08h42mYK4zHoQLbCbOu9&4X(VvM-QRCg*d$+E>p z_+u;Tu~?a6U{XEN6}PtfhP0qHX&-IW$_$%scK!K%H8NsNKz@~}D5^v3Y4+FGG3)~M z;+1!`5rsJYzGKAHai`OHm*iz8>&^eWky#mc9zRxFXWnX$E7 z--e3PoCA)3-F3Wx!tLeD_<={y7Kb>j;hA{jIA?#yJ?#4yk?FLh4YPB`Rw@UJP!4W$ zS9Ytv*_U7`;r;paMScDJ$cmc*Lj^Gu*V0#~36`bB$lP5x^=HgfB5kPdv4ISp+EkZ! zajTGyzn3^PqtiO5M@N$5G`#=c96w*Wx&P)uUdmgTtBSB8w_Dd(vV$j7e$3DQ& z!-s1=%Kt8HxW(+l2|FIp?l7<(P=r8|HobXwL;0pXJNf1vq5@Q%H)F>B6>}`dK>&=n zeFBhW0g!8ciAugjnou`wx2t|ioft^>n;=P2aPr%?hb{QhmLa)81G%mRj#TH@x~>dH zsZpD3aNl2CrdD~*e1RgTzMr$cNMu@6!?EBU?ifDoU**e@13p1g<@w>K>XFjx)sV`C z|B*5FImNFThnp0oGWc`!xG3n-Ufzf+pT@gwZEfWYjRvVv%~&tsRb9UqF@AMlb89OP z$t>G1s>x6OS`C&r*s%4ciT9P=%^e-qO4BOMt`=zqDQ8iLoPJYOi#lH%b4;a`v64aI z2CBej!U=U{{%5HFrsv^X$Q;9#+@l-s4}8u|Ux+2%WC&#X*xy-8O8va2<3h!fqgh9C z9B^jlD3$3BW-NY=NR4(XY7MPYzmE*5F4BKjA;o)g%S<&eHCRFQa*W7Ip8i_vYxfGH zhE3j7xei;#E&TTZcSp%pr}GL^oufRUKNY{f_NPUAB9eC#2wm6S9p!)9J;Qu^`c>eG z2(z)5F}EA%6dQBOatR_NHU++II-2z@3U|JVqhOI!bc=^uHrecgGf-+kY6 zra<1RRrF$nK(ui2Ch%P3O*P;aK8uurL9c-EV3vV>%iD@MsEHfT7;V!avuD z2a2XlgG9!`VXa<831MbEB~0_7VZB&T3IA@=3=0cev0{b(JZVLZYyr~t8Y)?N$`)P9 z1bWJKIHfw?r%#7?S_m~QZ&00zmzeNqKn!rY_(xw~@Jyae>eR>kh^1y1_mw03s0l7R ziNDN8#kEpRpr}@fA=XMs)(zAVS&MJMmI&QW5K$R*t~@Ow{W%wP>P^JGtU+IG*UDn)tk0=V~Y6 zYKb^z`rDvBx1ei7lGLGIFjI{KarS^eWr22VGX!W2lt7CHa|DM zn0BR4nlf#!G@4%g*nb-?9=s0s68w29I5WUy+@gz0>x6ymCD1#TFDol=K6;qHA!o6X z(c*mdbVDErG4j9x#}98DFW$GDd{^6VpsyTK`%6ycBpvAZ=C^Nq>HasHnRTDnPdIVo z`*i1&C6z|$hV(_I{8m@}J-`8RCm5})+}tX9Yd)t>7f*ULI3O!JxFcWxUoQdj;pron ztm~RYha#Un8py1hba40Z@)F#!9COJ>z+LY(e88(LrATs6NF@*l31IbO;3E~L^|wYu z;@Td%p_2evc!yX?0m}c`Yz6_D_~|EPigBYvq2xKaN7eQO;YO3&ylyOi?Ef_|0l0j9rMdY)8kbF<6zNfTfk7@p@0vD`CTf7>n)Lo zOIh>;N6>=px{k!O@jRz)>)?+%{@u!H<@96x@uKN43JHcx z#?)UKrF)Fu7B2}F*bAxFA&hSBGfKV+F_4~g{NG+vA!Nxrh%Xbsi8`^$s zU1>J_z^>muc-UC-Q-*j8ks_f+ZLS(K zk>ZK-5_RiMed{P(?R0az0L;>2f-RL4`w4`+V2dci=`~G3AB<4}l(ZIpI7n2)0cM5i z(Wre-1c*8h%c*t z*P!En$M{^}6hc^dZBJgEtHD^IT<80iY=(~MVwCLLiBFRbbw1|no3w9^+K98-i<@<7EfbU?Kwkt`yQqzSk0n+3VdVX0A_5|1d^-Y-kr9eY?Sp#l`TBaIR}cYIr^Pya zO-tdOd=Tj7KMKLW|B*lbtY|J{$LUTWn%?E!1LqWEP=tWCJfA~8b%yiG!nLM!B4>fV z%zJk)MGTTczkQ&fJ;i&)yQk5+lGY1Q49p>1xEQD$yhG7GlvK6;!i2Qa`b9yVBvNI10--2lNv@eAq)=V!VM!cmwiqw*E648Dj#p@{d$D|O z{sXY$yvRBSZeYJ(#BWI7}#$N&fi z;Htn)A;WlhdV2rx!zIrtL#>VTq{}uS@l=akQkGoSnle46eRm;MH#`%K*)|jQ(1Ct; z;5lh(gwFzoli=+l>PebV;nIK8DvjQa1+m~}xz<5P0ZTtDS3 z01m1OJmt%?Geh2->_ov0X~?ImbS40fMH~)c+Hp6x$hk{?#PhgRJZs0g&cxm2Ds z%p|merIT=W=$1tO+-Te3`|l>{cqZ1$UD>zITaJpV+FgY`yH1mYfGW=*f8bf63Zi)j z7w7X^oX>a@=nYCqD_5+b?F^i~w5In)#ZlSfPakPTnQB{yJ#>=upy^Da&3aZTbWOGo z1(N)DL%UD3IkIA2i1T%b4Et<-(f$%v%Sv~+^E*WH#b1=LTRk+H)nXw zCR?^Z8^o$bMQvu1PoF-$m9)#5TQ7w#0JTHUW}pznzxZ8xBZ4SCS?**VaPEDUUAf4MuMHgdYV$uUm;DbNP^W)e)Z1F3i z+}zAZ_OZ=xeO90x53yMFd&H_g{;*OSP2u(6kl~nP!DNcDr8}EAMoAw zcTeX-Str=t4ahYR{uz`1;$glVGkDkjI zMV841GR^+hBeN*@xrdPpL~S~^sdp1g&*VXS(Ewx5bS~pQiMn%VyTugG@;sDD6Em}( zuVckoIoar|&q(i}6n)MC`f zGN$5aNp*sb1f-+c>uOFQq};ur(fwCq<^e1ER(1$Jw7e~_O26vU zU!V2u+UwkY=KY5c0Gf&=P7@VQXGY$CH&`&bHs=5kH&{m0iA0S0^lbvS?YtBWVp*By zqrAR(P}30w!*Dbj2Mke9>*1+>84AD`hnqI0ar5vks4zD-`^X16oml5zX>f{%vR=WD zXB@QsRH4L`-dCz|V1w~L2etspBYYyjQbr#6!#$bX4;fNRkXAw!(Qo(7qe!Es1hwXF zCJF~sylaGxL&K?H`^$r8w!U`wZhg`x9tWth;^;;7CNvuL&UHkhzcp>TFiDOfNQw{&wM)*EZ!;mKg$g`xc=}{)O{K=LQdQ;2^`bv5Se$>P=QPb z{uZLq0!89a>Ql|DoPgehf5Q+U@Y-^t_|NY0wA^JJCJkO{9Z*n?iT30^-`w&)!V`PiCqBl{h*C`h8+JM@w@WSPlj%gn} zc#!U^w#V`hocMv^t*&Q&_@NxE#C<`WnnV#gV37$DuaykgBE2~_6V`Y^-%J@|A1>Vi z?@sI5#dA?B1%){;75CUpX;Ekir$yHOf`mbmcFZQ<+4U2J$2`IIgW@3hj)b!?ud#bx zu6c)qKxDf4VEU9Z#{MJ2)>Q~yc^64^t4Kpzi5>Z*D|BG<76&Co6y|{)KK-|Rg=>n0 z4~gqHBPOUOrJyB+Mc@JL6|Id;dg8!}Zd zZterriA@8;%ZaP|;c%Fi<3Hlb;~-5&A*ggDfkkFJ#^g7;7i^*DD`ksecd(K9)hYQK z!GSd$_7#EDbisKSusE3jIo>#4tm^2KD#6?$ek25=g9UF63XkfI9Bux6)Lc<&Yx-3J zSG<=oLGvhu)$dOgZVG}UMjBW#R0gN9yG|u*3SO;2 zq!xaT&C7>GayB3DR7p>yuUB?5=L;q7It{?V;HXCr=aoH|wmAywu+~Mjj+TbKiTsL# z`MN_Qiy5Kkl^pZ$Sv zsH$g`DIzRJMvP(t+IZr3X;aG_+^Mdbtv7l`ujb`TaQDGr1*QybbrbhpzZY7bZMBfJ zEvj7V&gAu7b$qu-j1+hlDP!siQGLy_w}%1>aS6z~V4IL|H0auXD4wD4wup+XkX8J7D8Lah z{U+*h<2#KnoThr6gtW*=)$Af)uQCe-srjFf=KFXCO99`3osPzn4!OYJ$A3Mv9$Y-< zd04($!|T{5wV{t5g7>AIcd#AV5?#gZ`2 z^YNDs8f;335(N%Va%dGqC0nUtSXi8NXNFJ$>5=c?pT%vq4av86o@JR(h^YT;yr4zd zF$#qd-<-Vcsg+ewPyue#{`a#h8AVp!w&H)${M9q?#XGc*eWAS+iSe_X zSFf5&7bgSay&H?Z=6cd*Ikv`$>khA89s8om7X%BiD{^AR(Y+6@?!N92KRd8PSrM59 z-kQq^U`@t8o=_e8T~6>XYx4_L79$(Ug0vgr^o*};ON@RzS_WwbE&_+S&=OW56Zvg} zB@VmOKJZPF2nB5!Rk11^eEE*&&}r;ZWZ8g2LLPa(DW3FuR1gjK?RNtg+ z1BHX`Vrck^;Ghb_poyjthrTB73)rW_uWlaf%dy#^GM0`^{5m)lTpb_vg76-$4 z^8)!wj@Ee*QEG<7(v~ZyiM7ZPmYN&bn%fV<%3I>iaaz|Vtsy9R(KsMGw-ciC2Y)av z_JdCcPEle{w3Ga!(z|EBMQ;PW6mKHO!(rx9kR@%-Yy7cSKepl?5_{ZeFQ2Di0x2dT zdBTloZW0%O#2$486!r&5=)?y>`H5GX7NilAeqRDxnbZ_Y&!3t&bxpq{UhtfH2o{yt z>xv;1IHJ6?@y*0^tph3KRaq6-RwaTCrnG;Bu*Hg`- zTYwL5lftnSoE-dWs3#dh%5Z~BW*vlPRL1WyuDt&-w@ z{c{;U8^4624mn{}_IFieTtT78fXsxIKzx%cF~iY9!3?&sWAyJ|^nu|MwpaWj4o>`| zHNQ2YEMZ{N{m@+ziBtlm9GMyc3hx2<1fd$I378hbm+fN!$O#16wO*q?dU44aM82?V zoc8`e3-~7)@XdE}cHlU{273ae!=0Q^PvA98eBrD@eD)3yz^G1vrp54KafnJIMwe_~ z_{pSXVoc-kCls(A%6{xg(0WNMHgVYj0SPhVPh$Gzf;#2Ez2bVVujmt9g%htGB)1jD z76iSo3c3lhsM1@@pnA81`!VgubR(JuU|2p8vd$?iA40zspA0ziiKvAMI=jxJQ6P;P z9Qz->F4_7tOD@2~Gr}76gU#DIFZu}?D)U?7OAihTf$PEAW2YoMk`ZK`WDMJmbOJL(Y_WvZN zbBmA{#)n9-82oR+1k(MY?3SC4)=c04=~_Y#`Xc=*9KP!hb;8$^($diZOQDvcV)KTS zvn=_nHtMa$t%!U^+S?YAjJa=pJGM8S#rw(H-7;Vdoolw4g!tR8V@hz zVxFkpRDvY4hRD1yVxm!qNX1%R3ZI2Y0JGDi7%}N2q60zr#Uvs{LAE@vr7?q)Ve+h& zfLp$Q+RyJfG0p-MqK<^nVXF2IvYkDZV>>%F?ZgeRbjX_ZX+_WPs1DC^e0S9`4gj#@ zUSsJk-}Y+U^tga6=yJ||j zI_d3*E1V|2v{fV@Q#oFyn}{rfZ9(HJ>1?PGk6w-R@PYUM#ZnMefxCE1l|<@uCG z5;cgtj{rf z^rN&8WOg1kNGa55eYT(if3TN4F zLW_P)zJWSq*!u}`;{jV;D;XuOk|K=B@KO#u86ON_{CYIiN^+VUYlw^yz`6PzcB};= zU>b?m9Mp*FN>(zXkqg*D#tjZpkY>Y8PF%8%1}od3=&m*mW;vd?(*R>}i zkgCBo1K3+lavdKxl0zK^b;^x0krc%ivI7 zL90SZLGI3CSbkmd`~+Y=rYJ1~g3yB1vE>Drzqw}bofZ22{Jw%}I4beUHzum|n%F@s z50$(i7vCa3m{VOngPtW6vI5hcqzI1E1w>>O zAFuHMX@llCc)?ZB5$&4JVl90+RlWjn!BzRl0jh`Xb?-8n->!W^i2>F)B`6n4X4xVT0t1uA7(HeOM$1pZ;P>*>HFmmCLlt$=Ch zK*K_p8mkBqA!Q~mQDEWEF*K>PP7p;00z`r6MzB2`Z09BY^|$(g?=}CGL&dE0-&w+W z0@dM43iVZ>v@^z^X4FuLrXQ^d28{eHF!RcS3b~<8Hu^Xs_IlueKHL1qAARIhpo-!7 zZn*U&*cOHaRWINGtm6DG7pQI*@J9nJs0XqjZ47ec>$L-{U!oi-AaB+CLC0>|o_<6@ z-8)fAyc-%XvMCRL4k{o9!goSp8k|UgBCtK^SO{!5T}y&iDbAejk-Ay~%z8HrnA`nO zP_GGf-l{a+qMrboi2TT*bTqr}Sp0Rz58e&=Qw77zIDkJZ;>n9SZ~dX>;}&VqH~NF* z}Xm9TLzLY^3(%E=K8&!S9ffeK;Ba9Ps}d%Xhfm&q@$!nG2IYmP?V8c>s3NoTWVM{Dt-tpZ9|=% z;f>2tkfvYVl{6xO3}B9jwNPSOiPzy7Slpr#;Yjc_V}oED9es@uZCTlBp2@4xl>ou> z=SP1}o}lI|&%xY7kCuf>g;0-_=b-Noza71y4eG6uK&H?e=qSHuSD^{7nM7qbV(!Cm zjH=LJ)v#$*gxpi?bKSe@5put-`T|(m>NILl57E%-^IB8GpJX$zw*7ij-FlK*?!*uE zH#b+T$uF%D5hp-W{K6{1_I?j}X-4csB08UL^@MBTC2|2iDEA;Ox%ag8TUtdvaN_5KJQ+wrg}V; zO7G6vywabcsVQ06~E+vJ;*+4!l2|x{fV47)_$#Gjq6s0zUvh`>{fJ(@t0aneU>@lPG zS$@qkwoJ~iAIM+KGZC`PFkqLC5(k8l4oiolmbT7@h8AVqUY^UTG2U-0g{E0$#B-N2 zDl7QV(GD=2Mz_HAULSXOJx3Fiwf;P1@T^YcYrfA%e%;G(UM>mLW%-qd`7hS2Lgyd# z!|sOvLb?V0myi9`;j1NERGif%zuVy`JyGtEC-^a3$*4@xkID9p#-l6-GmduDt@PJP zG!nqCM2l95sWKt*O5DGj!H)hT72O6;hW7Eia}_>*(>#$Bf8=^?S;o+lcmJr6 z{o6b%q^&Ul+qnhLgaA}?nQA3`tj79#GyzbW&1%4|Lg{V$p<$R#zzVo$(yE9cfdj6( zOm$>~!!BxAf>DowNDMlQ@q6#0R)fu)E0p!(@7SrK!5(L9uYiDnW6Ni2krn{f!&rqn zZurPdlkhHZ0TE#boGXr1kZcG7@DDpd)+O@Bvpsji=g^thIB@>rMQJJkDfv^=33!d_ z_I7v?rnTdt7=fn};gzpv?FZBAAHN%a0a7D_Ivw{RVdnC0%e^gq*Kl6~9^JE@XWkw~ zeswM2V{?C{M!s`i-By51X8uS?5023k(B{Fem6^% z`wLBo^e=5l9r*!TaC@jEzkd(dMsoX`-_s=<q0BGsm{sksE*mFWL8vJ3ocf8KE8K@zq>rE$?= z(gU6Do4=6~$e{~VB~nk^alhx=RckNLeP$(tEiR?TpCXnUR!?HdhysI1TSvw%!Jdwf zb#3Pc(rGipG0(m>1d^2ezaAET4nc+5=jM?()uj+@LV_h~;AeiX@2FM)6IX58!`7ZL ze&w2e?~;OS$iEOqH_?af1+=e>ZLLo33VRP^K~#xG>I|&h>@D3TE4RbN#ZBS|vSW2E8`sn_Z!&64%tAxNPG44NELfPzF64bRs(xo=F zO?8SNCIqMcB@7`}e5*Q=8AgurnJD!>Q0!fcjSV!&(LvW<*{8>!Y$yt$ zUJ-VWP^X4Dv4XMWgP#f3nlyWRD3&+SnD|c)usj1UWC| z7ESf3b50tm%9rs#qTn zb=JUMphD2%Is81-4RQV!!!9HmqHRcbGa_F2xu%+ARv|*pC3Gq!kcq7qmh3lKb>zIJ zE{`6fL6BQ3$aG+dYOH!UiVTp=nnly&MO+X_YuNTKd=<=gBV^WLL)94v@r-@wMUufR zA1_b=l{2h6aI(PXL>V4!G5&c;0pC?-DWVjGV;Gp|%Z^bi8Q|h7%c+moyH}5cNJY4$S6vsz&-VhI_X>Kjn1d$`y1LsXM;z1a-NM5sA>tQHpVwDAI%1N zi0(h!d2%&0VE6$R5^rCJsLjSwp!i;p#}vjkgHN&KN<>Q}U{Q90owTz8Wg!8q(k22vsEa?7{G6{VK+Rb*cyN4P@+q|o8 z?Szdh@(K^kGpu$#fPF9mR%I6zRT0~RPKV~^<{s~ufE5H`e8I1u%t2!S!w1EAc@i!i zgdOO}DBx-Hz)1dpEj5XtEG1|ug#Z5r`GMBl-+Sw}A@ScZ-He`r8YhAyMYD|~ilUN< zT3DYwOY(!lp$0?a3Tx03>r+R5yDtFIXv=9y3fv55qzb*NPsjw3GR#a6Vb=rNDucDH zqJSWu*uFW_Z?rZ>YbMjQ1ccMz;;AVqY@#$Af>j9gWJK+yo<}8Az$n0X{feq)t*hC` ze<~0sO5q5`3|8s{$j&*?jwpWv9mO#&UPB@KK2JBKradaZ3IV2K&}W&;gGg#fl8M`+ zqA3T|`s+zLq1MieIKYzp+tF3&n|GqzI-Kb;8t3)z2o0Jzh zb{$`a=Bu7LqJ7{MP@FY=!(s`%^Q$LpRmYpFUX7ZQE8#iLSs71t)I`e}V-n1|6&3Ce z%%RHU?MZ2;(u}-5qm&5z?VC9 zL0+?SQ{3Xp36NA&-e}u{Lo5Gp>Z1HHr?nP1CJQI4m!GxnI+BBNWDm7FM>CJ z&b40tQ>T~LN^zT7!Sjgp+jk`y|M=pN$@iS{6jk0zC(p=D3%%dWU4=ea$w#~CO%!<@ zc4@u*qrAR;?2r(7Y%2eyN`WFvSh%CpgHKTb zYp)pMd+xq(4+(52I%6c)V}2eu2WMhFpvGESiaHYD_S^Dd3GPOE3Pr01A5du>rK%Gr zf4c_Ch-1;@7$9$Ly#~gS@FRJL1a#au&|?jPQF6S1G&sNyw>G-~{!OimDAVu-tO5_; z5ZxS(T{8T=SEwc?P4*HrY5QWM@$oR2VJ2kJ)ie|)^H|ccQz_jL9vyV;nO@xoJ*FFO ze+BSx&qzkI_wX4=&lSSp+zJ$*Z(zT4L$a*81_p%0C-2@L-|_?vtA02V-!bR`+3ejd zgT5I}s=55}&1GYSZUv6gkjAJ*F7J?j+UONt$gS7QaoAVV1Fo_Q9!rO;1=m3!L@V?f z?cOhqR`IOL1Nv_9$ur0-x6nKiAQe>D`JY=L-4&FHkx6L{hHu!ooQc z&SW$A5m4(meOGb%qMi*Eroi8wq&@?p1msfVw?AmMwx-ZRt`teGP!sDH7jpema=UM} zzdKCLs0Z0Tqwr43gAyoNyP1)Z^$_X=2^=_`)}bAicx4Xk5-X6}XfgrLkI9ekmx2r5 zFCnU$tQ3h!=N=@%-w9ip@zK0PGx#vEnA6#YshM|>VOg1W!OD(g(f~jW@Q`bXx}pO8 zI+^|Ysb0v#Lgbz9(jXajDiN;*Q$0qV&WU zxc%gFW2kI7%?c9x<^X7KjVGFlZ9IhEzntoOZ`_@S4>(F<1%>EAn)%x`n(7lg(`NU6 zp^~1Un+@V1N1ci>VdB~8hCRmQ|FON|;!@z4otw)gb{52H8M*9)Fre}#h}G)=x~wO% zK*XwY7+f)fKy(G+l zlwp0iFhj`cqjMga<#TK+nAU#T%>|s-aT9-%wqvld;qV7qH zJEJ8=P*Wl+*L6y;JDLwCnxbcx-Q56HH-0 z9PT<{7qF!&dL<8Coxd4L9Z8|4zoS{V$!*!A7~^MxjKM=?x{D?q7?Hpgd-39SDi>gS zR2%+Ujm!#`R9-w{^%~-HODJ>FhO+2$vA9MxPCv@Zm5`Lgf$f7I+Rqbc?+H9%mZ194k{hD zm;BVw&(uo~Az zRrEcSW8J&d@xg!$e(4~+RVi7!AyKLMUQVbCHC}XAIKJOVutkP^E|qN0FdJLJlv|LG z1FBO#AoxOZnWIS#{8uEXyinMV;7T>KOxC^-d=;H)8b{!!qplP>{Ts?Qyd1qsa!5B) z>p`4fv1_qe%5#qp@Zh1ykW^7`qSRCYP3@rojY>b!!gQluq(83%J-J`mcAeLEhU0|#np#meMH2#qqqjaQfpy>65rr-xN zr{8*fR`QMBh1#B8$HC+8o|Z}t9YOI{Kd z?6-m@-V_rn7e5>PtR}1lFi1PXEcg=%)F7BXeDG|C(zHMY{1lb|x{(kvj02=S{e;?{ zPnAxb^HT6?TDWK9w$m%=zH6u;CwlAL`w#0jBgc9PwY_AD(K!!jtVMHF{2ko?-Q>>D z3=ctUnA!D54<`WR)*?5!dX3xzj>PHnT8+*X-xsWwLWZ5e!tqmNl8SX$gKm6ufL()k zT@5n?ym}E`mKufpIX(wq-Gb{1pLS#Me4y!bR02p*=JW8OlA%-1a#m|dT=}A(b3(Nk zH2Z{5d3zLn?AW5xAdB~)aeRs)tZSXVXhS^QHE^aRa|1BKsSa%W4_fG3ab7Bmgf$f^ zXDKd$7F-v6Z{HkH9K-}EQMj!@Qs)vU{s5|jmY`!BNxn4ZOtd!Hrel=ajw*>J=EFc3 zy7TnNdw=SfrPpwYI=6b^L+xc~1W3H|Qn)+lW2ml_?S$0Q+=pVoN+uplQ#b^RwM2z| zg@0OG-tg&P51GM+V9@!1!Mr6QP-;(5Gh8PCttZAvB`T2>gjCdH+Ni5nVxJsE#fKgV z>RSVkN$p-G2+)y)W-eg{+N-lGN%-8^w1^1jXxc*<2NnP3WZluMPC-nt87w8^Ht1-i zWJ638E6>61@!(3d&iSD)GN-25$A}$_nfdP^`u> ztRoNd(*F9xogd#Nur^i9GBah$Ht>hY z12Bb?h@=S#yc8Y`+zjkVW|1h>d=p6KW~BejFGg!<3DH{u2!Pr<@oQ+yOe@n=6>K1B zEFLf1xJL&vB~2|_s;`oBAk?{=P#igPb~qxn&ZevL7wJLTK26FJre%F2^yIQ@3Mypuu*T()0xD&08RIKKJ-=Qs}lkwe*uoh3@Z0Z zVJ&5-O$5rWISm-FTz~iv9Sw0z>4p$Gd%9%Kq=!g@5vV15Q*Ox3x9dx^z+(b5Omg~@-tt6<@1n~xs7u@b_;44>FF zGhtlJF<9YJM za4fm{)4Cv=EuleD4yG8ygP9QkXAHTkcezKAINFWY4HE{Ik)lcvDDi&V2mqN*p`W`u z52o@Ibd>>2C=si>0G*-A67XY(7-FRr70DQ+!}zxQ8wZ2ytV}j=qx)F|wy?H}qopR= zs8)@TcTj|xD*LH~hbAM$HrY!r%#gZ}UeK`=L8@{#`*WMZ&xxggFaa}8qkTX`Nf^}Z z@1s{??6vPAkrG11Qc8O)5+}m-N_Z(*C1Bx<1vRk^Ucr5~{GU&z$DP0=v~=pt!h8^t zKH+A!C`560gMFn71;l$}03qSY5;WdQiE%poA(3bOGx37fpuPL?71pZ4SUETq*iHqi zAZF0gJS64-&MnM=pur$~Je?_>5PAZlDVGv|T1vIOkU%E2aij8%r`CR{P5?!X+_xk1 zfb!D~Q`K70irQFOw8I^0`e#2cK&O=~O^oY-DM9Xb;uluTSGMXiN7t;z_#8@lLYlKf zgX}aJ(%x#(0&pQ5SbfstGYHqSV2W6V88!LTGNC41r8cV==23-nzh7)_*<4A;6!og9 zG);lChy1xQrEY1c6pdK?8?8(#SohA1Ee_o@o&JTo_nV?p#7&antYjKCWfc|KVa_VX zQD|l5l2Ne}ZO4b| zuo4>rc?U-Ej(VfR_N-wI>lDU!RvRh7JVt21kgnIVYeo?hh++$|1UG5ENiWKs<q74c|FOiW~9T6GSZ)8hZQidiRR?-iV*JK`g z0J<&r>w;U|PQ=PB^fcw7@hy@i4RbK7q7%@#LQLc}4Ipz~VNIrF63zLD!={{aYmSNH z4xZMxB++7mkSx&*%}w%Ty1;-EDL3};q{XalfBevvV!ib416?n zj3bz$X>I=3*EEsTh#|y-ff?w;x(tgJ)K0A7qczQ$AaWszjH7LaA#{LN_I{eD#i$!R zF!V!-yP6=@jk-E{M?Bs&swT+rO%xG{M9zq2cr|RPPYpC6@8XbcsW(dAxDE0tMmB|_ z4kH~H^BbtbYRTMVwLozvJf;^gu?)$75%Fu+LE%x`Gy*l10%S&%y^b!x@CE6H9-<3Y zG8l?S17C>BAfYCZhzk;8{i*R4p&cbZ@6F z;#yiHLen>g&wbf63?NoO5*f8nDuI0uu94#uhf(t#DBvHBjC0ZL%_<1!DXQ4C8p9G2 z@EumNM=Jmtcj4VBgU?)^l~KCD})#`(1z7Tl-VZ)g6?*rh{^7- z5t@!R^7vMm!jJ*928N3v2OraxlN&#FK?@z9V3#nA8txx-nb1*tLF|+FVB2Oi@{xkf z8uXA!r?YC$nPV#4mxl8+;Zz`zs*~t(8as8?u|KDJcb{k(JojXwp_zc+5hB_`2&jzG zrhqI4VBljqL3?r7=D|-O~ak@J|o~x&5?3p@M6$K%Y`2@(yO1U2E~Er*R6xjcTHo z#49X!LN{3zP~iCKGI?V@%_l(50HQGxz`C8?L<>ra6($%OTXB_~e%)l(^#eiTj=j5= zq>EdW3;VKS0@t#gaCt8SQ-hP9)m?OIU9YIVk%%jP843FDM%lG(Gj6n?>_rt z2)tIK&m5*Fg?>Pj0)GrsGKP#{EdsZvY&Mb=(~IWvb!^OKy_)NXg!R~jLNY_s97`z# z{SXkbF_76p$^O$@1_HUx>-;BpX~on>&j$&1k*xoRo}4~^VtJel$xqm|3@PdJ zsO!9%&N9mJV3>)@N#u}53WJgVc4H}FeVxEpI>dEGV}>4CG6Y{beZ0(&&u^aE*gw_k z>9D--2)RI<}P5AD=1O2*&VqkkD7}BIx(Em*4AN|#p1~dmE@Y0 zD#km&R+XZ8qpSsGzCuiMhb7ZVj!3e$ttH+rwK1Md;F#Tqk3rtn^!`qU5F}z4L!&EG zDYW;vSoGZ?DsnlZ4{NhQBk40fZAp6V8lW44YeA2N1&ggE@UbnRHph$fq5m~0eJbpZ z@Qa;am47o`V}?{!`FVrhXwZkPvMSKweTWVQMjEaeqbc0aCgepQt`k5@c^T+5+bckR!8`H7)2$7|J7C`O&9 zfP}S2_v(q0%+8}rwP`;kAnY;U)qi2j0Zja9@^3yu7( zVXxG;>eZtY_9{E=|v?e|1WJvpzoHm5PxBjs5m62USgrSF5FhoDbd4&$reAx!|ZB}2oA1_cPJMk(F z49Wn4pr$whu``$q2adW2PO;C(idjQy5bP(qECB_tHiTU?*{Xf1gr{{##|-ib3}ZdE zbO*J7czamMP#h;};o+O*A#g9#9;l>!%I*QuTdC4mL#vFTyVO0?L;4^Zw8;MlQ)@4Z z?_f(PwQA|BNo1)f(;eF?i5A&-QULo`DfrJquzu~pw!x8t=nIE}tA!IoJz1n9(f=O^ zbeIMuqbGs7bv$VZ?OJ3ijOh4@_CAl6n`UrTQmH6{flht`^^MkJf|m2xLyl)9aEr~6 zBqF0Dm2ObcQTUo*8n;0a3_-={QIt?Gts_iMsmK@_7nU5$JCvQ5w^ix?(DvqGRhH}9 z{{oyf6&w>4C6vq*oKSPX9CFT`4W_9$g(;AtnV<+bL(VuNCVN+EZBsLIDg+c1Ns(Qa zV46$bTK5-Kbmv9G!vFh4r!i21o}?~8 zxJ4SZu5*v^@~2JV1Y2LS2)+8a>uZu5c8Z+Yx=Sa7Okt_>PvYX~7MO6OjkB{MUp3Uk zj3TdROpLQ(mujhx?y`1fYt|*E-4pUORttmqacx-p(isx=M5BYiIQ z2s)Rs+|=EC7>>af7#+!j(Vl6tAb?(CJzqv0kDwx7whgeKbiWH|WF2d7 zsvI11|G0Nul~PwNmy~{9rXdrfI$Q+W96lS*yWLJLkHUX6SyAuskBJhBAr_aj>m@#N zQ}L(_=J+-nIQl>Ef$lvTHJj+!sF{pFuc;B6om5cJQob}P!idAF2XZ{BS={`FINEm3 z&OYhs^!YsPR(i?#YO8jbh4!Ds9$W2C!8Ur`^bHG(rXT0m4tJ|QF`DK8 z`KYV)wB*Xh>ew4?CneYdWWLR)Nwv2?;Sh?|o3ohVVo7Go;9b;vE``8(i=I5ppx~mq zf+!*L-zdWlsWC}AsbCDa-|rgp2!V-E4`*^jojd1Xo(WM4XCB`JIGU{M3-_X=jbM=o zjkt4uzw;pACOwj9sp<#~<%eeqnf|^3OB;tHpNrUHU3lkFK-RAKBH@d2l=}j!guC={ z!_2IdY1`TA$M?~LF!ym?Wt&fFSd!Jrm3Gv0ET|S5S{dIol16E#IGfU&cGBWAoJ|%M zt@9v>upHEcuF)FJWBDu%f)VIrk_8$ygTBQJ;$VOUHh0kuKUb>S#FvLyD22D3J2!}S zFD8ib(uBbb>-QO~k#+vnjW>^FSfo#E1qnXvDwx21?XpqEd_nM@N|FwcJb5(>2{USd zbQwjH66>m!#`N3GOk3!#n3#GD^nV?UMl>&XjSW2Y>kZ460smW(^>7~=%?wM>CnWH} z+*kQc`z~MJus%&tW3w6`F1t(j|4fjwGW5weuJ^8*X`U#`<#E zkQks=FdXbUBOfL($g_O=a2>+Mc3UlYc7iOXc0ZfH>#a=uVvN9y9;Kfu?GNNzl27ry zDG2Y5B+X09g!8G-X{LvrHsE2V89;3oz&^9azjMqAJG&aqec$e<<6^U zkr?=0cKj|Af*WX4w$q|DdRSq`0{<6fOe;sKN(l}r+s$&aC9S$9gzTyu^d-0nd07np zyu17FdfZmk^v$nwQd^$wv~?9`(a#=Om=GZcTXAFznt;p42@6aat1n9l+*yLls10#0_S+soIK7%QmZwHz5lLnhR5E7G642JtipKc7n`WyQIev;!zVm77>d9pS*e zzqCAv(|v?+f);Y{qxg7RY&E&uda`}na_ha;PojO_9cf5#%;T-m-hJ)elO)t zc3s7(Ag#h~K_|H@N!DQU2+D|akLLUZ?1N=zvZ|{)UhfUUWZHV2ZkBa34V6p^8Ym8n z&eqD7Nbrl9q`#P|1Od>%2dD8U!(g2Wi8}TQsC1_Os`xzp@q6Y4u$LU`_U4frbN1f+ z+!%S|sW-E=^3V8X&?Lmo>)vEe5*IVCpA!`3t;v+OmBq-w2U~bK246tm;yW!eYBp$S zs<|$Y<1>Ge1CMJ-FqStCyjyNh)6s@yrbVxwk=;k1IfqGdnx%EhB|HHSH0M|lk}*%< zv|ac(xk6BrMYvrG%q?D+8Z!9f9oZFNBTS1Sff8*1>HX&*0IVkM-)Swga*GjT`r&{* zNDcsWn(PTD(AY@2$bu?wVlk{HolOmJCR7pA4n)6D)n-IJem(g?*o^@UJfGyb=-&73 zMMob}D?N`1FOR&K4w3qChN-hkbxqa zXLtwv0kO*fj8NYij%#9~={+bV)@e7$`Q?|bq9vM1VU2SeBPc4yiq0BZtWtR@*wZ{d z@BtV=(g*k$#v5O7s zw7~({@+{6y*<2$0)o4pNTlyN^v?<#51}8t${u*9JbzPaZugEH6#_|t#87_l^`7V`e zJkw>THMXrQ!w}?@E!m<=m=g>(%Tc=+WqkXqtyX;3s#t{*4?i;Z(HMYm2#8QWBMjsEBN&n2h689JP0}qZT+4Z~bubDk z5&A47q}7zSgp3u4b#JgPFo4#GbE`KhGGk+aE2B6VD%@$fDMqSiT-2qcx@4{s*%y~+ zX~idP!*xfIb~;y+YXv!sH;1w4rFn{N(`_`5l;Lj3~p^E!ahp)-98k9hj6W_MaH!8nr%qK(S3<8n+z5{NhMn;}aCY;t(6pM8nnYasYeF zM2(c6(R38#Aa;aPk|+HMLh4%OSH>Ou3~L#N?;4fjTLtvijnW=^J?UZ2QE+#vS5jONt3xM8=ciL`u~Iq~=YTRF-gnQhh7*nGu^ zHMjU*lo#j!$LoYgG<7_mFnrax#qWLf$&R62WY2hg%J=a0a)U?Z65GNEseJnyMst-T zYT;u{J_b}t`_UG_VjLZZIynky-p$Y$XUwhY~gjpHvh7;0)iDl3$;5u!v_Wof?%{xH_z`2X8fsF~w091*$D&nt%^QF?D$y5?B4^prBE{_A;CNPgbpM zLx)=h|FpdiL5gyd@%2A#k+Kp|cok!RORd)C-|$dRAcGC`Hu6{t!%!1tMV2amCFUuP zg#l=E9yiaLl?1_3fn0i%pc2q@gTW;6iLqOpJv7%%KVM!Bt>H4-tNk(h}9H*6LssI#j9j*N973mRx@VgqHW zD!{C%8U9H{R4Z931^Z$hOhO7(gHsJjm1@X~ntJp!pI^5p5TMg=6wLtzDmb1+L*DFa z{kc$dv>+(fEwJu!mmR{`Ds-{^D23;A24zUNOg`DSqNJPQclzWoJk+e46Y+dw<$G&Ssvm3F^8*f5%Re0=;1$0 z*o2q>jBB-3A$B)5w+EWznl_BRW{9u$7A(wOVN_&X?vZs6YiyaJEr&E-wh1~Gz8KSm zNR7_e1Chzy>I6Bfp%w<9=kJ5&Q zP%hIG82=#*ecLfz7{t6+KWpAY?GBi#QOqR%^%7)%P~jL)BU^w!paVC)vR%6)0FFBN z38nfMagT6wt^PP^{?H^Y!olSsPE>#I4G!IA{NhsivYV4 zWmY#lv3Dmd<|sJ8{gLY7Pknrk9YynSKHG;M23!hc&zcp3Rj&UHB9ONjXshrP6b8?6 zY|#k69rhqkuuwuX6U1iUjjyomqLlFR^4gR3f}GUZ(86p@M;m>Y=6Y8(KZvRiC9Oy) z8J8JL?bb+ZOc#<)0--z&>)A|+(*l`frz84ih(@5Bf#y%Vc{S+WBb|~wi8z4Pkm%)j z9xq4pAwR<3q;D%-4FX7(4Z<$oj_c8R)fJkg)N$NA26h%y$K_%!IU1;#;Dlo;h-9nz z?~rrbS;SS%obquC5lmps^6u`lE=)JbGTYfcmE%8@k36f zB_yufP`6;Ar`M!&^=du8EO6G?9oatb{oj{0%-A>{sOQYooUlYiUKgE$jKomEM;?G{ z)lM-5{8EZQVN}(9czkW6`Nw~t>BH52qh_^rcaxE0jk#IvtpR02GZ((^F?mzd+<%6x z?fplKqy1a13Hk>#mr)t7v;UYCVRPZJ#_G!iBiD+S$L$KA)HPrnB+S&r_5Jl3(P(bW zTT@HEl_tM6!9q~VfoD#%`A5C0uiM&;@Eo_WByrzQE&p6-4{yuJ_Hpd9#*L`>Un=1K zdllAv+Y^r$^Vhypf23bC`^7&d>{{9M`MSNlzH9NH3tOELQSU#0`s=H(F3))+_lw@u zp~Pc8p+=RO&4-<&U#0@YlFt9BTh9(3p0kqm7+$~j2iYTLCVB2X8u4G(>#r9wpTv8e z0_eTB1?Z~aw0V`W>gzLNV1VIg-r+fq;cXg4gxx%z*KS1<`$ml%+WzK_ElC)w(Al|N zOw7WdT8G~r=H$`f)~H#9K_j!yFAHD&WL>QeJ-5dn^ABjzd|?p93~Tp7y-^*~-mas_ zxPLPw|Ld!VMcY_vRO(>sqif!X9MVY^Z-)Pnt>>*#AMLOY$H~NnK=heMo6eZ-Il3yd zNp-&oVLxq8?d^E}nf4W}?-Y;EiZTz6ze644i!x)4=Oi}uQ+5u}8`55WdEYPA`w1cT z2~`cf)BNJEIg{LCTh!?OI=AVqdG;>%_AYDYo&IWURju|Mu=y<)$mkMlq%fwIVM@oYs&imBIMNnD{PdXE$~zrC_D2 zH`L?S6Rt!W-)^(3H@#ND+O8uAn-6W4P<|`nyyxd!}XC0p>MRa%Ij3ZRpmm3X|r>4?8wET zI?z_l7hG(*Az%9r^+)q1E~UCrzyhJCBAZ%F(I?ue9Vs#iAGmP~#YR1ag+3x0IH%Mi zYq%wZ{PGpAf^Lrf!uDg$-pQz4KVg1Z(^DvBS}5m?U;JRHvM!95pATWdG@2|bIAC+l z@a8~sjtTBCor_+bccapQ$U2Q)G7A!>5KvQ3X~9uOVCpH9D@voI88NmFwLI)gE{y(S z&w^r9XYnEVyF(lMJQ?qkbUL(XOUdB`Yp*KQ+?_Uz_~Q6?JsSD#Fa6|DVAhJc(H?}n zrOkQ;L)5}96PC4!mVM6Lq5)&ExUPs>nErEl@ehXvwyR%I_CYUh z&xFV0?%eHmXWiA|_K}4%+z}fRxiXnd?I*KCf8QO3bl|9qd|Sp^A1|o13w<&vbY+iKlp|F2ymwk!Fpn%|d*A%kc$ z#9jC4ThWnAMexNToIEk<>1b;>BHdiEbdlsm_?N5wEjg;*Xe0g&u+B5C7R^SJJe6aE z)O#t!WcqmlJ5sAbl!{W|i^*^Rb*-B8?KrXzApOfdRQ%hB4!vNjy&C(tx2!keD%1~G zFd$6`>h0Bb_rSFf2>~<~)^AL6QUpoc?*UTlL`)wP?uiD(%DkN8SeS5oZJC@T5=) zVaatxugyAAE?zRP2UT1$Zrp-iqxUyfDhz$DAo7&|iY)#A=2u%P*Ay!l_=`fErA|Nm zV*}0Y%*X`mSbELeN4JV3t{PrT<1o}i*D;o|xRJwhx)cVJ1G`T_!YV+yt9KbXJ!e?I zLw${bJH8C3OO!7am$_k3OLlATdOK|$qEoBybh_p-v;2Z%lhDh<&fY7iTe8D_(Ef~N zzwgd>538DF-Rofsxiph_(2@s5Pgcei)m_p0#N?dv(86--z6q86+>pL6>yN%7BWqzZ zhC#6@)uH4QRIRPLJ;C-Y_K)}?6)aA;>lgG(lFNnp{T&_q{bR?4Nnv?QHF5F`Egc!g z7VCXt@_{}`6W!c;G+y{*Q{R$&pYq@Loh^9bLI2pPVM*h0jX-yj^7Z*!aZkQkgva*H zbMJVaP9Em+XjEurMp*QZ&qdbTSD!rF|LxFP@?+RhDEx3$>>P@@0`oGv4I zDROvovG|~&47h`qZoBAz0vJ!oZ2)D90w7Av^j+kL^y9NrBxN^t|C)EmD|4S1pm`dA zqKgo_Ag*wEd|d&qpu4hfdrl^|SZiSra!hW)?v`R?;}@&ob#@m@uSZyk4cs|_D3|5< zA+ZXW&b0wW3WOdeue`2`r!(qf6mT#DMD5R^9dCw6HV=!DJY~e`^$dkuoWv3o%G!-> z_0{TM1+)M-3Sm6i-Iz5&z&un;!tq*QrBY=Cz^n~%QrU2jC(uf7-CpXj_16!jdp7bD zrEdR}hU!UE`eaN^Vtk-Zs++X4S44@1QVfcltCjuB*Nw&Q=b57oR~K)DA>ua^t+g5+ zUm4tu-!*&h!wlc&5?Z7!4#n7WR2T#Rt-6UMLJ^R!W>`dV5wrxCcF;_H71X{h*~T1I z8ur13iwAGfoH$1UV&fA7O-rnr0`cmMhJXPE47Ea-cN-$>3FKa{=9mG@AlVFVO2>&I zH(@BzXy!p7|0ETNYw2!LrS2MXANa)kOf{^lJgl?i(6~ts{YLn2S!Jtd4-yj)INvT~ z>gn^N}nE-=hc1s!0cXw?>)oa zUEV76x5Cg387o%z%_$En?EA~({YUJ*tCOv)e&v}=YRaG6cb@Zr?@iSp z6T|MjG2fFnD{Y;Fly?0kR&E3zBP*Rzx>;Ve*=&TOTUm&XI@$MYI$)=L%*?Bq{VlhK zHwp5_hc907X#ik_;#*sxw>>A{s8-=xb%oj4TVt(}F8Po$=2j^Dm8DU=*Pt_^jfojO z6X-ggKs;s1d&S zjWTpuc-$kE(qt85;{EsEmzzP*mhzy!YN>Y_LprfcOvq8~)Nkp7fui7itKo>GUWbifFR!0#q|Kia z213?AqrKLdo(=c>gx~>tg}sWGBtJhUVev6~LSPwLn+(nO4sZ5#eBlIJak+K#rt!GB zR2+z{tlz%k)0EOh$BMsg+wOed6&d9V=PkW>egES*5EnA{4Os3mVMS>(WL(!qQZRWo zF5X;Jx~nRAONjUVA3vxST9y&EEhRpAY+%WS;0JMeUhN;!3>1bwPC5Jg)29(r4BC^w z$QAM==B^J0?2qA8ywRrU;mZrI-H$&XQ{3ODs`hWwdUos>IrE*i?Kki1*Q0QYEqj6W z>toiTgKc9IHaBx`^;^;Sw(W=dE-HO3y>P|j!Y`MX*t`5O(^F0tn9c^I4S z6(ypQr&jt(BQmAUA7B-EQ(Wn-f@8JY*k?&Y7!$@Ehuk6j`gb=N?tCZv-T<+cq>IBCF0QdG{zSi_!SqyEtD3<+C8a+feMXAmQP zk#2pyPAWxqQJ$e)*sQd!yCUQo=SWdtoIpLWM(NCH04!X~48kxykY|T@lbm8mfmZ72 zwU>~}g_`8QrULwnF0)d}~IHp5okN&<;=(RrE7$pes z)?3mspPjmh!1+?Pw`hwopq2uwLMM-W)_QIycLzUL+xbtc>U#doFDG?E$UxD69X!fP zk6o=?@vo6RW0sEFUQvK8bZCKl*q#q(r}PgjZGPYl%ex7`zwz#wd%m7LnBOgNp1SnO zQ==AUW*29NdspUp-yOXrtnKyQ6aBF7`r%54+FqIm$$n~POhmQ{Z@5!F`{vau@sCb@ zbZsB2AZCj-acSYtS(OgK*AYy1|9vx+bwKXj&-NEp?J6or3A1zetFJFdLWC>IbaZUF3Dx_e`3zY`hkvuL!B+JCs5;0C=Thrdj ze=7tFV2cQ47nzcOQ%Y8nf>M<5>b;t34Jrpb5tqwS0~H$sEvhR==;9F_XkjuejAzc2 zTymDcq&Lj-KOF?%)>lnpNQv5aQG#shp;AqLz5qrcG*;CgcIYS6er#Fx!!wcfkj-N8 z$dAJ@Wgd%_%HJf1w1Dmg{cH8RIg1Au_F@E`D_cf%v}CZF3|?KNToD+9i9qYCXJ!u% zsVU5^hYTL+B$siqM=X+>@QX7KqP;DS4gWSQydHg1TuNnVQ^Ir^HJiR{atET zS1+(Jjkvae8n1dl@Q;(iI*nPGfxrQa@pr$jc+r&njzkNcpJlf+Yl(Z471n2Nc4rQk zdOzX2n6gNhvXxWcT-oe>kM=!$o^QR518KZd2R7!x)!V&S&RdoLT>NNu*n-rN zA&K9V{dCOMIi%`P=)jTd;i&KJ{PMovbZjh%f6z4{;#l>iqJp=5s=u51Qy<3`o_|7h zS$G~Nyw%44)7oAyywYHYCE2#@<#+1O?v+$;UOiU!2UlyMFwQF9F%kM)vhB4aouW4d z?%Q*){;lD)Y?FI7YkIQf*GE6z(cz;GCEi2*Mk{~QTRJGSMbBCN*N;avHhE0SNi zkq(Ie%?ogKs#>cCOBHRW6gK1@e4?PptS_eMI?_^oktHNR7Om5`x5iNW2!28 z#bG|*7@ykdaS_dp`m{bBK|&%jKaXM=sxVzBx@#P^idVY;aFxUc&__X_>1Q%@Fx)5> zGsDo_IiT3uEqCsHcxRk(Ju+^q|!@I#194(I>{k`uPOn*Nevn9a_44LFjMm!&WCe-r${c z+@pG8>!1y%L+i#rzT!4NYTW-eN2-X1h$Y zUh#Vy>px|YChR1i2-e7dKcw-@>ad_amVz15X;9;mR~7h}C(t$c^wQ<6EXK}UYfREX z>Z)_C6>?K65#Cmzlpsf{aD8x2UVa{hGNtWOJA;QMw5H-Z#PU zsx~vk``G2lv&s}%FYZ~Ls{n^17h^MHQPlTNP1lVFnR?cwwGCU@f#I(uy%VCA(aoy> zDHB^xMV%#^l4{VlWFCnvav5!7ay};3__R|NwuhP?mPlhewhWuOe>O_0muOlq)lrI* z=qDyx5TACE^vrWt+?ZPo6*y*xQwX4fyc6A-NZ?C-!3%<*)RVPUrlrvypmuagVcG&A zLVlBTX3w@mhCNroAq^<-L&FaZ%>T@g;F#fgG(pYcGZUalP=X8~#00A^zTZ_iQ&&oa zG$6q}o#LfaM|QTCB^M4vM{c?{+2k1Rrfj2ico~z&!M|T094+I&Ty;9;V~DhF5xrG{ z+4kXYy?Ztq{fkE)#&qk0Bv0jwx)Rp3t%*KUdHF?}dKraoXjo>828YA1q`cs2^mVCT z+e=z2&9a9@!I)yikr-~P&w$Fl?a{P~70q(j{`xgHB0m5jY-qzfwqbX6Z)o)r1H;UN zMGdpcr)6Dh5|Vxz-;86Ae?69zW&7Eu z2m$i`f-{lnl~L(=zch_6ACR!g1=m40SA}zY>|N3eQuOjZH}2Q(f*0nOXJ4;KEDvci zz4U<3<}KC9TME8^Flu;?QGmXgl3x9TNAYNvvKI$dIc~u%>Z#rDdw^P%RK&5MYZpB{ zT6D$5286J6{osA12Y=b=Ei^+{%Bxy4dLH*G{M_+F#`Zhj=~Sx=;X%{?&*gZ=Xe4># z)!$EN*+t;RV1)RIRZG82V+K9-C~^(_OFl5VempAs$bSi@hDIk-qJ3M}K?(O?*x{4( zPJfL_G1kKhHVt53*8RC_;RrDQ9fiO@vL1O$CU_R*tuQn7I8e?wcTQ*1x!(%5Cb?XC zBC=DVn4A?tcBQP?%7&I&MW6hF(!;KjNDyJH|7$`7Y%TbSu+iX9RZC66hpY#z1Wt{O zCxv!X<=Y%SPsn`AUdi`W^5)=RG8g;^8d~*XJ`ckvmUra8*X^s!g$DG?Jam5j`k;h+P}4+h>%{;YN@kI#cHKU{pL$YJoR>vNChpBQYr-hcV~ zGyJ|O{QiOCm5-l!dEdZ`A>B7`Zo?;z>X(^y=!b7-9Z&SKIgS2&QdjDdBjYn8n-ki5 zYrxgQhK(PUb2ry zlC&BD>0x4nCO?QI@*=WlxG1goTnCz?u;_`UG?Nuu6hl)Gm)e>bC#*x!&#En}XoQ2`S8G088}#cr(xHERCbV=y#?*nHWsM&{=h~r` zlkNW7L)%Qbu&np&)g!0-1Z8DRE`D&V$DyJbeTLMp9^Z32^k|v1DXrsG0}=vYPZSa9 zd6nsKgJ2ih<{+|P3COuYeA67s8~C%V?D9b?e~dn;8IU#O%co>gg9)K6Q86ijP_@J~ z+HaP))Rx$lOFwMN3_uSqu9Obc0Js%6!}T! z3-Awe63{>!-jIYk2LA0me#b;!gG9tp7uHQ_c&dU^@G1C6)gMf0C`$Ad>HHX$zC%8u zSP5b$gwqvSkL?iILe@C{lFfqQ1$Bm{S>2V}OfwNChHR3|C-`{e>)7AIv9dXh!8nJe z9nGIC2bx35wC_wpw>|U3JalzrHZ(H{p_w8=Qd7dJJ_$3UA9Ujb$!NzThAB>(wRaO+U ziObHhJ;mI_2kPeM?yl@V=n|I=D!!2E%N{w!!Qu?1Mvwh~5iI;v8@52vY0F$!)zzNo z+31<{EdeC*yxTqK-TMzOD>fM&!ADWaz45o2H8^%=Osm(&_B+)vv^?+`my&7AZ!h!e zf2!mBA#MCUZKvxE>F}cEuw&yl3eu*ZDm&1rb5EHvXD z={hX2(Cb1cSM45^OxmRI$r6+!+XYmu6VFW48RDm5U@|tJOI~4xn0_){+oFt;xdh&m zI%s_8TD^v`g$F!Dj|A{L%80(XthfIyrri2(P3G_|%)mKV z3=tOt%>Z4&(Q%)0YGQfDw7#5SEF%*3Ro!4vwTS(i7DO_EVsU!yW(XvK4@ zdH5xA@6SK+pB7Uh1k z(N2HuFWTayCmiH>JuUn(tl_FwA07LsmWR`5L{soh*emnChV%b$en{4qYB%qy>j{^d zR41nsNBgj4qhZ6GL_e2LJqOklIBQ}Xh`nvTEcs~b-;0tH)<1Q!B`eZG2;^`gA7vom zx(er)vf12QS$^vtHP*N#TEz&e7)<`!C`_ZL%m5oCYU(AG4lEra1=>rX4M8lN!q~+ zO!(x+g^O<*xh&K?QBs; z4|DN!4QTlUnrP0c0n6#zr5zN%N7}A$L0N4>H{WrA>o6^2n-c2o-0}7EYiPM;Av`lP zbBGgh4ew1G>9=W0SE_Qp@-6jA^*U8y`<8#ie%{I7Be`f$=-ut1+2P~%lzaB^v?Y8p zb4nfosK>IO$8W*qe+Z7ovw#r7a;nVfkFN3a>EAL&3p5YkGfWUljb|f1ZSzJFd^F6xLIhRgR0_>73%}q zKii8q3sXtUPDYx*VaEM6zKHY=Tqp1%&;)K3E9j{XJOL15frG2-gzMtSy~>4^#L_`4 z+?l*GL2T8C!`T#msRSm)4V!PNUa3hK_!y1lWq6vnIWK8~Z^l|*hH&&&+4NCv0rNE+O^`2!6C8wDl+~SFdf6D?-!kS(pML~k z--f7UVCM9Gw>W4OkGYh<90yrLTab&U?T=R-f{T~!7 z+pO2BYo~Z6dq)3QuVXdzMV$`S9`CG+zYsaTr0DfAe;ZR9wz^3ul#l%$O!dLjVCIWQcFt@~XZV$GDAJF$8Ay%`B*&93-HP(9A4i|0Nj&r@cE-cWv#X)6Q*IN)SyT&TQn|=!Y8AQdg1L+%Au;n|0kq{ z*3MP6X2$#U_QIrx0^|9_l@)R$mkn5=4k|O-Qr1Tw>HRp!p47Oh$bgvZJvbKDz6|S1 zWV<6*9f4r-u@b4L>hUq#3&wpH!UK6)8)PcB2t*XQ9Ons3JPEPf`ck>-v&gFuMpw$l z;w~(gu9DhL2>$j}D+phZ{C@luzlcaBD-mN!nFK1zMGveXyC9j#$rjwi{pfhl24$0F z^T)v#Fy7}4NSKXA5PjjIUI?}$vd`W;h-Ax}fdSV1lb|5J3VAJePFdHSp~vqbiFHFd zaPyB{R;X7Q#D)HUtVmHhIj*Vc#HZkgdq8Ps4UESYAe-i={bi9~N6ecfgd-J9yN8Qn z6azKyv&;fn&%h@O3WO}hG@*`8RnJB;a+4hrK~Vl>u@}8gzQLsT?9|e%@lGe-KpxuI zO}KqU?6XffpR+#7db8w*<<5srUHrk?YNsV|t)27tRfib;ygl!W;PxH2o_YQ$uk?wl z=k}N}Y;V|iKV(}ET2s9zmu)`QGw)>k>nBe4?i93gOnq7GMoBktSD^t0hZ%u+KMj7E z`z;s^K`)9-08I%I!%DeNb<*hP5}r9%aP-Wrh^&XI8&AbUK{2O;8_Gu8SPC}8=TM=q z^w{J*0^2OuDe5q%gg{A`5=WP-oT-KI9tTE_B;2dU#H5ke4^!@ajV-SIxPy3+&znCg zPC=0(rrNYKNK1p>TYlIb==75lb{UTNe*s$49%hAS`*b*e@5333J2(2I<5TlS4QuoL z)^)xAfBe$R|Br>f?h)Bp7rh*ZiqRSL?W_AQFbbQ*ri%*4tRb#k7t2J(p`!+F2mqi` zAR=pe|AoWnembmm|E=8j#FJ(VUEOMJ03**c_&|c+^%Jiw3A#cPXC?ZgP6*7%BkrDE|Da4#@w-U8RE2rPYN06-=Z`WP1MK_GB3^hU?H*YCtXR zIhS>?mgv`ucKm`#Xpsf`y~;7Zi-e;yYulyg5f3gCY$J!0e7O+RI7j0mn(&rSte1yQlH#(ai1E8~Wf$TJX)5X55RRCuq!QVygH0FWRBAG5ar|vAej_dp6`e%UTBi}6%7fZ)v=%@5ZL*Yp zn?&KtMg-SLrO*X43y=?vq5o-!zb9>3^$CXEN}ymq>3v0ui0Y>7^n26R2UPbm=|V!2 zvvmT}CDXT~gn|sdQ*4DCyYTj6t=6l(vaDUlu1j4L6wgF_)_F*lS&N`LLxHlMqAN$p!3g_6aaO7X zOx2yJ9kyd->a z)TN9GTjx^*G0{c=)zP1!B_5%iJ%CJ#ADSs*i5ZeaS+^izx$FHwK=sErJNK%HRc*83 zTQA#o`r?Z6FUR|qx}>S*xvak@d8Q(k&NDcaTTxRKIUs%cx1eCzw!Z~CuF3x4RObHV z`JS&X`my0F_5~mASyENk$>O#9=cQSbDI3YslKH}$#8^xQtj5gd)+md8qal>C<;F`b z!zZkr*6Gd6J!MSLO*Da%(PC~Giou=SHc5!Tql|L7yOuD2jyFBC@m9b(bL8Tj3nS!= zy@s;HU;GriViA{&AC}0!m#~Rdbz_8IslAa8D*36M`h|40rxf>$;|leq9RaNEuj#j! zz&Yuq(#ln?5PAvYklnUq$n_uIIf5@a)?OCn07`-KU@mA4$A*j*Hp}3#08!@;QvhP{ z1Wk)O?;K~xbug4*k!d)=;CG#CJK+6|+%cu~#^{&N*xOm|&wJQu>xAR{no%g%A|%>n z+i#*G)x^&oUiV!hFBBBr&r}%)##WlR0^^4vQt)wNitr*`lJT=el};Tocam9!F<)XJ z!-F^g9nc?X;yoxLQ^q%YC82G83RlGfad!H{^4E|G5gSo_ES&x8!Bp<5M*>>I{6IuQ zW5tHefkvl_?>#>ImZC&VEEHv>&t)1xq{`WH$l?}MOreYgeS3QP{gcx-eE4Cv=?`D` zSjIu^Pq@a;TcJ znF0=6@4YRW28e+bd4wgg9qCS6YMbt?L=e@V-}c&H(g-x__{=wQCRH!Xjvt%)S+^dY zJa(*fD&BnT)WwptzqbrLGp4%w)XIc!0%qKZbZYAKYA=h^j;62n;ve0{*Q?cgb=u;l zTi1PjGB|quA8~hAFJp@E_-=~{@yq8dcO?2aY|ivw&kyo=vUUH*=c^uG&lp(oAaaDq zse;^q>GNVnc+7X)-Pk$o;U6pAvj%5|>2W=CG~yf$RsNjDI1))H<8O}1 zxgjEW`mh*N#b7gLBRUxi@Z}Tx#uJRdhX$swg09U;^dJd8J1_z~Si{zwrnuMr696W2 z@XbF?HiKfp9hWNrQ;F!7QF40whh*>(kEL?)7@I)yPQT(zrr!(0VAWtFpY_?KPS zF%euOux<1~QjI1syw*Ob|9M^^WdHfq#IH$Dhp17!=J=tt#qDG+xSw-VdR0A9@q zXMAuGJkVU#Xhl(PO^RibfPsVSDyT)Cjr2TMcjCfe=)lDYe$kCkhA+|x2)XesOuUnT z=l=#$J(}+fTN^*r=i=+OhyKieEJl!T6f|;@VW;rLXkTA4(=bcUFH%Mk*x9ksmdm2{ z0;8n2X2hr$m-yh%>{tKl5l8wMm~hd1?+s6GnK50Rt#1QC|OZdtuGzG&VNaLFJcP_lheR+jo)bz>J~`K?GB zU%9q7SCdbSsItV6s(&|y1j86255;$CGO7p$n7Q~!!_at@OXD}u5-!B6{{CCNJ4HvP zz3qt44o6XAn6lgw<7;{~S@5d%EmuZkuB&G5Azvz-{sPht1NnMB~EP z1+y0rF3Cl(3N?sWwFY!KV3DLscJ3COO+Gz|)$3@t%v%FyoNF)Jo3SdBnWXaQYkSW1 ziHffBd%F!!Q)u)iPDMCT@oVYO_s`q|MBLzgCt^0cT4X9j!^X@C!7dNrUcYO+cxIOO zC(!Jq52uZfk3By980Rj|9385ADW}P#W+@ZEZ1;a|sDB;|r-fU(V&FzRdzzI5O@9J? zrtcO>rvOS_B~6m?x{w3ivnG7a_MLbOauhb6)@p+gq<+o2y`}e?PXhV6D^E zb!~Fr^1j>XO2^<|2Q?euoV%-6yVxg5t*i%q?l{co@0Q}Vy6NX57k@G4W&b)!b2i^d zEg1LPv5n!gonP!SqwM)Q-xnV!3XWdj<2!@P zr%fnzT<~?$&#jz0WhcK;e9wE~%44>RgW89TPwyL&J@3KIfPLqF=y$fozB^q)tfxQB zbGKQyrbUk`8@e-m?8t&9`#pvfg9z+zXbUVUnpM{1j$CHI;AV?hO|Bh5HNf#GPF|LQp<-iHbD^M zJ<*~xt2N++WP-b^7@xl}FT#zHWqgs^v70~toUeA@LPYE+WpT&}*62f2{(O1-_<1D| z_U5J^%b|A5{=r~Two;3xNAkWwdxV6l$cAKOPG zerPq^$s*q&{hwMYEM0?U>O1x$f*e0TbpJVz52kcok);?2=+f_(s^T)Xk}AOj2+9Yt zvDq$Fc@1;IH0HO~j#E#aUTO(vlbW4!%I?i0b>M-Xyo3eb-;0*F-0<{YUN3B8Eau~T zFyxB}@b49yz9)V8qeqP|YXtRaA83W%Z13#ZDZIkwcDL$e^R@AS}DZ z;El^p3-ui52LL1`wY(Z|@$0fya}#PA<7q{Cl0t+yZPPhJLy$sbm#lr&xyT&CnQ2_l z?UZnXu_pOYHUfph+C-q5jQbq@rp(aVo~odQeU57-zYG~^d~~wT5RRl9ZM?v4xH3u= zQ6}fJd>R}cbUFD^=^mgC?IcdRZ zws1g80}bj5HOnPk%~M%zpWH=FC*iSW>^q0;9yK}dx3GN0rReK9t?yQ zCBVw)F68Y6S`r~MCY%}=X{chk!L}*G5+01j$}i__S>8Ef$B=1nXTA{>gD}%i|LsCz zPJIqli|j@+v5xgc7oxg^h+V$vO-Kt|g8a2a)l#=LO z05L29BD)DE-HW16&%1f^e))s4NTtAM;(}o{es$<)Q;90ekM`l>Uy$;{p|70%HEbXj zbsF$#{fUjI;m$*&3^Fcaa1C9sVyK~#7${i2*-GYCw3-Qa3EYQzL+N5Q(IpI|hcWiG z=+pzxW0hh2KM+6E)pjOTLyTRH1vB8x}{MG;|7kgO<_mo z%~xh(+6&Xmm-wt#7nd&{a{m>N-SRSj`T7ES(7Fq+bn7cTX<~Tml$BE)!cWXikNQ)B zo_r!Qg1jMxXFT6>v6gM*cK0m~^p~}JI%7L;7w&enscbn!Myoe^;NK*wnI`{1a!IE* z2v?7hN5liY#VT3)6sFzzZM-bUU$Ynn5CU;VaU~?^UE1%g~%4Rzfaxao=XyO9c zOd47+7cWfrnZf)Ld@*P`>b8V8{5BPKa)ZZ@nW#}BfXm3;!gHLA9A_eFsr>zP*>lHA z$5`l>wkXHx_ut23uvdSyWt9E6;T8=GUpw{L$x=l&-e@D!kt$-+@lVrMAH>evHt8i@>u|qK7z^i6hZuIt^~`@Q%{_Hu zlY7?nU;gRov3Hkm|Ggenk<;RToxAkr&3c0${yw3&e^Ch2KWtG##^=A?{smZ`OJkFE z2LF~e=vaAF!}!9}kAkhAZJreT!@jJlZ#pfg>e+Bz{m@@0G(Pj*p7&nQ{F{CE_HHgu zTBcaT`)1}0OL*AU`(euZ9pMu~Z{>yDTfRT{v+{9Of5f&e4P3Bff_SG>sdQ5 z+eZmOkCW%GX_|hXA+N#tuikxS%QDpC*4<(-g#vbh#J^m|uxgee)Qi;`?j|xH7PqEF zR->c{9%a9b(9w&$kODKiNpxC)a=JP(C%#>Tp>$tWw(J&H$Lbb-Ymq@NTU%8F6UHj0 zKJ|`X|8k#RYkprZrZ2IB!eS7W6J*??2N1GS{#Y_niSj42QiHQQyw zu;uY!{HgqW!(&D`7XH_`Uexmv5$ma5{ffXdHqc|_c0{=^kQst{Ki|%=(kNVSC!EKPW z*!W>3$h%;znMBIq3;+yXL$g@xa5GQVu#Nh6wPcIOs&*<*x#HlUlcSgr6z zURT8C9^-e99{j+w2cVbjaHaN(sHZ~j?|pFgiU@#m`I29OqW-jD+lQl_FV?%5y7cFh z)XWSpS_1uBpyVw&JKPDhF9|2V70N}$J&Z$46|M@cEN&OJPZPP&y@hZe*zxl(i2J^yk%jZgwz_mjnZ>PrRxSa4-7MWaV=8(puxCo!Ki#6V!e-T{=`Mh@YvCnxpYp@YieM*Ds z@%W?kLdene2@KcDowYHZ!1yaU&y1mVT}zPgPOgPco&** zyMO(Z+;6iX)xasq$`{e|EISdkSZ19>##Ut#oDYw{b(6j`ewzvbJ4XE1iiO71(Dd2j z(~OGj1tR4vN-XI@=4{p#Xke{MArj05I>xYm)`~>s02`X zi9#DDy3@E3@KT@@9t?^ zXlh)O=@r|0-Kom&8?@fzn^w+ldz?V-Es>4t-tyLLd8NwXWvmF3g(Gvn^gG%=iSl6%u2Q+)W7b48wY34W`PY=v zAlc=p{;Js-yJo#^( zr5~xFZSxVEbf@N3Ihvy|fvlJ;m}0LWmIF7wl?F?LMGzC2?JS44@xFmzq=m!QXu;gJ zBXylbALA6y7P*L&qyaP)LETVBxulqG3``c#wgTsz#)c1MJaEL3BUJ^V+&br@g~FG3 z$8N5cvZ59pWxGQJP_m+Vit+n_+sQus%$rAc&1o|_?pe1l(tvfZlo=J0NT*2lib(Ty zJaMzpfaBxAMsywV#RzU-4n=m16ΉM*7B?sdqxmAe{c-j}dmbPkr_hVmg}AVp;D zF(De*JiN~R!bMUvtT&WCKfUPT^j4!y?ySyAt0F3mwnXu4aa`EiVa$~-3pF$HdbDgr zf2ijZ)NPdque{4OOfv!YP^>6rL?!xGDCok{yY=-ewjhnGeM>XOL7HbL3ymug1f>C( zyn)+|4k?%$>LsY|0zs!&m|2z=D@R2KfT^4{eILo3;plxa_kk5%&7r4TU#wSBu!!%1 zM+Z$)d2TwUkN!MUBo2s!eu2|-3w*HB+QwdD`SRtOT-SHgZ1X%4+P7vV86hn!p=a%5!(TtrREb#LmJ3qWvW+bYwJ>+%{IARaA>;E-FB zi6GSIN*2itP`u#GEDJC}tA=zcAZ2EZ7V_K6mv4~pSUyC4!8qK%E~gWj7As`OgaOWN zU1xr`uXd>cd{#-h!Kj{-Vd=jGL6b;@qxY?PbXf7W{p?-mb#@P?M#{c@woGDznn9w* ztBbKGhJk)bWQ-G2kIIzAUO~#M`LULK;Qfhu8o!)!M0LjNnw!`;yRK4l(wmw}`@#13 zS*at>2`5fx`Ixc`^?K_R$I-OvBt$6~5@bmd%ouq5=}Hkhsf;F-FTrFDUZvldv>1p4 z+bX$74zD$_7}m}QM%r-r$00{{-e?oONj$$95Vn-^$hb>}GBIQiS9N1EEgGA#1jpIM zIGE|lnfAXwIsaCC?5}(Nbz4_L5MMg@8MP3Ws`ZTo$sU8U>r?WG$Xqt4R0~Ma+>Sc4 zzQLLO`rhCTr-K?oa(((M-=pUqy};eIh%^TGx&-b&|-5m<}a$BD2A+gNX(2f z_Tl8%BoTv(;k@S%55)4g9#Ky|{F@C1r9TJ3p6t&g*_hgY{)S)A?X4~@I(w;LvPMS0 zJnHZZOg=VDg`TL}yKa{KZ>a?Ui5Ygq#j6?=wyQ&L`Yg`(g@HnpOfR!Qqs)yjyhXh9?f()k2OtJF==c! z)Uq(OQy#Rojzm50#RSp30#;IdaY`bnOd!!EM6oNk{$6bLJQH#%vdl#gl?=r+Rx6#W?~!O2 zH@YT(!v6k?xL^wtz}ET1Sbmhmj&`yw8QvLVzMk7zHcC;?z1Yz7?@6AFs=KR*(PpXC z^=TP1V{FeyS!1uRb6PlWUgI|o*4oo*liQ{G(;s#0b?4rXr-O^mMX=k79*m8wupR!gHrj`B6uWLr^vs0JG6&Fr;dQbUBEr0nZL;eMi^9NP` zk=oVE*Dmp}x9|A@aldwUDeatM-MF;hd6&U~X)AWOS<%7^GD5w6-|XJlY}AbXuMVHd zaJ3?2v)lCM7{16duSA`U`lx4gFevs!Y~0IFblR<9@8T{8XePN|pj9rTKg6}hY)vAt zJcM8F72wHli$7!h>3PgHs6ET&$vGtR!}jsyVeOM(NI#wP!)LF~*{j4eu_;ID<=ZhW zk*55bCMT*O+H^6-VrF1Ct{Kro+7yX7i!L~OvuVfomwFsOckUvbNN#cHYL!NLi1FPu z1A1Q-Vv}%uzS6UmI8|ZXw|MeaN{UVq=wZ+JCJ(T_PIx+9Nx}c}O+9_NFLd&f{LIKh~d?0|R0Afm> z0FmX(ox}C;%#8k*K9^GT^jgA0N64)D(j$!lP+C<06DDdw^T@|@04)l4Ybu5

_x(-SJAvdk!Uo|BVuP%E>{Y8-K*#qpRHJuCeOpmT6NLnAW$5RJc^4>)+e#l9eNq z#$X^@7p_9+206tQVolLaNoidr-Zd~KCXNqODz;RcoT7rM1mfx>nov&p_Kfzf6_(1$ z(m2s>S$;e3EOj%aK#JsDWFY|Ebyo&~W(>!&wV%sI@oxs^Koh151HL-44e-=gRI6J)*Qi@+v6WseT=ZW)Bcun(6*ZQZU8DnBPtFg3dztll3^4;txFt@GanJUI1!Y;0^{U_2==Y{g=(3g%x<#^~bZsVNn9t@UT04j*Utbp955?mc90j1x4ZNH#@nKdep(l9vfgWC-0KG zj9ndInsBl&I-Bfe;4W@VlM_?BE55At4T-DIiA*F7e!lA3GgT+s70>#?ig7&Dj87(YyiWV z2A2yQr%xd@H>pxjk{*1}Nx{$9Kd`p)DAX6v?hRdjh1mQ~|nd?{7 z!=Sk46^^IDKKq;M$o2eMT5XO9iX0S_)J)=l0D#7()Tu9O`VM$|#2WSyf_6>BbW@sc zXZ#Di9_JZdfXwXTte3m2Ep+cSx%(u=4Iw2J=b?RBYaK)zCpHirO)*xY)t6C34wDms z-0c{8H{_~f1pRzVLEMqTk8-2{BSnfldr3tMS~zMJH(jBKCJvZB8#eRO){$2rnyWgG zFS_~jxlvh@5Dw@>Y+VNwcZA<2TT!Dm9g-8q)VuL<@P?1 zg<(c*k8x}88SR6+x)4c;jdV*#f}W~$KC61g$!=xlZR&a$qjWR{FDYfCW4>{Ckm!yc@ap0&sVR-lqdq#UW_C|WY*)OS$UXnpcB<*m|iil68xUce&>-!kBq=gOZ5 z8c#J>Kc;>`pl)Dqj-0Y-x$UqO(=qnbg zghBEI&m%Wka9h7SoIRWW4`XK@S9O{G{ez-{O9-0#E`sKo3tHfonu=TLl!dwF7NVm_ zCL{{3K&B{Yim0@VR^#B35tckLE!O(WieIj}IHk%~ zC#r!1N;z>~o9|)-`GAHE`bV!6&hH}FD_OOeY?H8~7)q6T3m9oUqqSqFmaveZ+wSmY z6{Sp6dTSI2zC)e_ashbV`H1u$DiFiST<>@^j>^Bj% zoXQzrW~F}gMVCpL@*>o&`|r~bVzTp-K+e9DW#_MnaxSOb5P`ZNI7eg)(#=qdbdeS8 zdN%4lcACH#bfjKAF}h5FcA0`Ux37T}0{_`yO(zS&VzDtamCAc`|7Z>m`h6~_yn)8O z8}pJ^DHnIlu|kL1Uj_2uVkP-~zc;R~fe%|j#?UwGmLi-M?= zMVv59A~5~TS(WsSx9K@PN!R!i7>uSlMTblaB~ED3SE3zN5>AG@HslXI1tFH zVMCqTP_dkD(fB=}h~9@%z6)c$CbUY4W!Onw;iW_o-S z2cb+JVsv8XcSZCdtGvksQau9=>7$#p0p_%q<)rK4SR-2ysadgBnVHd528zVsBLUae zDMX*MWP?&v9>|!z#3Jaki9C!ixSzd|d4f8Ly`@0{pcMOzsP_E~Tn#<fhLWS8L@#j!C1B2!eZ;n5>E4Dgd`$bu+Sdj zn*)Pq9P>XuAOw5r1cclBHJ!|De>dkh|655Nu8V4gazvKyu#-nSu_;L#M1WK)j2r;KJ)mDEp>TF|J1G14(kG-T_Y zomKarE8F8$ft4T;HgLZ!it(kv!bO0eh2Bp&;z!<^<(t;Yj=|@OQYLBy$N6$Edrr*6 zuF9c8>;+z8;W|#eMQ+z88CTWRa>z_EREpiC^A<=NC^Jl>lvr`<<)@#+-``dPCoVTB zsoTuyU&c-KBy4How0W&ue&y1`agV)KaO%mW?TF$IzT#=4^v6(Ae&b%3Lljbc!kB`W zRv#YMsbu!28CS_@&>4-yFUoDio}fq%5+zM5QyvE7Ljy-_35AUAuGcedL0p1a6# zN!q1;BkfYp|KlKx-oB5*wgg^xcq{w%yUX$8*>k)#hvcc^}VwS)IfYlOr9oLc_R?zv;SeB5o{ z-4=1dY1Vgo6Mh(yzhcaxu%>-KRF+j0RHKGc6x9~sj`o6;*SQWW36sw)w;?CT#xnCJ zvt61ST-O!?Ign@y`H+p_JbX0GPz&PYli31lmE#Q0@qXhmRGJnFEi(5p(%KbXyWZpJ+4PWNKqS5Rjogqli zl9S0p52QlLCBo{bj<1%G<~1w8{Xc z?frtsTcvz!{j~vW4s<7=a6e~T6>lI%t6D$gEtmEw$Z^5m2-A;2I?G4bS~WdoB2d3q zi!Dvem{wzn(fALAwS3Ko&MR8Ab0O2sIA%-qIDkEfzM1%q4SIL8>WaKyZ~LjXE&Rk` zO?OMwp*YjTiOCByYE}dz5!HyHka=v-@wHZvi5_vJX2&*vnD|M);3pbbz&?8G(>rpk69X2ZyYnWkSgV^ArP&pI5UEt-mH>1 zjXi7lMdl>sG|KN!KHy<89lP*SoM(t|@w9{17!*6V6^Hx^h*WTwLP93U*bS>lL|${+ z^7@XsYmw+0<0~jK-vs4V#N z!GG2C-7$eFQy#3a{&F?sT*>o&I=GCf+5%CAXu;IKkF&N@`GTy`R9fd9f|t%}nkfqO zlx$&074!_x$p>W}Mf7ZeV7n#bq^@sg5w|u;0IFe*QfLY+iw>{Y6P~Ap7`FHO0t}_j zGj%P)Xdf%}{S1+u>70Ih821#>Rab8j_#U3kIm>0|uc;l&zjky^L|d&KZ7LQNG+ z#B8CG6eO$fog&&S#5GEU6QLc?k}u-kj;`q*g_ir~JH1ngga+k+5@KKU$6t#s2+}BJ z_z}VW|JCv53%QJN1(O@#pO&K6ENIUmXV0!#{G0#rmaCsdz5+mnggC8ab#kqEVO z8{AUtl$NWHx6Fe8k%!|;MPr?AJ2WQ;4ciVcAMbmvtc?-%sv$N?Bo+Df6Y%EVGB?`H zp+FRF2$B8EEa-d(3`s(<*ar~bot#bP_k)(NA~a<}BaySzluB{(h_Sv9lgNn2-Iswk zX3q9ncqV4o4vV2^SC5=9(XfAC6OO3;mWQg0nWHAFFYw&}O68T$Zg6&Z9*>5({O^0?(BNkPR8>uydr&5y`sU4DLUD}Ohe zF0@$3v+?`-JODa!Bg489wOi*YEd%WEEH z%4bX{O_XuHrd5qpPeR&6)z}K`vpp8a&fjZ?1InPTVRKu_P~+C|JX&?W-XtTUwWh^) zN>qq(DZk>%II!wjX{PnPU9LO+(>Qw$kb`lT2M!Xbhep?8a{X%7eUpJAa``<0k{Cd^g?@(F>zyP_gnSnv}>?9)`AXIo+EH^M`W#{ zP*FVZqjPzxGsFs=hj6O`B~u)3Bu{&E2L-BfgUuIId)JFo&-dSf) zO+yS2p0OAHAHq2Od@eh{eRcl>hB1TRDhmmzKIXgh{a|WCdNspJr2(0ptp2`)e)?yT z7dm5Y724_3Id%6idS24Cq#fE@d&-xmr(V5JSNvGxDJ$OcbQgiN8#?^nNwh?0W|-)T z+(b>Gdw5;FEr+rr#f(C5t51y2CE2zlXooX}rXXomdjRFW?ZWJ4HILvtKEv$4g}&+u zhkA_R{7|Ar_V5p!vCXTAp+c=+d%(}3CMk&-DR+c|)YjlD=k}{}&iQD@U-8#qha0Sc z@!(#&dy3S*Ptn0f#)ibR)yrVMAz?=c30iW(M95Igs;>Uo)`T5kI{<=7-=T zcJ@m`gPmTf)40<+_U>&H!>bc*T;tbISyS08@Jid?{`u_j_=$HPA1=!48}n_!qh`Be zsv}2DUfLk$;bhl5u<8|uVB2VG_Oh7c1(j=8$4m)0omX|S zWBcWmr^3!jIFI*f@wRgpa@sUa?;?h2{zuW!4U&gGuvbQR7f4W6ygCi_g`Hsf`v<4j znbXdd?uCbQWqD%9s*%HTnhpC%I7mJzZ))fFAJ4gUy#28JX~XP;*|Luof*XAn08Ghr zrYryjSd&=@ZPw9WiN^X76@Y{I^!Pc3t35cBabajrSr$s)=` zs?dGZPFW^zv!Kb{?1#qO=+teOF1TzfisWui`@0`a#P5_^SFUCi$2-n2nNztcC63RKgf7 z$oKATqrZCA68OW|HMR=Ky9+}y((d_~zVg{=XqdkTrZA$)5${q4K8}ndaAP?T6KcsO ze$h$&*5r+H!0mc#VXK#0Y`L&yajTbEB9*u4f+RGHIyB$Fcn6z9mpAY*Ll1_2$1(J3 zVyqeNKm35+tc3WQf&kVppEIL!zA28AqB*`b%sLUNk~w56i7Suh{M^q2Lr0Z|8k8P`AhAvT(D{cYcS+tpf8$6n?M>vW6( zdv~m-e6mLJZHKnkiE#LlPE`El7eD8iDAXMoZ(9-bZRD zwu$W)+wEGu6Nk{)(VaiF;DZAf^fQ0>Zx{7YP5l7cvxi)eQo44mu=5(gaADb1aeA%X z@(t3T2nY4Jv2&->4_-_{6j-2gTErLPBno0J<&0P5Mi4;p(-@x|xxKxRH$xZcs`oHp`s88m}iOuep;f62}{)T+*=tiX=#lBxcn|bf3Td*{rn! z$itGOPN$a;hx+rJjsb38yIrgXU1K}Go+;?LvKz?-ceE*wIM#LIsSl5^=-YMfTjJDr zo3m#RAVkEFnqzKY+M+qOq3Z*dr`OFE8yV2BX=l$bX_Xx7jlXl0FS>fbZBm97mNHg) z!J{H&N)Gbf!(yC?bUDRqf?AC&s>5^QX5AlTrnu$1cTNS5#J2RveQq9<1@Q!yL*>G9 zt>k|Kzjs~J5Z>*1QhmmaB!=C)ed*cJI4FgT61n-!O*aFVr1!1hMo>dDH^`^(u}~wp?6Ma-nwgd zV0m)FjKG|s9u-G+jB)LMDs^*!>$wLl`<6@#T5)C6(cj+j?hurG`;-ipQ^W9c+NJ*RtEI?xsQ!J)7|- zb+Jw^*?I8a%?wn}X9H67bL;yG;gz6%Z_^7^JvaESf7h}~C=_k987u#rs#_e%%aI3u zpHJl{!^BIVi74frEo%Mc1y_m?ISNWmhsXaKOW{0nAx0-zc$imcmtSVeAU<;7yC`z+5h`8yoBerYP}h|1xkQoZi;)$L0t7M5fj}$_4`@@eb;m0BK<{hw3|E)>H)CIr z4M$+5y4p7Q#H|fwo*5~F>_Rl*LkU@SCM8c8BP6P+VoSBxzLm~*4_TYb16 zX*-Q`pAmnE1#1#>SFdhhum@Ghi5hXB!SgW8k{n@S3RnOjsvcTvfxy)A0AtW)dH|;d zz$0u1!uhIa&TJn{z8^mxy14gSCztQQsp@7PfMHs>tTl0Cewj)@r?DZ=`XdRNC!+jr zk-v%24rnOfzjW?%xK)G+xeB&rSCDI^n=U(Dw;3J7U)PU_01>*qB?Q z4XXIPWo47aY?hLf#& zYaLS(MOjkSCw7tmeWw_dTnWO(LAa38*MXV|hOCeu74GRU5@}Onl1)EFU-X?2hepiV z<>!sP`k}DY%lHUE>>zL`sJae z1_yNqm6Z)fH@84MJeZ+$^)O%%G>xOx*cSahx#)_{s}d3Vm5kAjZuvZLvhk*&hcE#w z7%iEZZ#hkVFD)&Q(rI}}pcxT)+N-oZAWTK1#ov&hivu8wO-G9v2#8HpBK-%$o0V$$ z_${w!;(auX6!B5dP&<8}*JKR=c>ePRnY05KN)gIJ@RH5xP+_Ip*UxP=WXP=s(qGe~ zIBy3E-HsfSBOGE?f{jpZ; zdx7^XIy=kn{6zm>=XFdnmrcUpFrn;y%*!))Eu%(XT}MbS$PXhTQ39aOz?a6YfLvsdik(9 z>g5w;(tzOd{=)>sLH&kd`M6D*g*>(Yus+KJ0F9H2r79Ex>mu-#3TZb)h&tKm5g|Bf z7?c=zCwWb~Wu-35QrQ6|#|rt#{`=wTdOL*^S1-Zrd)2#Nf4UQ_ zXa*sAp0z&pSmO5?Up3>2ZT*o)N5AvxifszvYF}r-UE-D2Ev2cx|F8{9s%96tFHI2a zZXbOpWgq5~y0^`*iG7Q+JLDYV^5ZKP+4O zTfJ+EI5axT8c+;Wli#8q4$SsUC+QV`_%agp8UAn_B^r<@0U-=Yd2Mo8H?459_YSkMi$xq#?9807P+g~|2D2eJ>k z>Lm0>^U;kj+c6Oz6U80s3tk|UE%Nq1l1ah%Ys4}BoT2ziu^rP;7O^Y^Gc_A8YJ`C!$Wo2zmZM=Q>7HaMo{>GE!I!uUicH+rcj|Gxl^g8mw@&xZh%XyYF9 znh~NrERaMfXuOQ>5dt0ha_%VLwgWK_-^XH#=#=rdFcHwC0blIa!n6|~OMV&de=#ZG zbo^=`xUqLXq!InZSn3F!leK{`j`Cv;m2=d)Zo^-~F%Uw`vlyu?yP`xsggizQuhn`f z^X$)=9L&WHKg>fa9ah-F@l0CgnP>a{@pj+qkQC@#>L^*Q%^9XXy~R1cj-*eIdIb4< z{(cex_iH@gg{f*@Z!*k6<@77j>nVy@>JMb23OJQ7wkMqof2rKt|NEWo$U0^5rit&+ z+Pn##%~zoN+{fA@NfyFcJ-e_udr0V%D6ShFAvDXNSTFbM2hN5!82Yb~#-~usLWs`C znV*s zczrZcoJE7O&wH;y!3@1XL_*Gv07+@h>nwkYh&wF@O*dy z9PMXMRzJEgR)$8(oDChT=5TS7HE9yF>?7P>w#nITy8ou>*_vN*L~$7*6aAJSOIHyh4oSa%aNq~D zhsp%1TU`}kP?V6Nzs{A3RdvtNwwz32i>s=HGas!rrI;ofk5fhGD*f+u?U*QxmKYVR z5E%Jj0#kBj?63&&7UZM?_9X{0NpJw=6PO1cwn8%Aq(wlf#&DUQE_I=w_?%njC2yM> z%>#exEyGXx_1AWJPX*YJu@U(tid8Q~`RlqvRMDH|nIejvjPD9TirQb9Xsm8|4q6zQ zD@CbBHz4-IC=WWO!yn<%uxBrp`Ve`x2Cq~)9gGXEtXTwJo|5GTO<+CjKSI9u!a+E| zeC=q}!uVzxcZ?m4oy{6e>0guq;T0y}SVkDb3=FNr{L^D>&NMx$n&QI=5d!ucs@Z0- zgS>nMahudra<9!qLd)h~mD^MY)1JDS7f;^yo5?d2{6UMak__628Yj!a2{wusaW+Mj<=e%;bsl*9dvfaJGMYq>x{t z^nstk6iBSQE>ghN)=RH1d-xH^JB`Q;<5w*iA%&6rSkmCzO%NqcVcjb*ca@sR@CbA` z(_99>5$*4Bq$3y_B#jd<>B>3qAE|hlY{rh0 z$R=%@d8coea*Vn=AcHm{Z$+(GzOf$N z^R$}a7aRpdS}*(m$NeL{7mIG(NNJ+mFIq7TCPyhM1yKTS-T6NQH6mKck=9I>E2%nu z)!v_oivu|){CX|xeAafJ8)Nr%`LTS^%tt2#D)nzW>);n4+b>ygLPe_mc?;8IB_#2y z0wjmUH0unOgbt(k5?$fK9vet#%jP>RVeVX?7I82KZ8 zOT{EjE);yHZ`8hC`O%NWJGamr>|+ ze}fP+tW@)cbipJwI=Yde6xb;PZ{~E%o zWn|QUVl-okU|1=SF+GRbQK z7rf)*+ax44tTk<6lJkg^uDl_g6R`^{FZ*gM@m_UB3BXU2C4hGcJURY+y)qvt{OjhWzv_|tr*;9nCs{Wq|RlhB5mf7g} zLDgA@>-G6HI{f_`O@`fX@AkqQd+la*0%H|$FN=gO;!E#kE`5`21_gSJQZ|*R85En* z=3sLXP@&{3Duq((qDeE<3;2@wPhtimoJm;&H;WM$eWH%A`HdgGh%4@R$t?m0M`l(B zwRO>JF>w66gkq#rkWF3l5;ablv`p5D4}%(IdJIMieMKfq)KY`ROub4BaII#Wp_+7T zo9B*jntpb|FLF=O{* zHziD(zX1c3MPgfL&H-(P7wg+=b2RBsSlsa{<`e$M2m8T`H=pKdlRzdXj`O+0l`O{r z0k8&nL28c%5g@9ycZkdRGmEm*06_corkRZ24n0MK?B^M*lSDFwuEQ(~m=FwKq)nj# zP83RDBN7$QF74uC@_2G&2mraWd{P&|8ej%-^IZ%f*^qld1NIu00?DDUXE1Hjxu|fU z+;f!=WbHDlf(e?Jz(*Zl<%~w&k(7U(&>_AC%awtSETRZmRRq9m?j#S~C^4G^ZWaIy zMEi60O@Rx3H@xPrybGtEMG@kJcIY;|^5;j<(wD&`VX|7yyZP5Y-C9lHtK3l~(bjNH zxXeJ6A#!%*`=A&Krs;lCQw-iQ$8*T>)`5>Jm-;Pq9r8w$S9wru3q-wV@_VM1{}6vW z`<5$ui`sC(?H(j+w?uxG6r`4ST7)?74%lc;>8ge$@w7 z-eV>@pAgKxiSW{${G)Q0sL$s^KhE$RU0k(u>KAcNxPh`1`SCGML~b;7WDxOE?53#= z9TV%0;cqm@D2pT#fmYCt^MmQ+=#>{Wrd9eB7 z-nua6Qo0nBQT#+C3&E=qtC{0o*RJrsle4)7k9gd^UZ+6QKUftI4T9l4QyO8f=1Q^7 zF3wy>w&pA1rUgK?`~AprsgI@wqDSp623@lBZT* zTGf>JHJO9`U`Q^fTAu5NdvgbH9X^D$urKWN5u@is*DQfVcjQz-8@amhXasPzGN!0P zKv7Q<>xST_uxid)Mu(=FGC`;=V z&-tmNpA3U3CfSvEZb{qiQ(Hi!Oa$rYsCMmQIZ30(f~ZZTydHBNt1}x%8(it2U3&TK zb6?eAJhffTYt&1zgE-1hz3Uy$9QxPWC%_-Izw^V08n3N5<&5r-n@A6AqZ~F+3@kKg z3$I+-a#}~_uBQx+tX{TI4*u2lxvb)~nStf29{6P)oWHuW*ZQ=YwvNF$x$6RwcJC`W zSoBT35p7V=lt{oJB>GJh?~fp1c7FMjP363)-b<*ZGI3<-|)<;v0rWFh~d#G zI(Gs-YT&axANb9l>&u)rC`v&Lxt9i+JmG(HQULxIu*Vh4aOezD8YfY45;f~BhB&oa z4wZGLtp4dn1!&DUA!h4kYAE}&1!p^0to&M-8I>BzwuUNf5JZRtwv%U1oA;L*o#4;@}|JUQUY2CrQ&iatKFNvOsCVoh(SjUU_Y4E}7_?b&AKCBf>B92T{qs-oG|g4N1A`2YP^NA`We zjXW+5DqtAa5fd>C@ZX^;N4=O|?B%rrfKg-D@8VTN89ssyBbo)^5hXMi8$&lah^AQhqyP=&97w z%&ZuxH7lPk#a< zbRkHPI}ewW1?;>QV$CL2at;r$wQM%%9Gc&lLcXSTn# zj|!PRy*wv;*yXaCyB`!)|Mu*Z^G9E8^!sZ&gPoe@1`Vnk_0KjRdSAv6EPw2~aP<24 zf3|zy^KJtM+Lat_;RqN}jQ4Z|*^X1>tkklCy}^qp$d@HT+KZkv8Bfq8w8m)tVhDqk z*LClJpKkm$pCR_qiqq#;w}S()mtmqqo>;t$_)Pm4s-#wlJaY+y5lJW}A6@E}fbyh{ zS5@F@$npU_JJJz+7FM0mERMPqLl1ZHy3<65K5_Q!B?PpU!nqL@QOu)d_^*R72JRTf z7(T^*!^;|(#3k8yi7r=;w%|46@MT6Kt?COj=J~CxQ#`ro_ibo1YMP3HO|F(90vx%o z_Cok4jPsu@{!n!XsLzOG3EoA+v0Ek!;z@lLfJ2y%;O!0;_mt*zGJ|BEFg;Z7Q`cC~ zh~1>N6vfEraVO5G@0-07;zpx}NuP zbmCosOA!{c|MV-as{zjul95QNi7PM#_HeX!Rhwd2AEA@L+92JbOG~NZpk)7qn}D zv9DdF zxw6DX6z|A@=FTZUspKT2kpXGk>$86Qan>%)Fcz^*8Sj#U=w*c4O1D@d2URRRk|i6L zQtPU^u6B-_8a#Kfc=ne&Upi#-Mch=Me0gvND1;W~u&Rx=MsDbU^UorLA+lmHdt^Dn zllM4wV+Sd*5*Ym2;yAjj?F7O|z@N&|Xr$~KtVT)4SBRTC2u#h(V$;=r%QgKvzhmFh zA$4W)dWXQr^@^-7HhOU8pSOk%Ts`5$fz+CXb=(%c?bG?lZF~OHxZOWe4|bilzNV?7 zui%LJ;q$0X?+;F-+b0Uuu|dHg;LdhWP3w)dB6V6r(%qbSzC{k$&BTWzh)VKy!$ z%tHb`W_#Z}*S6CD^_m$GG}G|~GzT-@jJ^ATW}Xrw+|zkgeiRT)*VaZm=4I9UlDs#u zocUqGS!eC(~}mHNhDX6L$ZBn2m>}_2xlkt1(boeV3Oq ze1?3PG-3Q+Oz>TV1lnrxlO+Wbi9v%oOngK5v|sjQ1+!md-|di1;m+H}h>*?(m=cbhnX?jhp@B<*tc?lGnGL9O~4mgyBFby)#OwFp;9n8o#Ro6(h-oSWLrHv&m@>2AG{Xp978Q6EV5`zE=Fta(9fg`wuoqDyi&_nQw2P$v*FrgCRZ;DV*3^vmwdaGN^;q-#&D1+n$m#B8jWPxY2QCo5q$$KwIK{~l@e~Kp z8C0_cteU6cCFUXwKx~4WL~X4cV=Ep{Q-&+VL)MN;Tn3y{?zWlfR`(o#!A(gEd~j)V zBcntye=-UwKk5=jqMjgQUih9y!CpR~0%R#kJP_M$4iF`xOi;<8fo!7BQyB;Zaf_YblMoeu?b<`o-_e9Ei;UtUp!z((Q){=aYb^v`N~+dlpG|Me$p z;+y`*?)SgE(&Ma`gkdKw*gMgWf3_gejL#jkfv;}*5pfO=X!Z1r0D&5O}*5PCoGRYE2YAb`G03Y`31WL(R#I~7B;!ISZ<8klJ7s!A!|uzZ7bVY?+=Rf}-w z&_1FMV20i1-IpLeGn)^VE=<)^?&7 zHn-;D`OUQ-mr*XUr?)p=wDQ-MCDtlUzex*f@YR!RHMi_j{lDtZkE8KlUt^~`4dGli z0aRwpZ5DFf%OI=L1~uZwRgLAf{(ChBwR~>+?wVhO;2H7^If}^_S!eT^PG<3$3O1!o^z8BbnkkCSK%0Z# z7Rh-mobMS;ugS8JMt3d&zAP(QiGLA?N~l2336#=3he)gWG{cjCUJElW7)azr-q5LN@PRuK@*?*_w zWMq#`JyVBF=3BIokWT-5;`vEXk(Gu_%G8j_%u2sAbNJ0n2z}u5b_?z9fkZCMOz4P} zVED-m>K}RJz8St}v@sX{?#wX_2`9PngBdVtdhRoRBMbY>xH<%#z@TBOpL%bM$&=Ph zl!$M{HjFz>N>Z5J;#kc?s|T12_z`(15aC+5C>=(Tjta3na{tOi+UJe7Pu}lf9J!RX z7_zfJ>Z2|ikT--)VJqHHb>V|BFnYza0f%|#kv_06jQ zD-hc9Xr$F$K)!O~mcJLddrjyc`5$MY4#38@IS1n1S3T%5b2?hwu2M)>SKH{xmjl|7 z`w#`CzYZBky*Ym2iu3rOi|syS2(~|vG{3=objb;t0@sIw9 z;Vt}0oO4akGN0gYJJAJi@LoJxxnAQK4v~7;A*mm;L6mXF=eg5(!;2+j_3qtaaO{IG zy?vcr65P8oF#<8+OqL}8G0&{YI)M zs);w)df~z0St^W%7t=n@zy~n3Qy08Aai%%>?k`SMjLlrPfotgrn2#me95yzIxUk2B zs0VM1dXw?OOHOUuHiCeW2eFSQL#tX7UzGZJ$>Es95PA?9#h1x`7$GtoLs8#$kkFZl zM^MrVo;NwTTLVUC+_AphhOkCi4A=4-x?`3i4)5Gn zMpQ6|bZbxy1VYS_K{4jG;!NfV##NoMIaa=!PVrgPHp$S*;V+&!fYOV5WB6pUlNeRo z=;jTm*v*;}FP+xkpU8}f3lU6%5u=fzu{If02otzMlu7v@jh*=o#Y9*hor78Xp)5XxJ}-+6hppHZX)-OCwsmNuZIT^b9#_>36q9w6T5M{`N~BOoHuV|)~fk! zm{pBhc;djhBECIn+;4AKy0|xcN*zpZwS&tiH9@>%J3Lclrlxel8v|=y2U%*Uj)hRs zLEpY}XMVe}5XzY@^!pI5qRn!+34Fbj`B-|4w&j_yWP>KNlP3s?1k92}AKBkYZmd7r zZETu&D2dB8T@GLT+pawR^Y^yYHqRIS-A@-``E%%wSLP?r;T+Uyvk@ zvM0jFOhq8B1+XBLjKRdR8oXeyu>}y@9y2f<5n(NNEd>ZwV52f01eXZg1#J<1ID(@? z8FUOXqf+>;HCI3j9F%C}R8q7V>6BoFq<)x+f&8&A^CBohzuB>o(#&r-^;Vf6gjzx} zm*sa&oC>u~$F?>RB9`%u#z}UqXcI5Ev4QgFg?RPk(Dw3y5sEEf z`}VJT+SK-Bi}>7$&_p1C68pra{4(Fb$M>H*XzTW*zJEA80*1p&TBfzd{OsUT=qa~j z(q60i4fd*#ol^^3VCYscHn)-1`g(=6WqIcL#|?1ss@{yo8&d94uL+vP?U#jEp)gt6 zKP(_bsjfgQ{{jOn*&w>65K&>Ad_R;3oP7qrU~pMtK!{{=X%c*Whv@`Ooo1LfK*&=p!4du;IdW&D%^*k2v} zAKi#aBs-eogex`69Gf#=O|$w*Y*#@{*9;vAipy#07+j9Md^dq`TOo3I14~vtf6!Ku z0Z0|m@!iny(?fAw<9K2hGURVIJn83ES=SJ#yrR^+&C16$z#UvX2Xr@>&x|{V`-mdw zcd#Nv%Xpt3n4y93WlcnuD{Kt@$cONDs~INlm7#kfe#yer?%~zpD^cONT%gM>JR7-4 zcZPg(3ge{;Zs14*p|cHNj~Sk>$%Pe(`)>2`F>PBW5j(NO17s`FM?}#?4wryCc?S~6 zfv}aJtM;g%moh|j<@6ax2T=fIb=wuN`V8s0&QU4yp5PTqKQp;vhK{kJ!@G?%zR1Dp z!9f$}zZg1;gg}yXEP@y!H+wT^>Ff|-r`)@iFFW1%L7sv^KpiqAC?p>%Z?DghxuhNP z7(!mcce%O~qLQE_B!YQ*o@Ov$bwLHvM&spmShjl$MW&1dRbCq@R-;K>(Cq>8mKVlg ztQ5l+Fjk;kWH!DQ(*F*`CA0IDpD1<;zUX3QMo>1i%c?v({0i=CS;g|a)l&J0&#+Ks z0_m6H{iMJ()Ld$$Dvqw2b8Zpa1}QChrPfIvC1Fzvc+B%;t-_fSgIQk7LcqGy@~>c^ zy9pu6vU|SatLw;BL<}2b#QEO3OA_~h{*9N_hbxE z^w`37&G@AV1ze6E1MG7Ppo^rJcs&4K+;3yEZq(zRw=Bv4HUY!&f@Ci0di+*xT#x_tBZOA7hpbK4)gupzCN31$sqw0c$2J1g{Qsp+!H&;$J}`cYqfuw+;X;m{gN3klPhiSzGpBwi;YWssSx zl~XR8Ctaw-jdDQt7pzLE)p*%i@0XK~N9m66YzA?MPvvbpEj`Xc;^lf4Y*ZszSc}#>R$8MB3}w2{*Hn2P z9g4!4eA`ia8e0*0HC%yEbB$2?6gj6_A${x?6ZBcp*)Axl1kTuBaHQ#wbhhlS{IcSg zhXxh{QlBxV+%%w4;nUSpQWU39G#RPyDwp{2+)jyEqv z(LjeXx2{XhhkS4-KVJ2mOL*&!C<$2Zn5qrit$B(Ha?(-^2;1%7OErRmmuDLnYA7Y6 z)tJU^^4({EKbk|SNDDcetmw?k8=(yP&TdN2vt;2)+-s zouU=I8lhbYQyb5Ob_eEG#!G^E{iJ5!@Jxe5yd1krD9JVpug`NRoPy>bK9hB*VTaNj zCrs8zJrrnV9!z5qnewrRIw+y&#wJw4k*?vo#{=Z1e+F-c&GdT4x|{->fM2N~q#N?g zk$=LelFMz4R8PG`-%#5dKXmUF+Vp8jw+oi6eTywGm_Mkg_r+Te&%ZSK7%594B*;RE zXpjOJ3a*-Ycf)pQSg0iZ3*GK}_YNRC^twt978^9x7Z`H{B`^eCZ43s)DZ>P5@QF8; z6w4=PTrQi?Om!NdbDB%w^i6XyW7(SediJl&z$VTFSp_`l>yGbWX zPL4xR_t-(_C?iAV&XEm6beFu;2`F{fR(M@gDk(}xN0VO|#zW0ySLrAxuVg0dkQau^ zJhqo+W0#qZ_a%j(*k8P(ElRVOtAuu*xzyX5dt)kzFD;waVDS!mE&)1N`2EUV+}%}j zT?b0KF>r3%eiY_nL#UC&g{m7&)}UaV?`P+tfu8wBpXthzEZ|}`7YO>ALnCHUb+b**4Xk2WU6eB6_6nqc zs)tX_X8{~allZ3SO==o{bNo~)h%aCy+^(_y|J?A`9Hg(4c3i|6p>HYAYm#gwdzo1+ z3Kbb1M9@<~Ll1K=Iunfq|07D=_gGv+gE#}RcUL@-;g&PT6WLb(GB&N3)%~T*6I5!lrl}YZi}>ki{;UXPsEUX4jq? zmYi^#U~s~3Y~JY|fWhMorL#a*P>xb~scLg#vCa)o>L&s~>D-Qg*ucBnF*7xljR%C_rgV+bAoNvCLJ5K>ReHoTYaQdPBb#$3bz_ z&c^(3fX$VOe0t?Q%WTUel+uNik>CywMC=j9#l>%Vpvs+wsJ(VJ8IR>^ zIh1LzS0{~~FFX(u#&wl-uw9RDA{1{6j36d8hjLBP8CO@ zlcv-cj6bkkfDCJ}+pOG6ijfq|jgn`@ueaQ!ElOikO_unadlqsrs1*W>1Vi58nGC_Q zWae9|e;Y)8E^E7p{KEUE3ieJDb5S0V;-HApoe?bwc}TT({sTB}+#wZU{#N z3@8;!iN+I7yMUH2W=0N$&t;u_>vg*aAjdhj?I+i9Y^-vd-#3*C`Q3hpotxL zrmy#23_hj)a2GoCRi6<(7I~Y+4H(f{5z@BW8L!HL4|ko5qNo+?$u!HHae~(b4m6Y8 z$|_Cwr)7N>)yDBAtr+8Fld(1_;LF|b+k*<3xvv>h0Znl`wOBaqL>zPmGt{xGpSOu;rMJT&y1o~1}3nR&)QT9EP;dAyy z8K_(=$;+&w-G4^>1W6(~af#HU2gYHe46I@!8y8~`%p5M?c2YJAbI z)T{*#GuS^Wdq5UKdZSB}j<7Jf}a%_?J1 zqZa1a8@=SkMFf=!Gsft+VSgNVnL1dWkoMvXMbauAEu?H%;IAu_MVV-X5^;pV7(xbf z_5ZzE0d(Rj6oN&nYtEP8Pv^33(*C=@w3l1INF&Zl4eHkz*|WN_DBpcL(Y|`4*GX^z zz29|ADLP3Y2n$5{KcpDs!C%5bjhw4@a78ooD1B^p5)T|P76LdD1^|=Fz+&Q}>m_&= z(O8OsiN|AeIWkby$dU@{L~R&IuXb1q)2&M3I&#>UQDO{aLYl$pxzCFNFMqC3V2u)u z{sZHZ(50~ZX5O^vH)*TN0Y?mLkzQOo!6^{n&-JYQ&S2nB0grG;CKU-w#y+BU_ z0%TbEli630Ns}ZVQi_>$4oEPaSVJNB_lYN))C`M)D$TRDwa?TRlUX54YHT3Rzr68e zjPnwayxNKa-Xhpomtm0c5qtS|a^j#TDcdd$mR%WA_i;!V>)7+{It8~%mXLEB zj+Z;kv6MILLD%h;)Fl~K!3VSm!`MCS#$&PtgZkmgg1tU(9G#BuW}Sp38C~mW>g)y0UaC2`s#4A&LsaOYv8q}Y=6E4P*OZ_$F>?~DJGtKfm&1_xyN=Bbtz3} zW(lXG5x>qq+W|+XdQ2IHVwM<$tIZiYoM&(Ra4e03Y=%{aL{*$-y^TYS{vRCFjm_yW z#i@@l;07D*K~>(D1%Kfika2%~b7R-2y1W3oTN<`IZEq9&A;pW%luh_B60Xo8ec;yr z_w5XIF419aQ<`{5n%b%5Oc3zerJ+SP{{H(5Pjmp0wrgb%WeNn!yUlIlvY#m`{ziC1 zS2B(|YJ~39=){{2cJ55)adRo*lv^)0S0M+`XQ1DBVsOI?%wCMU%9|8~7a4XHp@qB5 ztwbCZYWL{eXZIUM4S*VGI|5FTSwAR5hFT<=(Q^OVjZ!Dx7xQim)@R)K& zjIDIEMjTisHpf(W_Lj<(GY@9Waf@aWJO_Yr6pq?W09st69Fdp7czTy58Ie)K>BAQ7 zAUcMJBet8y08QP#{z%?QXHqBpn|LmkquaPSt}3l1ni#aiO0|(8F~tjqf0Q@CV5T?B zBQCnFnOz})jbhG~G}ba{vlQBSdTNj5fV}T2r-lnO`O@1hbY?BD*{~x59r9 z)Wr18*cT)c6N?ua_XV&f{}#^{$ICJ+1j!bRe(Xv(4HuaMgNo%f$p7S=Mh4y+yVefL z#(-AI_C2=SwrZ-|eNAwnc{ngd64eKLtXT8v*yF!*d{1N)HUp4UCi!5SGmNuPZUSUi z-Hk!14xzcj7jg6VqAMNIGwWLrO-w8QBK%QPN+ibi zS<_{tyF<=Mu%9Q930l=_*~8bDC1Mr^Ow-f2+=-%?2rMrn1y>mWrwD85#@tj1_jX2Q zX}pT%JS8NO*_ncWLq!QSuF5himh6oFQABHP34`K)ey?rEGT)BLxOL62s;m!$MBRoW zD+dfo@o>)=Nut1$V`kf78d**l%4z1 zSweD6U~9G&1@*cytQPOoU5%-%Op~R0f7AGatEo?xHC~D0Jm$X9%u$5}eaihnzQQIP zOuZRF4Aw3zN22?RwR=qyRh}XBRqI8vzDLp2O!Y^(#4v0FjNxTkB@?e&UOx|e9c-f0 z_OH5o$=C&RSs7q2(|k~RmH-6r5CTvagV!Vz!=*gWU;z+1MBPmf#lomB&+ZRtoU@n- zg+N;euJ?478PG`mdRHnFh5b1nY!1WPOS8hwQqSF^zD~2z;OTWtsXQL|hUbZ`X9xl}bBQwJnsihQnC!eX`*My7U_k_n+JzPY}^@k4( z&(w@7Ce_VtbZCu@Af1i4{$lc_)LK6pOtT0P!9*Y@TfzLNgrnAlokpNAX}biUHSP+J zU|}l=+72o$+&Rnn1D65bu_ZdJnSjH|D#bEx#hE+T&JNUMr4Sn&5?7hP16tY5^^1bL zM18jI&v3T+acx3TuK@Dp$(;fPW@ASp@ElbE_;XTU%D9-6qV`GsY>v%xV1Qi0Wqw7A`SZBk z@{&BYI=K>)7WzLOX#!V3I$Cd87<1 zJ2BE^x8VXxS0wR0kf0gz*8V{SJlwmn^d5+-VAII0AU>^vi4YD3hXB%eg8 zqHWgXeVAb<1*P0)+}+{ zgi7BJ9rhAzNZR&e@ioEbrLTd$XfRGP%AuQvGo$SeQFQS&J}{4O-(`O+Ty{}YzmoFg z;Wpz9SVaqSFs!M30ZG_1bv*EUKd27|oX4|KB*4sIn*9Ncjoy?UpW?R|`5Z0=M2k|f zYzj2!!+lPq0kVYygkJ&#fRaBwkL>w_qRN1X+17!l@+m0|ywrD>hR?s!Y6B#S}KXWT;QTS z%ve-RM=)3qv}(y*b@!)Y*i!OKLumwxKFhd(&z+uUl{??qgf_+X3Ea;Cw|S%+=CqK6 zG-0x3goRcI+*S$cb1~4cO#n94sj42HNi!X@%+cSJ``=dW5K|knzhI5!T(Vw%8bF}m z?WL52E^x?OE*qKx+bV4)O}oR&S%q;rp_&@0CvypvnpP(apGM79fs-_Y^y?pq2KoFw;jO$1H&HdqF8^WSk6%oZvkc-1tz?0Z$iOeb6=qPwWpx`fWq-DuPpt7n+bHv}Muvx6MHhxO25C}%QDv{6*xB`HcuVRdX} z4%;!CujGC?_EfLR$a3$wqKcE@m2b%LV54s6J$5XsBs?Q1{j*@E`XjDrU{t^7iY)px zefp+Kda33~77kv@(EUd$pJLAb{}KG;Zo=wCVG-cgRr%rFO)P=7 zsTBnJIXbchl+t?c(o5!G=Nc`r&372jP%vYsZc4mG+sUc19_LV2?!G*?f|R zdk$Te6Df0z-6gt9fj@n|roIZzOWMo7^D3qM>8e9cV^Q^F4pQ7FD37lgCQF|xF10>0 zlbB>BaD9aXC=&@Zl-d`ZMRovP!e>k)#362`YB4t4AY(qkON$T3MnBVbuZmq3dRc>qCs~C>R3Pee5@8XBBb?S3}(vpx~)EW>2 z{}Y6vPRXaR_GQmDzAET@FQ?Azk`R`Saht3D29|GiG)e`U@gKaPvsB9IsVfF9DEsmQ z$aYw&nQsK#OLdZOM`NcDK#KQONnj(qQP19g?)2&1n!jHK-vIGzA)lQllKoB)O_ZH6 zYQX1e9Z?E-r;%;tv4v!%*1FY~coyNK^rLFBrxurY!hl-~v69yF!Ip1WMbz|9B-+j} zPrS_Gqe=`XprPv7@2OvjYoRdPF|oTyt`ZeMK5U{K(T^qLbvr%Ot0z9La-JI9DG^3l z-0NNwB;0}V+nZy1>MI{?Cm+yJxb$W1@Ky#XhanYWNQu{}rd?pQHwLbt@PKpc5i&6{ z%5rOAJ69kUY#X2@NS7O>@#Dp}pHFs+9mLUo=EoUlU*O?CpCCxXh`Y1OI4F@k-!g*e zuC=DKnv>vkdzE$ciVw999N93k_O_KXpdP4QX@iI+QqnacLD_#LOfC&}BzT`VZK{tY zmsWLb03~-bPB(T^!V6#2Fz0~e_KR9&sQ+xvgcz%Ybj9lG@(tnh`d_;5u;{r>nv zAk`j7qF(XDua?vTi-=YTBrw5hTj4O4tEyObYHynW~x1By0GZ z;n4DB5h$tyeqZ7y+DSVZp&Fh*jmAb79PblX=e&6F;my5Ya_SDf-BQ z!0%eFsVDOd?2+-Tj7pi;vGN_g0C6lAxf6&+F1XFXKIhh3F2Enc;F-Yhhd1P|X~lIu z@G`3HD{8cv`0GJzD$)3VedA1dNIBCNrT>EA=aEEnl(FP_u`s#z3k9d_?EANcE396g zF6Dc#ug;#Gq{jnEk%~`+Au?d8-X^=P9N83b11U6j=b*}tSKv+nV*;#q-;Ra}tQ%)F z$pHYXyv2w1*;83!kOO%?>TxYt#q70UygXZU$ma)~e*S#>Et zT#cwC?H0pl)Yn!I83pa%=DA&N+!85Ra}9=Pqcqx~#rZ&*bwi(16CrtlK!n^>`0BD@5h;K6gY61e^ z=yTKfw*v^YQ#6zY^UAk0pV$mjpFtKwJ#Dib6nBNFvpkPy-{}pgpbGHkpSQ@Yqa$&# zPrp#M=#5XMtp|25#j{BQYICrA3`PSO^qkS;rl-{p!!uFmz5ZBYDN=hCT8++oB=>$E z`n~QblxilUjPpCO)p^z)1hN+6ASuhfT60V0@_K7V3}-sIjjP%%<=@G?$*>R<`=j*i zwvNCl>zn^My@pYBK&~OK+C6L%AdU@J_wCC-1#(W=3U%T-Lw;X_Ui+`PVmTa3!+-y? z08C;Wsq!9X9wcI`6e}$b8NO3z$s(w858uxgwFfy(sR$5x z$*YF=z8!)sM7W`vwyJMLH6_f(B4i~ZQdPG->4lt|<3<-lcYA0O9*sAZ``H`rI`xWa zUvhC|h%cqWc#S>`*;=Tqu5bS%5AUzf8LhR0=~I*>`$4NQ>Q8b%GZaNT*;~_>SUdLa zf>dKZpNE=|9irk&2$ofa^ieT#I0krX1^yMQj8x&)bM{7v)y{zz?_2rXDp_%*UccT# zgLFE9WeSScHjbn!Pj^sV$@A#{>(kauIGKkS`(SeIq~?;jRX+(Au~+)*=gSKLJn6I>guoN(XA z)Nm<Os?2YKg!)_)pXAINV<$Hiq;0Yji-1D_JGMa zB>)J`GxL}E&-DllSdGZ6M1w*(3a&phWy4%C%s-H?mEeQFi{;d!i$DoYw>pZI!r)hk z4Fq=taK5G4(Wljmd(SqH@>}hT zOLS_RmSU+1p(9?ds-9Of2-lEf<7p1r=hcBM1p=e;NBnX%r;_;0l~8s5AtsiJO&FJG zmf=F7piw~1t`L)nnZ=8EB(h+XENtbtXfuL@i8 z{YQQr71eb?r_*U6s^7Sx#**|RRX1jUswzF1iK^%98nu_CUwUUHgfI=e)o18Jpv24x zjP$xfhA3S1-M4+WjPa`!hyWpEF@lJj;&BTQ`{rR|89QIiXLYL?tkx$>kd|B>c zlc63>2cX~-CLlSJ*%<(WEL{j4K0Ocb?llCxFBomY&R(i)A?rz|s8%Ju?T z&DtvX#m;(@Y=mb;zzePQaBnU2$AWZmK7Tw|63-VHWs9~~G3M3NtZ;2^-@vLJiFDCT z1pP$ye?@vyIde=VGPA%s(Z$o);Cz`6wvge>8fNbcC@cY3$!7e38-^i8{X{a(0-1Mg z(TN2;yb%=e(i+NONulr!fG~Yt^ zKADr^&j)O=bc((=i&#v#dedTf0BWYCMHvSj(eXp=c{H-F#?BQ#cCFy;EY6xLM5nYS zr}4mn1TMJigA+5|qM9*LxA^S?&sfc?%HP*<$Y1y-euZ^~97AbczrJUxZ{l3KBAP=u zAu0qqFjb$JYD`>+`}!8nE-xW-#Er|ms|iy+H!`sE&JeknN~XgQjoJ=%kxRrD^mR_NZ-a#4MU@gy&wm!#uaHkamUC(GWDA!i~tv? z>|RT@W-W%pvO)fdj08kv-8oh?;V4z(Y3gp{QYwKY*u`86C7Kuh z9c+Mjk6`y*vS*S@Fh)hipkf{0ihER3{v3$LB12D4k95S9JmTSc&}v$mkp-0|<_|(v zeLARKnk8UN#NupV2Dr;`-VajU5Mrx=L_fg06{)bOa!@BwBH7@=vA{iznit1|{xb?E zwlfAI(xlO09P}H|d>NpXdk!4}s%4J+mrpRtYUFiw2AY~cwy#KI#utclfcXZFbBZRm z>0(p-$0iRZpCB?!qIQfq3e$tnzgFJXli?Hf@-(KRc5t?p*TWDtLPOYBw0bRo>NKt^ zrBmFa_BU81D2;ao@-ceLmCd2;<-bjn!6Q+h%WTBXu+$iwsUS8KsUu zC=|pQg5S(+%zLP81~E-&eY>_4G7l)Z=K>wm{d{;$ z_G!Wt#x?%k5OJ;#ybhgCsMPV}OIS7K!31e@85bG#13qdssr%sl^b(aD8U(X;PM!`g zdtgl+;z}9J2l6qN=IjS6whPP`ikCwZw%W)p7>S8-T|wD|=>VXn1|NZA&}jh~x|MReY&4v)U*Py{ISf(BCAJN)mMvqsR?Nd+b<=kZhk8mwM1 z-Iz$4L5@&g4bK|{(_K%N+M+Iij4+_29Ct#PvLv}V{KLUCRP+lVyDGo+J+)TS?LE}- zr7?i0d&IhoJ9c=jt609BHq@yVn7>^3@`M}Id36NlgJ;BJ>rf;%u{?g7 z$Yv}>E=7%zcYN-ht(_3nshKzl;7yf^qeZClG_zqzbTjc!5AXP6H0a?BsnU=|OpYfC zCh~u40MRh3e53ssfI4SLBH(mNH34iZgL}0N35PfPPkY+s#;F1tI!0l9q1YK%1pC!eK#EfhB6>U({<)(fc$FgxTNjW$G)jN$|TE;&|h` zm>VB(XPdy^ucn{@Q&2GJ$zqySPm^VTo<}6Z=<$n?CJrG)xwN{RAsyox#AKVUhB{2e zuW4P(gb*Z3t+uLHq4f>)@H#hB;|AkllT1k165GqeIu(zQH#MQIp8n(T^ot^%!Mv=^ zgA6^8zo-AS(?xtQxgPhDJ>{=lY`~fQ?};|#sm!Sur%> zMv(sTfmQZuIcP^_!&~6cM4_Ca6&xEOaf6=y3)sA?m4Y%5H3s@J`WgZv4Cc%ISZSDB z2uHs@*OftD4gE$uXsxmx2VjyyUB((NeA(y6RT8Q65JV#QX`bYW@2LQs+MpHMMKUx4 z66yyuGxL}!orDzL)J#I%#zh{@S6uNi-9PJ136)w#p5uWE7qa>qdY=Q{n;d~TbmMYp z0B-wIfVE-%tu!w|bj`6GOFv5~oy<@vNo}qOU~U#Q5hM6;$g-#1m@b->DURi;qCzK& z?rn=1-p?;H^^V4qA7Wr&iQOq}bC;;Q^wUN@s_`nT%e0kpVBX+DVnNA1UH~^sC$wF6{s~n7X6*^Wa zm{NZ$TVeN`M)s>LI3pU8B#NRD3Hu-(I?AO?AzGS_T~LCZ;<5@53pfcCv^7IBMXT3w zA`5jXvXIJ^9||wwncCWZyshF0v&46lWMCFoF>s|k59dE8jgJHl{X|*WcwrhWo#4g( zhZY;FW570I#bkWi+GMMTA5K5BcB`UGYUBJLqxg<_L~-;B5&I(xKHfW;vb5F5F4QPa zeUv1?Vzt;KV%&qpEjhG`sP*8%pYZVk2cmzzzjH^o8e7I%7|`(~3QJl*tEW5y&BN?X zQmG0N%wt=swm1A5O&zkJP0THU?KnsZL|xArJUD@|u-FoB5Tl z(fz!9D~V>YVvZ+Q)i4^5TxF}^#0KYDa17TuE@9HBO| zl%1@^wA8l8zi6v`)b`5Y?kr6r0FJ~rIe7Wa)>-Z^q4u(ecaDa!4R1kIh$+dx4b0vZ zJ|#j_0oUV<$z?{jDZA;9E;GJvj&}y4e@aJ$mDGBoyeNd_C&M;Pwc74?D^Qz)(=hpN zM5)L!gfWa_HXV^&;5sRT8cO^loQj~CIDr1;0MnxKdaEYilvYx*KBBxB8Y$EY*&3-I z#cY*Jq%1FhcXEn`yDAYKoCzfX8(bBKa(RfXah9apRs4pBJzpbe3wfLz#%HF~ z8{^Rg7$ZraB8H-4;O&ai#a@If5C0n}s08azMTr-(W5)KRu1>;MZYRh#F}Tdya^ zxwdPuBiJ@x|8yxH*9E2qfFK2@2ApQy+zCB! zRKQC`mCEPd05Ar^0T+6t=fBGME6iiE0uL3g}j(LN``5tKUT)l_lM0Kp=foCe* zNHS=o&rr~T#q$jbT{<&mIqwRzuyU@R<24L4E_vH!7!VC+K6TP`x7!kmED^v&m5X1R z!A$_Q;UOl&dR9J-#@v~VRU{b$*P230n1!daaA2dtl454nTqDazdJeLygj+xdfswG` zv0)XA&V9s9%eWE{o~7!N@zYs4M}=-A!mp3-xNh?(=G725!RXnuMY;o?hH6;>YjR3z zFvOVPR0UKceu1p@Ga%DUMtG#c#DoM)ZUFgxgW3T$0k{z=nuHQvc1BwSOU+J^ore(YHeLj@;Hx5 z`+c5HISmvg+!vQ3{7^`y`kjY8zsY&W$fW<3yHdugNWCSWTYc>32Ad>V=mA@aIJSFb z_H^R+gr_ZU5>PEhE%8I((&R`iTdK`V3Im;_L< z)SmF_f4cnwC22cPbQFUBKuPq|q@IUMQa9Pnyu)M@+Q|rH4xGQ!I0*10nNt$Z#`CxD zLdm|&-C4rP$n{lUG<$c?m)ayKPc)u-Q+JZk8qpz{XR7J%VM)KQqLzr(HWeH@>@vRj zW+O}@b4vR`SL9&9#Bmy$SCq%&~H&kZ9yhR#7KoVY%;ujc2}#o32k zAjZTzaqxbCgJxDp9%u#MFIMeRX^$TJ+i65Y18%t!;(rZF+8(qS4BDPzig%u1yBlEbg?N?S1393 z_sWjLAJII=QIlQLK&iC9p6t3_H$Wp?k!E#mF7_C~7HM{CfKBZUSndL5EkxsmKVqA` z5fzWOznsz!o`R17?b*e&&e}3b=Y`*F@XP54wXMlfbfYY%m&;lnc%3Mx2}0FHoZL5V zH*8n7s}y!QEh69L-4*XFr{-Mv2sJdf5q?xz^v)F=92;=gXd`7!A84gM@z64k@TU^{ z)q(|UEZWOTJ4F@JhxQA80yE9V}-sF!$%B=2?aHl1%#9$tFgtZq6~!1ugl2 zUGmsBdluRi$?6is57JDeh_TlOE`w;&^W~8%Up$LBurX$nR!yV{;8wyrd>u4#EK%1C zO#ri1dbxkn`6kQHO`@^Bgg>9)W8x$yVv)Sft3}@1E_PDvIzXA^d5rN2#aPe~ zh22Q!D&&fhpj1LX!O!dBXz~$UMNx!I2(*5}$ml2$#Lf~J-UGeHRH=501;VPv-G0S) z{g_1Jxu3Ux5 z^k9Oc?}YJVL9~~^3E|SlNW?g4iz1oGBkBn;sTTsJ)1MGIRV2Qh;xB4*E9(2lCgIo# z-)v&Mx}rkJTU{{V;*Se;1ZmzSMZKetkz~09XiVdI) zv&5-9ivsa^Dw0z~Zf}*e|0DA%;{ND3h6Q#?SIoxDHq`zqxgGN+#a?--f$A>bl8a0% z!X*0dVd9*0MY2GQi)83D+mQOy0pF4^te9o!b%GOfJ-Y7hFE10T>81jfM8g!R((h!Q z9q~V@9SxnqfWI9c{Ha5No74-ZARly)#C){;?5oOd=%TJ$5zq1g$$lFE0qlvV!ZNl@ z+rmnGVf~$m(vcwTacl6(4i`BZ&HZeNj)%`(tB6nPBFxLg(j_sv%`K$#o1>LnT|qPjz$4;l*pwf0)=Hkp>mpG8%W=&MA+b1xYJi=ZeEWZ0Ak9mPNvNYc z8da5(u?nt5N5*Z6wD7d|CVZaL1=%h1WZWX=yojv=Hmn$NFoZB7?hs2o%Nk=XyRsyp zW|fZN{LD1B8T)!dYOt`H7@58p00=i+Sa38v9XHs?I`Kr_wv?P;EQ1#ush}}modMWW zrF~jjZ(i}h>x>}bM2l^QnIZ+7XM)o7mbxlxmt^CNLCw(RLYu*#O}>5$F)C}C(6t-Y zne2FeTvCfB8yrKP#%HG308e`m=D}5?6^uMr1YfUOQHd?5Dm|6_b^qgcnqe|}@y!EZ z_1}JWevTW|sw!zTUi4v+=N`?$lpNmk{15PoB0|yHhCCRPQamfBbgFFMpaSfi61$DN z0OUz`6_uiJ6dogjdoK^%g?vCNWG?;ASL1{CMgUOaiOe_T5>p{?y{^2&0fNu+m)>u@ zn8Sbp>W;in@$8}CdBhl*3rxq5G9%cAn)}nQs(ML#Ii=I!s^4P-DbHEFQ<15sx#eEJO>;rqhSxDuKFW?aV}flaM^SKaYa-9_?x-rbTYywicRjH*xXQem zhv`VWpT0c&H7A<}UUX#CDZ6h&z#G@hdp4COOp(feNBVxs?e^VZS651z0W)}t3OeAP zyAoWR9(L2M1>~bfoydz!oe9{{!Vzj@oh0o8xOX7Rt99A^V;XTm)4&=L*Ue&W zGHzX;K~IyULoB(6<{mZemZ%I`i?VFzng&imweD0}$I(9`A%Xm*H0Jes-0OY=uV;nLK;;@`sI{d#)l$n8<32 zxM*mjLncEQA?-W6N9U*%ZXUOR?*QOsOcZ|jVR+42TPcELw&;!}#^Uyskzwg8w3vSl zvX^qM4^c95Bh8NpTig};^S2MCdm7?o!AO!X*8OBzZ*WJT^i}*qt*!en%lcEV@7;wj z?O~52<<%U`7fJ-4ncNL&;rb{^P7qL!B@enHP+{5r>{FYsXy>B#pT(H!PdpD0v8`cc z{;8Ys>4j{S^)oNhpA6z383pP_s?n*XV@WJX(1fS}aJ`$X0()qtdJGV|gaI7GImO42QXbE;3<$jV_^=unWW#Bb;vAU0nX# z#hgJcWhbgQTNf`uq?-TfY;O}?5*d9F#q51Ewh@QSnTbtyab$E!&wfCYp+d)Qa;`;) zd)>(7Ul{uZ>dvFZR1BldsmXblmn$B-39uGwT8PYGWdJkZ%7Wb#4}I4-b#bf#R; z&*nQu=f{c)Uq^hRlX%#TECMA=sNiTq*h!XOO(CP%p*lfpn`%hf*&@O4JRwEY0A-@H z1zgMxcm_X+^z$mdxKULnRE>`5GTV*&(<=* zld6TpH>)Kf{kfyZ)BFo1DLe-0sFA6*41JRCuwCqdcDk1`y^B@>c1gFOm*!Nbf%s?V zH_j`#V=^{jt~Y@&npDEyR~3xB5g}@URGl)$>uBm7sXsMNTxSwMWwd^hLB^GNaz>Fv zM6wiH4Yv}7Z3)dQ0@rRPE;*z-+gEo#m^gr|fDN~;C3{m77XP-dZ-{GijGs9Q??;eD zE05uYYpZDgOxm=z{uO{nsHQUAz&&9TOFqBUL(LO=LMCpc3@*?XEu-_!&#!_9#-yNf z%M0v_!GgNXf@Okh-1!US9ew+X8cyVtZ`9n= zQi2G7BprPDi+~Fxpx1C_e#9m(QkhI$y|Or#gNbZDfamr@k17Hy4Q|j6eu~0MXi^B2 z>+dWNe?BiSi5=^d<2OFpY-OMx3CURMI{)prSI}L9$Ju!ebjDQ*ti2!E^|hP10$V|u zX{ruuPvs#<`Gwj&B4s+51AER!-}gc?j$E5zE}$){GHL@{v7aQZ1mHkkik{ve(8c)=CQbU zFO<9d_+w6}%RhVD4Am{E+^bI?+;u^3xN4-mES@(nm?1b}S%LbY=JHm4`Fs5Dp&Ls|RvvXr4 zQy)icekl-nRy;gv%Nfp&Q||bLedF5y><6d{EJjK#`HY-hFwdx1c;*q3pnFkt3glq& zB6z;=>?hi}CG8`KJYYBO{en%+bz@I5zn#Y0h5~U`{HPLMfOzFhO+(F9<&MH~T6TeG zAhh7@=bm= zEGeT-!n67Ff1akj(UhPX5*y*r3F^ocqc=MGL065)lj<`XHEwR$!!4E8B3J&D4o(Sb z^7kG+njG0>rY}N*mPUHQ1)PVGR^dcG%EgPFBvY&wjC$&RmW+QYBDHX(3YRIF+ADZ8 zcTSB!ftPGVb1++)QQ`SQxoamolxoA=AhPx;8w|45bv30qPEnwde#Gddv z2PmZ@DJ(hzzmk|gA1H)%?xu>RIu9k+Gy+H`)xjjrnz<&6u6&3H_h_PZmrp?F`#^*A zOc)`{fs7Ob738Fp5h8$zVsS0Fu5)3abLvQse4my?#A`7!!k^8cr74k78~lE!zirpt z_rs73ghvxzMS#$p-*tCN-~?g=(m#{n>PwN)&hAk7;Yg_VNvPK2{Q z$iDn_^Y8jB3)}9T=gzO=+rwXhdYj*MSEL*1PL2P}w%B|b7bGNOUQ*Fpc~?Ig4A*l z9y>GknCvlhAliJmdD(Wr-$#k_87_NljI=yLM!g_&YdIm1pCKkqI72|CAib!P7v4yy zu?A(wSnhjm>yl?Ru~$sa^IU5e%8+A<=oB$eJmmx}K)hhBy%A@o*zpY(*`oP*ekaE( z-S+JlKK;rW7M`RnS2xark^#C>yvNlsG+A~R8pxtKlrrN2PE=P)jeFbC`Nw4soBLW|ha7iOLj>;3hQtUS={WIhlkT zb`(5dAe$l${ZKzbJ-*L4r3pLLMEb2`1rB%*@myH)Qej6QlL1$x;lE!8mw=AZsZe-U z0&wtT8)qYUAz30-*ifm#7`yIsWE_G!i$UlfRn_uIKaG&B_x!lsw@cV;+i|U?9q^`{ zWMtNvXbfl#rfyCXHIy(CoaQwz6MHrjG0A$P?G@)C&%dl~%2DMr0y8GP`EOKRwJp=7mnP5H&YaorRE)|KE zYq<}mGYlt6N1DH=2uQo@9{H0mEt`lXgC%CKprEDB8hIs^6xH`lxe`zV%t=OqP6?kL zJKt8k!8c2(Eo_k1q(herhgQEh3t^t|h0{Di){cksPCPu@)l4XuaLY7ZW&!|HWR1D6 zb)~e2aw1E*Ecql6b~tF_k0op(1q{K$O9gT(k39w9eUkB?7eK27!%eefi@1>?KnaYR zj+!GOe0$4#wovG*IyI9;mXrNPxN07XtMWHgu$!)f52ozi#xm$!)}eSDb&dQPUlS4P z1ND=TyT;2Tbdw%qswt|vL81&+_A(eQg+Uzw4xMBg4>f5fh*g5|%!Ade_jC?6Jbk(B z6cgXrkMYkfyj{2BeAcDno6phM?9%;46Ysy1&MG}>WSv6n<(@#Hkqmlq+scvz+!aUii6gi z{|Q2SVkG0;=)WFtP_zCAzB55sG->5w3O4-Li~r)E%0_R&Elp+1n7;qJq8@*;@&4sQ z{l6`5(xf*|b#ZYGsY1M9q(jqzJlBzq{Q(LT$EQ^4b6^>_^W%PT#RDAe+X1{)_49S{VHmb;>x9-J7Aw-Xw3 zmNC$6=ZISiLR~FnA!motwlemC@kLDmYIA316m(`dS+l6mR0IMbr@VWoH6s^}R^W2C z+Y);CybsW^h6NNZ(t+X362ilD!;MI&8GjdO@92dT1o>E?^SY_n9EEboJ@Jv^CY6^r zP0RUf`J&v5TP1A2>J6)o@j%W{~TcGz_4Ob_~w;JoxLL zVPxoiym^_cNt2M@t)h^j`k&&MOzttl=pR&S9mE1@lO)*qdX7ObW9q_RUag=eKU=sw z265Ic@%Y@8_}ZvZ%=0a^i4xj`0se9MjJKm~D~|_%6WDw4+P;~LZT;Ht@lB=%0BL*R znZqzJN;sM7b&im1jH(~zM*4YvODdhjj@7^P7N7TZ+309Lu5fwhOl>oMjDenYtW%7}aW=I2l$i#U__J&eS7C;94j-c>%GGt{j=h^c*f}!Po{5B6Zw9GtB5`{GJOzI- zK`6NEll|c(4nTy^VN?M?Ez(V0Rxq}8V;2{8837u9xP#D=44lzd50vz_8y;b#6Dg!-5N-oN&% z^2PGpTc#;50Cb@QsK>r&d)#&p22SVonVKhPq%Uhru?MzZZivp_!6*MS9x3te#NpE zvgAL5mQZR1 zjrYsXqQ@^Om0T6*xJrike9|+k+y>Yytaa$+aFz#OsdoVrxL$1YBXiC6$H=IrY~JRT z7I)fFe7dy~S7K{OUn0(n6qz2l;EbXx8`abidP9;70iYDUEqbG1hjrpbRIn9D7Lh4Q z!MZf@^K35UEe{wN$!c3i$Bi^7E#u1;vwl zyI@(Y9m{I9Ai=tiQbrhTEf&+Mos0LhJ_Ke0dOSQnf2WajY9vAWzU~wTz#O z#3+)yA+qd_hDojtg(?!b$)X>wkpL=Jof~L%UtqYr^Xt1$5X)BS8!_E*^_ffz1q+O%y;g$Ja>uRDKvep{$pDg+3 zOeHkEWHOs^aCmr)_}YV{aRTKqI+4?tt(je;13o8XRo;e#Y`q5kOi45j!s{iaC0mzI zjq>)Y1M##IcY*OS!;-^5TeXW<9X_Ou={g(=dO$~=GWr+&Xo>p@DO)Lnl{Tg04AI$J zYSqcX^Dt895hs_l(G*3lJO2Ui(<`(PMU`e==yI z+kRLq+p0HXz{{Zrj}@o-B^VH@t$lo=Kl(^fpNu1+PfICvUIATNqy-R}s|LeObH9mV zW~ezvG=x}(Sx{1eYA;jEFd8&!+wnJUH6n4N0plc9NOEDS=CqR6RXW0rZ`Mx;Q_j;9 z%jxp&Y9~OzN$CvLJkC(+9T}f;wGr3OHq!}BEubU@)Pk$t*TNi@aVK4I^TX*Nq8Aq& z!6v5sEAC*4Xb@5Xg^Jk9id{&UOJEN%3(vu9(P&yC8)c}w+T;rml8lA=GLR0L*R({f zRpffmF2>`BQLq$W4SQrSS$f3e2F2O}KDOXiY2)^^ZQ5KfRMt8~AiF+}zJ^6_42;iT zG(Y{wXs`4vma>&sf-J`tRQj@Fwo{A{uNieKh$da*zx4MYes7%2`8rm}T8 z_I29vdQ@XTeCDQSukSvO!-*;nd)QTGr`5z=*pi|HTX#V>ZQLdDejMSGWWrKI*|69J zWRLt3EHXWB93lD>5g=sr+Jg)o{PIK!HEw05#%}=rja5cX0*}12uf{f#K!xW2Z?6x9 zN@M&>@I-f>q|i?hsO8mL$Wym!^0b*Gl8Tnf(>Rr?a%G4pZ#d6r(I{+>ZxswIPgh97rZ>m>)u&CJxeY*g)@c8fAi)B)2nV zL&dFD|0ge)Ts`n5toEvbKMoFBCmuPMP+oW&B=-YJt*nEz1e5bQXf6WRL*>1aElWys-L$elC?Ce-$=pxK z_(hB^Jd6pJJT#5xx!_r;*^*W87En#Dfu60oEWsQzi=f927thy0rMaTA4MP}0tux|OsqVWm{}L8ySir*9b06|E z28R@d{a^|9Cjn9%7f3QFE>Z_EzLP0#>W4=fWL}l-0`Xo*{T69_ot}gN%PxmA?MBqW zIu&Yd424BF5k1pMc3Sd40hw(SQah$4*nuAskUOh%I7F@obm%hy;L z_n9>4mc+6f;jx_WUvq6AflC_XZ4o5|n;V!Kz}W-k$@~vQy`75TY^|`v=_=z&6Y`RJ zj9`F*6QYzj9{+wx#`aq@$Q`bie9i^-=2Lpfpykw9vT9SAN3kJ?2PgSwgvoICrp6h# z(_hj^x4lWJmG4HMd$D1e*nsLmXvloYCso^A0T!$82+i%I5Xu#E`?GZ+J5J{d<1Uz< ztnqaqXU{s_&kQUFE)~OitE4SAjPg4#3zvi_v_FuXZWuFxxJx}Y2rtlFmbP;TwqdGN(3t1}&~#I02Ao|TP+Jbte5&Zdv)7wI zQ26>;Jju69KCktt$_ZZSeU@E^Oiq#_2!wUr?~Zm)kP}>iIVB*XTg<;#Ubrcch5HlC zK8!8+SQ}wG@E_%?kR_|#lKmqe4wDm5s8snGJp40Dz(;0iuDMjVJyX+=VtW7HJnwo8yfY#*kwmSb&cdhu*28s2?5?cf?g>SH;hBs zij2?`+076?7&}QaAdpuGON?P*h@NQD!Em5*4*`HLtFIuPhHL~^koe5m2WYq7iAo)8tMs%K2r91vL@a6^O8Mst z#Td-tL`B8#Pj?V1ogOZuO}fjK%5K$OBQ5{7jFe)&Aazj-bs;bSeo6n zWZ~8qOC}))rsd}KuqL~uFDXGXdbzP^oR^i)G4pcENhExb_(T3j4`tFH-qJKdYUJ?@ zef50F5W1;^<3>DAsk{{cL*&Q!((9>@`b+}&HOUI(m^HeQ`Geoh7ENsA%?n+huGChr zQ8Oh})MQE;m&7JPDT-F0+M6+i19h8_{goe{d5)HWL(o;Pl;kMbrXa3C!fWN9dW3|0 zxVb6pIJ-giJNb{{gm}C%M4YHVO+LkX0QCmX9<_`i0*H5;m7|u8k_i%o3(-fOaoX%G z8U@v?$G?8_Pn!p}8Ng3gbxL@0{d;KtWvdvtsNsoBmlL5#r8}A9pHYRe z9FI)W%4+oAMMUdO!V@5J7tw9?B(Ey$<~v{;22_QPTZzG?FQ%-6(3u4fgM=+_rvrgenZOpU4Ief=-Y>l)0k);xi+KUL@WxuKHHruL{TH_ zp&}t3PVczRbKE_7L*5T6uf^=30Kcv^(jXHPV`dW=u!a}AFUxo`qD3JYWD}JvETvGw zHpzN2CCfyp+p0NJ1Py|N%>a`w>KRG2jK?Z-d4GR@rYMnwXk7&c^uT$|hxT?kUKk{7H^*FuUIW(?4n!fTDe}3W-po!a~4zCQc^2yoo zGJ-mx;HgG`2@&D1lqF z=ve|G(Nn%E!B_`rwj=GnW#n&i_+(p5uy8&&W-qQo<=~W{US5`bluS~zSdnL-y0jKV z@ZO0$8^-NDgjU(8m8gD#lW2vUHl|}+N#S*(^h&3ux?XDy4@KewU*0kq^uLy*a+45* z+Q2f*Mt(^m<=|r+XDt-zRHt8Ngu)7`RW4Zn(x+^ustgrn3$~e+tDO7Ky(AqmQfi4H* zL0k4IV{7X%3!#H2FhYg-Z8gr@Bm7-?J*}TAY?ZH-zhG!JPd+tWLlSyXS^E+kZB!cPpgAaK$Tryc%sB?T0_Q>9W75lf zUedGJlt=tgc};1{iu$|hEgkzaeBhS{F>9I&tOljw$ix{_Tnwq8;x~^q<>v(vVpn;v zB*?nvHs;Zx-iz-vA3ug~^=EFncA+4 zruk(P|G41EP`oDS{XOzwbL!2?+dloOnh#TcsPTim~Lf<-i$$55a&H(3Ipa}c_7~TYC6fDT4~UH4DvXtYPrj`T_UOS za3{D)<)JB8wWmKUL;gv=OeXho7oW{BDv;s)?v3)dlpN(79#3J>}Sd zq_>TyabbrMUhP_1T4jXe0M)qCgOkUq+KVu}9+7y$i)GubJa2m}nec$+ox+8vi(R0_ z8`F*dKMh81{y?*W)1u2WD+hHg_j&pQ{YdHpaQ026tHR_5azSk;k^bk>GkMsH2yCHS zlsfDkUd~3mQUn74?mKa{GIow6yVzT3a*7X3eE1FtlTcE_fq(G#Zq9uFrx&7vo)lI7 zf)LQ7c`q8nFR8Gk^lk>js40?xA&v@`ka&PL?x#<}Tk%r*B)kBFek;1LO?09LceXh5 z{k6RQFL{c;oZ_0D#QCuTKL(0dhT3n$AoA7-phL1m0yrf_+Wb4$dx{_&scx(JKsocb zk!CDW(N6iY&!E*P9G?DTJClVcm0Joeh_ z3f?#^;}QrwwgAf&@K|v7=ZOdMG&u%T|n$r9a{)P^{6Q+)5(fa zs)XzZiYWvJC@JQBjiIu|#%8S5fX}u2f&2PjE6&rb3%Olnj`vLPL~>;^q_)SNT-lZ~ z*44cb2+NH0P$90UewGw{FIpxHLqChztFODfdN$y79(D_lB;8RPx%^2z!jM3gx+Nd< zUc7@i!@QPVMA)cfVQUJcga35{17mSTP7-uDn8Y&jmisM-v0=h!}+4M^iN|@J;}%+5{%(WCQsA)5-lD&ClM>v z>328Pl(?t;Ztv??Id~%9A(LV1ZNo_=No7Kq>%8XhJhysyoD-wYbj8u;oMqET6+0%?B!QU}{z0*e(5u~kStd16@*`S3GG+Rt# zKSvOHG4L2(Z9PxDc_2r@5=Z7`+8V*9vSBm_|BR#Ft-NLD8u7B`gj&PA$=*`iMT}SA zF}Upw^lcO68A~0hPgGAhzPwhDl^9$grW*rlHL=MU_fOT8oM)k&>Bs$!Zi5V7EG;aC z+Y97f#*n-IO!CC!abg#5P%i59Hpa|pYQLP1#1VMc(zE#9F z5XlT!OXR!_vr8c@#Guls3Dt~ORD%+uKT`~wqc+59@sy@Aabey)|AVt#$2^K2eCy}; z;6H9okd2oma&r!bN~K)9=M7g!rM0fFan!`uaS^Q%23BrS3t^4ZI|;5p@y`E47Z}op z#l^-gr{&5T@GCTEs7y560g;=W)qYqo(JSlkWH)$N8zYglqa#X7OY?-L(qls~(b5vbM(Z(vfw3^E99uCqxIQgVY|se97Ai!A5kSb;Vj zH`KvY>$(q3)bZm>Ivd{s{ON%A=teJT7zMc{TT^Vg z@aY%#MQUdacB-p4J1JqUf>j_6JrAK!!x8Cay)Btd*XyWW#z@p`4ri{q&3L@7m>*WW zBpOAU_s7J{mGNJ=j_?3bFxk%C)JVfZVR{ovRr!z#*P+gJ9HAsSZ#^KV@0;8wwThV< zOK~}-$}|bz0DFK;)bR1E?!H@NTlpOFZmfI~z(#8{Qd0o=ISILxQ}&uc!7Q`kLs(E8 zk3hRrEkK+6&-Og*WlT+SQ=@bBwCb*oK>#D`oimCQz)s|OnbgZMn7N}d*wwlm>Hp75 z#d1#+vQQcAh%fP2w#Kzp{O{@P(P6)`M}ylQyFB}ND|x}5necrqM4yGp?Yb8MX$|qT z&Cm|Ow8h_#rBT_qJqrJ>B&ImBw>$Tl!B#T%OEEXTo7+>$?|X0VIp1}-(RWMxcM{X` zG{WMQl4&Wb%tNd>Tbi5lNP1yo5F}q%d8M95O{+mB170HI0l+K;BR?0&JUW7C;Kex_ zm>I%L29cct4bc+eDE7zACxhJ5EYmJ4{|1kjp7>)#r~0z^#}n)^!{Dgm+ykCXRg>D* zy|9J*N#QBB`SATO9=mb-czXbzCZ~-Z4OAHim?f*!0p`HFur+!cjVYABZeg}YC`Z?c zKMtCBa`lp{gA2jvZx~{{rE@6N*!gI?H$mZ=OdbBNyw9{W=`pS!S^M_*LC!xl8~?=^ zz8e&ZmU!SYoB<1>p+^fi*N7XQcCd=7pZ=Y;F2q8i$yXGSlVRfG3>JTpNPY8FAh}~- zZiB8!X#&j4VXg^KXW$jkuW~%ZZ&zxyX{`{;eJa8V`GF4ooe$!`03g6N!~d#_+ck@g zV&T}B%j`3E?ssZZG~Z13PWAi#rQh$VObuj-*$y~!BJap|<&|D|@>dewb~U{xh<0_= zhbX>XC%Xj8o()YNV|p>mscxiZK|M7~!#o|bVwziQ_(%ic_UoV2cyr|g#h{(|m&;YD z@ogq8hYFa4pUT6=!4gjTg3p!?erB{B1 zsXmDJESnilPu5Yu}q2IdR#|!DVEAcRT%;9cUm4;kJk9J%bcz{E<%TOk>%mQ$ z!n`+2>a_89;~H-=6$pu7A2{C8$N3r;HzU(0_FM1&yW8;Qe|^Z6HaFZAkq*u>ws2Xr@o0QjIw( z1M48G(;%X!GCS1tDUd>wOX+ql^1(!Hf6jP+Ln8l!;@oPoQLaYRR5S~sT;Bk$f*sYT zG}s!PskOqHEmJ#oOXT`0KFbgMN`v6yWouAx8on03*Ryw2iNCaCQCp?}0O*jk9OD`E zziPOMax|pn{6XGv9Ebt!83jcrQ*Y-I_qw=%x#MtV4ZuoiyO8oKm3$$1f#^6)zcXHYe)oC_o$8?4iDJeCC z!@wK*$bhF~;Kly(>@rZw1D{ro!OF6NG^@4%E_6#N0Y8ufs-B!ux=G`d&oOzD1Qq#L zY}@64v6<}!Y|Sp%6Z(S+QG^4Gh>lmrh-m(E;6iu0Z?MDO7f!o*w(kDQvlYd&B?DdF z!K6Uw8O6~@e#esuyTbBrt28yAY-T+DiRc^BW%M-%xcc2w>WaxxDk(A(^1XcXf@a|u{vqinD zX=D-Ml*AEfbuDjM9>|mLc#dj{0=Wp&Kt>c}gLG#NNXw(~)t*5ms7t?3l_CirSUh2#u8;)ln-sgP=%gn` za8N}3zP_@g9WaA~oM2=KOp*>xcVqXzM~2cYW2-8SYToggl22zbjd>(nW!v~S4`Bbs z__0GOMzXS(rd7tt9M<#8`0<<_23RQjSy_7aW+&@VIoOE%%D?&_PFSkJF(j`h+_$MB zYASS@b{dp8Ki*2ip1zfq*OutdP&Fl>G!pqlH?crDk7jY5$k@RX2s`gSt3!07AGk05 zl~-%3!m&R_4IDUl`aC}fvZ!%U5DNY%deHx2FkBzv|NhxTc}M0>YfrkBZ@roWKw8$} z@0i)Tzh)r)p}*4b*P85tWfZ^X>?H%W@=M^W-ZKjOhZm>Qu$*p$32~8|UQ2xXsLq5MVb2)RfS^P8_T^sxWL1;iUuJ);lq%D1`fX~c-O49fn;FSz>SfX(kL?Z zY8)vk_&l0VZGOv@mnD{s$>YHitRH&P?1L^pL_kn#VG_mF)2rD`ipp(8Avw_5{JVaE zkD`CqyV*R`E{;*+^&LNc{K-5NR5qdjqy7IXsNt_v8qKtOhd?nNbE`|g>Tyb-`0Belw#=9*gv91Q> zc3seCS2=lI&r@zDhJRfSr{m2&9mdjwOFm>mUc|!PD@!p|#AY;%WcI!3G&PG+-!*Ws zlriw0p!{6vY_fY4_7{WWV%uXUlMAQ}7i&P`pkdgvt2TXFHu~4P7>X(Q?S1`grolc4 zmz^Q3Vk^l|IHD`kSLY0&e!D;%%b zK6iqg##qvX8!ak?(A!(U=?s$im^&;78X<1`c6ar5b<#x!&$9sp#wpIKZ&Mj|I5Lc_+q`;}kS!6JJR1^>v2r-3h zzgFho8HN1;x*JH&~?;@9xb^wZ3 zb8xq^6Ov82A;a<#0;T(5$m0%Qb9HM0zL!+Y-qNFvf)}287^xFQ?a~XkYc>VdR>|`P zkaq3_;Q8L0NnmWOzFZCzp>BTQn|78Y1L1l(yk)qMFt%-QJ1ImFv$_--{R;rJa(W_ph?C$pOWN+ewo)*W7&ACG4r=?l$@LW5dD+E`6(* zsUu4^r{=(=c9C0|2RclEne+;`xNI$yY-wqX zvV-RaGWa{4BQ#*hfD`l)JZspT(BZZph8R_hCa4J+5eaTq3=B_#K&dxJ=8o=nYpUOQ z5?B@3+4vg`lo-ZiDX1i1a&BwVCD+Hc&l1Ehe)wdig+?N{54(vr?tpTLoa79w9!Ct{ z>ac0)q@%O!`ehu~20o9$!eFGQBS}IXW|xm}?6wrb#9M|+oKO`SxQn`i1ZFyL{{f#5 zWdeoo!H_<$Bn#@)= zjH;>zf%6LxUpB1>e;kxeUe_Q6wH|#QW|yxXGJs6ihVq6*A=-~i%M^>34=ZHc_nrAp zqIaDk+XbGQ81dDOrcIAJ+2H-&S2L6a8M1kE2IY9qNw4N0mRbhZr!fZdP!{!hsfqJh z@3Gp?nv@qrtLzvBaTp?dj}gbxjvjAe`g>waklKVz<;;~^O72VjZaymcQ-=r7O}KMw z!uuBzvH4Nor2+2q=8{M^w}3f~jt3@tWlLd@jm+WJW`|VK7->6CD0RUbQ0s0)Xd@7h z^?*h#AP7H@75eX|!@#hiTsJ21!QeoY&-YZ|7$4Q%SDORX#39X8w4%!9Z!_}$~nIldvct^2L9jnv!(20p`Zk7*^RAn7I374YFs?Ra} zdhmFH*2KP^FUz+1ED?^ACTX~6{MCdkl?564Gr8Jx$ zCxKUn_8yJ0B;7q7nr#XBneOA$#HtNxbImZ+&hbd-V!AP|jS|wPoRf+~kgEh(@Io$3 z5)e}OJnVPAs&I4t)g#o9c!Vj(omeBJlCAm`i)o0e{vjK-o7qQrFMyD~3=I%pDKGnr0Gzwv@Z zI2Yh}PRu7i+7efAc86<5&C}nLJqcN0;Knml-4O)^JOdk*)412eZ|jNhdd1T&1vC4N074 zYQ0gEqv`T&waH+_x&N}#U#p3zYZzmZ%gZQ0yN@P`{@$P$C z(YLUUx(TA&O%}FL>alY4CKuGQ?5J?gbrpjihx%^DUT}e*RG_|ci($|w*}j-4>G?1- zPN~|q((qxVn&7Z#7KjdiH8PhxrA%(&IJmTVl%(G-=JSYCEhj>a%9rY=GL-~eLOIn* z=K8pJ?@3ncTf|g0Nm93GbR5f(*)dprBC+drkuurHZ*9zA;(SzN^w*?f+(M~+C3h)N z8gW4urqh%R3*xXfmcj1fWZjs24*yRz51)za!0y{5tx4im@ev^0H&juLor7ZJJ0!CA z1AW$)wA@8fg%y-uYo#EOmLJ=}fJV~G2w5Lfs7Z!OSh^vFl%9X{fI4QSA<;pk%s^%; zD$7zVxdFvKEqm#A|DCw^6jCf?XgW`Jg+A_7!lbvfWlqAeBU>X=S#q2(ULSh=%c1iq zz7)VGTet`z%4Ro#C)B-2u~2YjByu~=*xnaw*wNfatQ~U6@~Nv0SbKuXKCC0?i*yL7 zTUG3ia%zSx{F-nt+!qzvD<#G@MZ#qXm}GLx4uE~lUV114gkwM=;s-5(i7T@ts$01o zLfLlZvb>;tOW4p-F~_2riI)_BI15FlnGy>+*l8RCyP(&a9EHY8`kPZaJ@&nks*ohO z2QSS&SCHLXF?}f9l;q47POpxS1|0Q#z?DQL*xtMMOzROJ$SaDU`YE?2TctT?*LdLYNFf*luox?Yf^rJ4JGbKL=mOyAV!r^38V^%mo=#_DQN9^E-6->&5aAG z{Mh77<;P!~qm-IZnC&@R0e{l z0ni{RFnDkXLmh?W+2V*o6zY6X@Iu0~H!GlcSP=w?nko;)=g3rn4bj7vDOI~XbQ_*+ zh)U5dIHxJs@%uY&;RR^4TAeJ-i{3}@!S(QEXPc|ICA3tv-u=8Cwt|P0(iK8*WsUHV z+A251K|y)Z5!>V+Z;h9d`-h1(;-Jd?q@TW}P-XlCYB|1&0O*2G2 zA+^;{@U#2?g*)pHjE|qtA;TB|b&w?qzGa?SgXmlPyaM7Z$XaMug+1V7!oMJbl7l`E z-x7f+_@Uk)HC0ose1$nPBTOn}v0m}aYT;h^62A@TC~q-Em>5z#4(h1UFeMaCxj#6B za50?!9yetpZmY zE^{0F4*=EMtgL*VnJzdM?|tAh#^sZvhK(MAX&knKoO{?+$ww{}odHi4*lVO?2;S-< zaxe^*6Epbks38e%bb;oF$+ki6fd(&wghAI8E>1jEr zPdD2R0yCSl!}`l_|Bx8P`snf)Un;T-fbG9t9}z474*I^QusgiPbbJQ?{3a&EIWYq?pqx9%0@PO>Oikzsu#EN=?Jiy z1eQPZ^g6}N)XT>{ePU;*ie)~(eTWM7!EaaM@pSpOEAju{m5^Egq~&IcU-7t`)&KZ^ zg8o{??2>og?YDedHs<-3-#o0D_OxQM57PTN#{T`TyTy+u?Hk@^K>FgR{rIcdJ;G;I zn8ubJ{S{7~7)}2Vj7E)k?l&smW1G?6uEcLwf`$F~BK)>Ge!CL?{W$#gO#Jpt{Ps-z z=N9?fj`<&J;@_9!w|e67h57AD{B|Y&d)xf&nfUFQ`0bf^+EM!3>iGX~br{-Exm#4_ z&Z%B~f)9UpBywhiM4n+6;#MBHFs@l`Nl1R-M-o@%r{4>$%sk=K5L5?2BTL7IBtOv{ zDX6kR{mb8Yt4`1AlOkq)5!xm8$S`LHUQRa7WBNuhg1TgFKJn3ooN;%9qAu;f{O6th zL(20K-8$u*j$75~<5`hCYAoCRy4!>0cq%lINT&4ae8$MJo!{@Se!h!ThDAqw=Wo5+HGfL2 zK-VVsKg)Y5{k{m$LB46}`MHZr?UwwoH>LblTWH$;{#$=v%Z|8JhUnG2N@qVi`lG@P z9ST-G7wt23UF|0<;h^V#y2yUM!G{hWnU{R3I0EC|9Y5auWWl#zw7=AK@=NK9rHIQ| z-DkxZ-;%Mu#iyDiXr@$ERXHdneEL$X5TJjRSL@VI&RB+^?03 z<$8MG&XjkLOz{Z{!({^#m7TA;b{rFuJAHTU)EPUXY?|$eu$g0RteqO|*D5O6b#LTX zJK~NN6^#4+9h*t(YCG6hzjXH4y<`7qF=0!m);pqn-Rx`==VwWTeyHqjdeQ#Y?pwCI zj95_iWj9zrU7BW`SYFh%d*k`vP5#>_HfU}7MEQ=$sao(s^+|TY_QAdra%|@CR_83l z1POBweSI-v$J@Q{-&ojTPchzu!>_n+#$&N#aLPWyacfB>K{}WK# z@>oNSDxcY=>eiZJ#o5?h2bLTPT)X@B;adZ0Wx*Awbn494Yx|UVjm$d7dmVXXccb9P zZhdav^kO1d&_C-G&33W*oo%z8MhsEw?&-N*F3cHyzd-wF%SQcjJ zoGy7Y`aH_oUPWK#!56XJ(k9JmF|g^5lu0`-{p0Pk*`j(DxtES!vuyCOs~=sxeiic6 z!iZ&EqFVc&{!8WgU3XUrEuXcn&F^}Aws7mMueTn~JGXeR?_b7bCM3P}Vf5V<%Ytf5 zEz8-_`U!){N1qV3C7sf1Lrk1z{l(Xl|Mn?1l5ySS+I{NYJ-35z?0LPkWMXNhxCd?H zPEIVl7G01Pw&cZ6Ml{YXE~qmpqEAFh(Z<@r_HGV){ho20(>{WK4xYPrfNgaL*S$xE zks&^5PP_w*?~U;}ogE$fGX;n5Wi4E}!oF$yeVu}91Roi=v;XXkHPW`#A7MGTcAOOd zq2*dTB==M>KAf(u{RbYi9KHyC#d2=2(dyr)wVgwCdr#Yf(w37fyMZ>IA@xT%XGKt{M8GxZhC$8M(XD;Tk8_~1lf9BUy|OpG&8!aQxKwpk=yTH_e~yt`u8YC&@MzC zK8d2m^>n?1evNILS=A>Mm3rD*HV$@I`*fdbdAr5)%*-G3&;4_T#n=Q}r|Z33#j+o< zaGvkkm_vP+|2bjyi3hU_tL`cK>-B81wtX9~iXHy9#BR4|H61u^Mn?xb+h(;Vwcl5x z;)MeoSM(1)a=|V2rGeunM?HLY^pMGYYpm

D?)3H~n$Fue0-{-L)s}7(URltsd^# z`r9YO^E?dzBbJK9~g%sv~3$hoS zBkT*k{yD7`5e`yjUWaFh+(Ndpa3NFif5Q2}#sM2mEJEs|v8Y9w($yTeq~n4f-9;PN z4gQ?{{>ym4$Q>BY;zC?I6~+J!OH`di7oVGQ*2wdB?X-bF)5_?autJcjd2O$%CG=t)@ z?QR!q-LtEfK6O7U}+Dp z!Uf$=G}trm(%MaK&F06&t}ne_CvfY+)kpWw%HoTXfDqgGvWNZHsIhu2Sd>HBAR6<3 z=>NAdvth+^;6ln032Nkyk1sE)ajN594%?l_hfb1%E{7;~Jk`e?TC%2l(2uj5hvpuf zu%}1RojxNke)39gqlf+vA@6)`=|<^>;f!kQ>grVWGjF(FUt;g(I;Trb`D|uLS4`r~ zm)=m(t1Ddt)1($%WWRJMeUS@H+$Q(G_U}Nn9XiRw*pqEkKs(}V*Sell4or~5NAgu} z5!@R{sxpeTpy$C}^5=rx-V?Wjl(@)IRp-91DuSEa9fO)rE}!QKgI9eUHrF3<-Rm^> zeQai5)U(E4SI*$;jSD(eqt7>qQ6*a=Lna;ZUXXtLKA~0jxjjjoi29`DI<;GS#*r@Tp$8@%4KQa|*mym#i! za}iv}U2ckR<$?{q@{TflF{+N|IhH4~K2{`Tt|ap)a*YTd+2y8Bas2*`p)p&qE_rk> z4GyKaW@OY}x|+lKA_R3IxvrKn*;h^e@Vz0=*B031DUa z+#oBse}zW&ziP(vw_jBeS$?$o?`&PQe27KT1!wu-?BY6~{O0@@$Pd#xVjW^&u+d}o zdO6xl`wPe;arNkjBYp7DGW*+c%a>5w4odQO>@(UeazVZ%h!Td04oLYMNc+6$; zr?di(e8t8~QJ*988|+*N2RsY?_mpV1lBX~5_4S<5IVwfg;lA*1z2xNrQ9B$@nWJ|; z!=YwvT*kq)x3ZxfkE#k|uG-@Cck08&z4~%iNynylzIIRbDg8EY!IZwA+-ro>@ZC3u zCx3jq*NE-COhgX3xZM#ZTx)m~a{0}qzDry8EeK^_8uezgx*tcd0x|xYAQgg>R^x$paPznANMUDx~lQ%Tn+>N?gV{o|{*ywr2c&337f zTkYur7n+4LEnbIA^*Dt+zJn`N{P!sCEer@Uli%z(K*DNCl21TL{I*c2!mvn48P5Of zf-ht^RYlh=(QHuac+9|Tid7Xkzheaq{4MqkDL1$;ZQW~CGFG$hwV8neY2MHk?5JuB z2!Aow{P96q$P?j)3`hTSH!3Nx;F+RdVdpc#f|}c)R$}7?3<}I3@{r+n z?OcE)jQ}1CmMR5}Kd$ zhA1BdbL!fmHr4oQV--$QO$gzwVh~MmHpMwSKhoyQ59`ig>CM@ty&>2NAdh%8NZ!kn zUFWFst3bpoIs-L3&6WRZ;ceo@Qi_93AKCAf$M_G_VhCFXjfDpXOmxM z*#5y#c-*6TAxVq}1M6c>(oc8*6lndvF8y@DqE62#)|&Dp1GAHQ{WN5szu>ibaHH$v z{rkTd)kMpr8lD~7Cj0MA9b8cc#S7T$YoZ^+R5Fad*+Dd-jD`R81h5#p!Y-Qo(aCRS zK%rguI8ai9kcB-SSpAZ4=YkPqhUK|J}C25qLv}I-g5Y>nOiT@zy}_<7bfe%0=^_;4Ua^ z05wyoI6-Njk=XHoq>8mH?T7hm?crxg|7jE#5Zm;}3Yru;C%~qPZvUxVv9J%?#89XU ze3@H#-@qzF_?JT5{EWwr2mG(uTEAm=B52fH99qGpkFYymKO^i8Y{8fg;I2SD4vz!> zfUHm1YB^&Q*TW*TqT9P27ni{=W)ygOY%pUk)6UvlGI86Xm^ufI)VlDfh$MWkY zHyFJMaeEt;3%l*fc8L%NB`gd5+>rkg-fEdEs)i?AJ2Y9^UL3=na^=;p^v|OQ8e85 zDbWKA=uF6lts39rK*~P5L;fI#9F&?I)&Sco0>Y9b6aBnm%a7?L)deiEf9|>?cU5R( zi!2d&HZ4)B-pDz+e(upPu!(7(82b5ee96$^C4Xu*ZO`A<-2Fvr*oBU@p*5be4DPJT z8k|1ZZAp}+rfcgU-K3Jx<`rg5{_UYAmOM!pD|f@hQ8TptWlG)8uO^v&C|eieYSML> ze&`!4MW}z{Qc+*HS7T=ZTFpv% zzs=7Uhi2@*l`$MgG&{7}{Z~U#1q7v{EK1{ss~QLc4a7PvCIGC+3dSxdkpNg4a23Y_ z38M`qE?isUmB(F&rZ5cR!bb6hk6Nlqph5NZ@ClUm!WFYtW$cGG{u>6$h)IN^@Q_iE zPz=)zex&i@RRsw?d!n2+rHFC3iLjPjikMjg|3+U5D&FPCxbaDf{sU1C+Xdn)a8wL* zA@3Q$)cQ{>@n*xBoo&|jsARa1erMD1)aHEI(>`YJ0av*CdulM9F|PT0q}vf9$^;4j z8EQuq-KdtW#RtOm6l&{AkKLE{u_+p81Ft;jJXh1OW~|uV!+eB#Fog9E%1HVs{?098 zL+Shjtv3rh)=TAFQ~@2dD#-f7A!oC;!|rT)^FRfAC|By6`*Y<~ycaf<@cd*j{rv*$ zZuZAokpORK=y2ob&#A#8`}0&hOoh4?p2fAaSOi1AwJUc;Wa4vU?NVckc__EKLeCLQ zPh>8L+eH5d4X#*C!ag!nt{3I@fUu@LnY3oE!O?ywVmu|e;yzG;iq`-V(&n3PSr^;y_IGe?p)rl@S~-0B}Pe{ic7*8#`tu@j^>-Ex-AKr^3?CuD?SGqLuiGBFaTQ6eL>*`^%L9Mef%pDASI8Ow6EH~1M3Sf70#X{eRJP) znkHh?i>p5I&6V-R75nNd@}}ii{J3(wbc2h>GcD{eu3a79Fm3U~f~pGzp#Mc)h> zmhal~sTGz}BV{$=vN`Efm)LloFMEEfup_{{^u==pDnZg5pqaZB6_d&g5r);o+gh@n zOIoYC9d&)zlsYl(&z}qmoBv(!SMPCQvUDwK6q_@~xgl^YlJgvDbwp}aq*}>(chp=? ziFEs*u5v`jAjj$OIB5XXFPbgqQf?y*1pNU0-ah}r=D0$g#Mrmf;`N|HOhmnuzwgZ_ z$d5Xv!VW?9xGBFvMb=XtsD} z5K|0a0C$qP*dumquwgBn6`)*1oJpFS=yD7VZWTAqSR}{f(SNqQ_L<$ zCD&zNO-5dCyk6KittzHPlUfZ z*CRHgKrOO(^bxM7DT8Ld46h~KtKo>fj+S&cVTXnO9fdLXvcV5z3>8d?!{Frvt^0Vi z$EXJzyE33}!^45W$Ui~gQ@%prbXI$}37;ZpibrukXc86(PvoXuu_XsGm6K3BaSL(n zcC6qKF0B*|!_uDhDQY>IC!Z3M{U&p*DDDvRpCY!x`a^UnaCJT=ha^aG zKMV`-b@%Bf$L>=OdzY>GWkXzMVu^o&e<0su%13>1kKh7rlv1pqjU_JL`YX+?PR*Ll= zDAMhQy~(dX{)Wp0l)uw`mV?qSG<|DKsW>q219xvhE?khi)6v-x3qeOjQJwdXA-hFkuole^$Nflb`zS)@x?wDi?E}EP9l*sP>eS z=XjfsBW#XC;CtsxxGy$n@H0yX(jSX zp>5pl=mv}s=7hAoaps8?(FWo@?1#gGRd$3NWT3W*EnW0TpP)msqe)i_f{AM$?BM>a zlk3%)#L+RaLm1{94Yz9uc*F(}C4IirJc&Gg+nYbbh>2$!>xO;Mb8vz0LCCNIkvkWT zo}8w7kmZ4gK)*WnM%8r@6i#P z_XRM_Y_5ng^`r6`!4Nr~zL*dqa`TaRJWPD4`l{|-U_r(;4Vs&z+SC+S-DX099? zq1o;|ZyUieT#RUer&Y%WJgj~cEZW|WukO6G?z|X3Kkfz(LJ~x3;T9iky1~EE^ML-^ z0G$zN7+29g1K+8cjMdx3TM^!o9i$(Yh0!S*xY6Alp#8ulP$yS#_@r(8z9m%6U_9J5gG67QH|KtRsj^AU{xejp9%3 zjfA(0AW{kT2V`MA5F!2o4+Bq6&yQ7AwGcJ;fH}mE5dv$){laU_HPWIw*qI*j=tr2lA`l+sFCl!4r~$wdqmTqN zZCJENL%<}uuJzdv++%A59~A2nOqzjI5)sC++Y-SKDQBNWcP;cK@Zq6?9tYqy-UA#D z`cSXZY=};gv(8~gBMgXrz&rD|mH3*~{-U(GqbeSDm378`YJ=SkPrZT>%`iIALxSpu zT#oSp%RF>#p-F%fDd+gZ4k3~}N7NPr{yVhb4GBz8miFDz*AtzEQzmi7i*UIE)p zJ*717UGh)*%~P%VjxF769NDqA_WAFTU*TPi|re>0O@009gYS(CU7Evt1~Lb z#aW8afyp__Aly#9aTij`kKE4qEez7+nJ}b;(i7nYB^6#n4o0Gq0izP&1H2&1VvxqE zd$#!MmJPreup+?1!58ll9_qEI?Z}kSs$wl1AQBXoxTV1V=+=obiRjzl*>wTRS6CzA z`)7n1TdJYaqk9P^WCFiRnO@~e)BCZ+3W$+TTKDD8RiQvGs=9}2b?uIqd`io4&&J^R z7HVym0Jl&;s$%tvk56Yo*y><{Q#n=Qc*o`@))V6!bdjd6CZAPly|o+^!pa1cyTRAOJgDIUv26 zs^`Y&+nn&WMKjD-%&TD3&b9_UoTiNa|1wM+keW!dpfhQr~XJ=K(BN!D{DQ1{_5$v z<6fBc;Q4Hg09yw`B^nE?&1ItJFabd)g`hKE6>gt`WE$WQtR%83=z}HNJD}^|p!5d{ zrYT2aN1S&>7NZf6Na&r`-`_io(3LtvXbuY`tX9RE)UFGP4?y_L4W%4wapLsUMCsoP zxm=Oe9PjMr;0mkQU#%SwfVtn!RRhL@IEvU8hy%f;Ohs(;=&DOhiR=EN5%-aku`JY& zfB#DqEEOP0zngOlMw{jP=tYQ5lS=b$%(qme!;uEdPxaxVZ4As8#Z>{C(sT4mqA%7v zF$6(`7N}h|OsX{7UmHG^=T1czYF&bSP}q1U9xIgq?JkS{Zq4a?f*la1hGB;yuHf)jPeFYAW`5c~Wjh9);wfnUva`(Av z?)@}dx=;7uqpDo`E5hN~&^1`N=;~le%?MlMI5!u=;KP z=-2Yct@77+v*C8L<-B*#=11Yxo-7s6l|MPs2NONO2PiOU63J^=ua2fD&t7AmXj^ER z!Fn{o1ZZc+W4~V)6PMde*T^c445ALcs9l?~?xA>r`ykOzq9vRz)ir6r92DzK znc>0f69J$20_YRi>H)3**090Og?VW1FrSJMp(AunP0il5#?}iZL)e!zQtwCshvl1*Rf6bP<#ES@)ofq4^Rsqy)R4jQX-NDBBKG0;~`t zzX83_r>NXoEJdxw2z^38&QMgN`MWD4P#LgbAv(jtf<*;p>iiG_B9_%x$TD48Yj4)F z!H!!LD6!}w>3VR>P}V6{dtL;nhx<(4osnC55-UN;Du`UU4T1B)4ZHIrtsP>GPT@9l zjFMi8uvqj&I6n*+Lvv|o!gFC1dCre_XhLVt?TeD zd^g%(rt4a1k`}nSwsl9;?YKJ`7lPP$z``xW#w99na|$`zhQBHrhlB3LEg?(4q0hqeIs=rm+a17}9+Z z@qit{2nAol<}u<7S{YPdpao&UA6h4ZFn8hi!A1ZLLNpj70yZSD0TZ_F0^V`QG`JNt z*ElgP=9}oFW{_7?Ce9aP zZzHED95AwvWtZFz?L=5Aw zgJS_C><#BfqI?i7IpOlm&^RSLu_vdOMsI8H!jY5sD-I1&KM}&i$Wn%GYb}exS^;YaGz0zB$~iRc$q+lS(E7@q^H91wYshk*2{}3F(|d)Jx@2%AWK<7YluV$3)rr{pbO^uXv+u_s3dR> zizy=B#?CidRKD<*g#G=Du16l5twV@IsV!@9m!)x(&4CeKXe-2a@#yhOJw9L3%D}0* zmC=TcMF?CRk1=r(P{Rdh?T)kW^m&diAr9b?U!ut_=*obFqwnSrn?nGdMf`|S3fIhB zff9>QBv2JWk74teJrQIVE7f}*KMNDxQi374Dt--mLzMUU2`qiYkuTC}*8za=t!hQ) zg%XZU8exp~1E;>+zxc-?%_QDM*S4LxAjcs*rI${qTioU6UBeBmLP`Nu!xMU~XzbAw z0CHsa#g8Rsa-hecSwyXa?dpcTyQ%=kTa%W$%z&qd&+B9o`hl+$yp4d`IctDjxGU=~ zywdt;wo{2op-mx_#y8W{)D%M@)+3@pVxC~=K#=MJdu{t*)~`MgMb<<>@xbP@>8MT6Ym+Y4n^K24GnCTV(yv&OW4l(2_*7m!IFQZ z49h$7az&o#c+4(OVX<6bW%`c;9&vzaJ7@}M_0fZ{(6P{e6 z)gje)WnROnJizfGxRrpvn7=-Le8q;*&Z;ZVpomU(g8wq8}qp zwRYcVArB2-9qa>hl3$QT(HeCc~kL}P5_C%ajDhq3Dt6k z_;!c8i=7|HiA^`be$}BWwsMI$FXS@K>2-Yk$P4Or>3v4zy)Gz5RR!I|G0vZ1()WFc zsmDRTHC?kn%Rx3EqX4ahg)KBOU(=As#I`efcX z8~70XCXI6oA14Pn1oquDA0eu6H8%|`4l*ga*l7?B|6TJPi>B)locR=Afxll09iZ36azpDc>CI1 ztlZun6aDHt|+vgcUe}<-CHXLr)F~KnHt|m=P$F?^rwR^|6 zU621I3Le%foc2DijMq`}d_T`E^2R6?Gjo&EqiCcHBv}~A5*=FF8$gdJ=U$A7Qzg`k zP6>Dy{uk&wiOEc)AIK+KP%+LIR1D-*i6G>%nH?=6(9jgo;?@y3d$~WbLH$eUk>Nwzc%eI1R*>uLV>DlU)=@n+? zzu0;@1n2Ha47@m}h}@s1n6Ty{s5ybz*K9ZZayvG4SUkoBd4oX6dYw#E_dmub&(dEn;n3Me_laeX_-y) zCJfy#`uk-YERok5)@@&7>KQ8TUAU~(02sbv#3bB1ajtxG_w>_jKjC%zPGm;&k57l+ zFGxngA5&C5WSPT1^+8)YiyrrHbzEfKb2T+l@zef>wZ9lPfV|wU>9?;XSAA>QgkZ1G zX=#zeGQV@U()v{E&i|YVTxnDMAjRysLw%NUdxDFbvq>V7_bpBV6_-Zpq9#^yk(E4hyF2bip|GwR`!&iUZ!S#w0)L!K20et;B#HsSjz%mXtYEj*4g0L6j= z!FM9{lgS*v7qAZsuja~|AYn@tIYYWK0MlKzkgU1ktSWX_i@l+ZK5DUfdRmFoH~Aw- z?OS*)`ln07rrz+pFwb>;gYL3L7Zs0XSQeM|D!;NVdv--$c|&IVSXtEWsSk65R~OWb zh;B$azb@{R)w#FR5=|Q36u#bEamKmmwU;xh=TSk9FKO#KB{s4|Yx#Y*2cZ(5iWK$z z437B7M|-@j)kdFuOVlo3=X5ys7OKRXiJ6Fr3gwMxa)|UlQ$M1d9CziFBVYqz?6n`L zwt!Djd$K`;4g9_<14zur;(CYeXf8PP#l81-qBSobshBuoI;}uLu0Tb}{$ZJcz5(l6 zLqVk~iu9NEwMz2@u#0s{?!!YiP4CxNY3;U>WIN0YvB_o-WQ13KiBg$m;B{_IoS9-P3|t+QRLcM_Iz2we#N7SO$xz=?p-IpeG$1P;<5dm%DgGN zj>ionKX3 zd+xh_y>e^KSdWWUMIV>#3pAPZTJL_$mddAuI$oXnSEO;UYwgR?<3hHy{%+iUc=zVA zC(pZ;8}1z&o|CdFByYov;D40-&HUePrSg2o?1tgkF9NAp?vGwQj#p$IjBz|aa{2WU zSS1Mgmx;;byn@#oOv2Q@f1JEO8V=E`B35W-S!Nb?kC&L zp2wbUIOJ^`GWV*h{>AUMCTy{GIK9++sqzZ-fUy{fEgakP`R$WqM_*BaV&ioO2>*2e@P1-T+ERA%i&vDh$Ad zC}MGAAwSb&?=InRqzr*C#LVLq`azw(iH{vkIJP0Zu^7(Wlci9>cobB9D)2-2QBy%u z1ON>=nEmaQ-{9rG$c@1q5#tO4KtS7$Eu7^!gADHZ$>-hg=vVimu0K@9mjk}r7JoNp zbH_Jnt{JA5xXx`5zZ8B!@npO%J#xLqya~fpj#&48e>qBDcN7d>V}__7o#g4+vT-Ds zWf2*c3rDNpd7csD+%j=iz9E!tJ@z$(H{A&9(5dgJaxQ4UP;fxdJF7h%F)71X7z?k0 z(>IL|Tht;L6HM(qRuV*~^%2Hj4(TRxX6Hd;F=%!P(*<$~J^%_ufU5tW)kKtMM zzMX?ack_W2{}JJ9+~8B2F2S`f}u~Bco=i&N`l*gbsJ!Jg?JB(Tr`J zu})dXDDa9UCKTygQ!)&0TpBa{e{o~f*|WA8klC(moW4Q9OeAz82&5+h2J z*0JO?;MRrDK)X%eu1T0{Tff{pQE}b6yitFAwbFCRg&Fd!A-%$tjj|W-LE)G&u*WbEAVg2!bwvOL< z;{`FfZ<4V8=S{kDjRELN4_NfHbF`a%ja;Gi76*4+R36b=`$hM~rzIBNJGvCV>f0+B zcN!%i)Vo)sx_42wHOjLG^b(vQF_K)}lxSl3>|(I|o!aEuu(HMt!6nE3(HFG+Q0Jh( z`nCtajUdHygMB1&LpAMCb9qRTte-;hw+0rm%Booj`z^eO9$7Uuz4G|ClkJ{or7pG^ zplWNBwMg}RLZViyk!FW`$Kyg6{ckUNREK`0`y0a2BznS_|8;EZqIiwfb^R>N%)6u? zo4a4ZuKaDtqvhc2nH`gw)KwdV2E?TA$6(D|=Xs!ChI7L5_=I}LfQ+U=v-T?}bXPn# zMzKq}anP6~^wdKh@0a#3?p+s~bSDVH)>wSFeoTMPhLDFFZE(((4U$#2lwa? zo%*rGs#zK8!;)m$kf|3gF*B%p;#0BFW|Y`;{J#$$bTl1*mwO9hzbpP9nW5{EcP}k- zx4-qBcDLkA_?p-YRX-nDx^uxX z|6XAx&o^%L-x%yMxg;~LCEBCH+D3VS*}7u$(CEdn+TG`vhb5#Ad1L4?{eIxuebx2x z7ppxFNB2JdeqotUg>zZ*1qbO&eR&rj*afn7%5yC06x={U9IYMO$Yl_?p+z&@#CqL zFNfUMh}V=TcLelOs8O9f{U`T5e@?brsf0**CvT;)%9<}BHeb#L7Jt846Z`l6p`E|p zu4M85NpRTyJ1#t7u8MYhcJv5XX`krng;diTydN7bi7 z+s8HJ#JeOPJ8|J{gMX%M9`+`(o<}6|Fv}66Mnr!&qGo%%pK6|@{C_H}dj8WRUD2lX zRVdJlElJkj&cge;jk1pZ zxBv-0W7P44{f8T?H}&m~m6n~~y6Sks8XcV*&G(HUrianMJq0eDY2p_~W5eQn?+$6Q zt;K(hRytC?(rQj6#rs;c2CD8)I<`=?W0k(VwdVopfy0eA^^-a_-5amn?xoUG!B!G1 z^HJz^Si0}UT(-5l7WPVScLvJj#dKIVu~5 zC@7q2ES%Ld$?ZgvY^2VAIc;<(bz(08N4IQ@vmU=hZOI`EGnvVJ44k7XpDj*a|8u2h zGss4b{;Iu0SKHs#%;}p51>!5mxpQU~Hw{g}9=jv@7{Y?PatZPMmgbMI@>YKHYgFPn zM`#bho>o&qO8d^?TUEoK;6W1ahl^TVvfWpIYvF8fnUAz+Ru>Di;Ama1WYhlX_HO@v z+A*@y)mEeXgn)e}m#z<2?I?5n(FyIY^&%{1g5Vt061(@bFF!#>8gb^;Z{Tam?xuqo zj23d#2(U6JR(niV_r_@CG6&c#a2{eC>}~7dcFZ@U-&NO* z9q(9$BXwVks;$;Z+C5ZS<15M^#j{BMi{^NRgf81&X#t2tIsB8fGfN`dckxs%N zBQ@R1bz)y1vKcD27(}~O&)C;CTGmCqYwdhxW%UhEl_pIvaT68X- z-!F4?bXHZdwP+3R(`{bjsTDD0SEO&f;&fqq(-3@H9w%#ZgB#2Dmu%BzlY`Te2C8ak zXo4HKXe`resYlh38FdDVi(g5fr4IAB;Pt2rS5ma*6}J7$DsJ1fV^j6U zcR2MuwNlBUzM+2QxD_8Y-mJ&nbydfW)|k)G8X0*&zx{3Bo>yJ5>+0CzWp=2nF4#?* zU%aYsb?Z`>^v?VEpHaW#dS?i+n)*<7Aim>Re9^nnn_o?v`tuyy*@q?Tk7v*S!`coT z3rTMf;QZimnl(x0ZDVY((Cp;AM=vj-hgLMvW{^l9*u}B=wcDGjxA{@-T7g59Z7qy| z?1nuIPHC%Jx4q5BLHZ?kcR$UhpXYtI@Tt1%5vQv;z;Bv{?d!0pUxg<=)Rz7?*%s;m5hzpVzrZPF~U>YC`sQ`-ur;E z(#F2!h?!5+4|xni&Cng*AlMgc^KY4~dRgAbK_LaE$$yMgL=EG1MEd;Pn51Lr-Hj@$ zr$7zYuJyuwy5W}>h8a-?Sv4Bzchrq<8i(#^AJUo_XI1p4=I0mcdWRJh1_yeYz0Wt6 zHVxBCD@d3DLRC!6Obi&Rs^phtQe6@%JkX*{0XAwA;@i~4XYhrXS+PH+4=nZw8l(8j zyPMVfbZ>4eIp$i@UmZP|^0t&Cp3)B`p-CGC+kA<#xiDCkcj?zFH1$d7Ya1G15o_BW zwnAx&*k3G7b}}oGTfL*??w6Yl&1S9TUOcCLP42v|YrJDs@v-RJamL!Ai2+s8p|X~T zvgAcGK1aqjI+zJ`AZO@&Z&?tZtUSce?<~c%#Y+21vOKWrHAznk!>Qtof{-5TT5N$x z-BWmM7HnZ=SSP_)lD9Q$2{}UsoUR+AIwh)MXTW)KaR_k<3Z_H}JSfBT{^+)Sq30+}^ z4HJ}z_?D8CWedx8LKGzZIM#pt`H}F4QAho*Y}GO4(M8 zEn}NypC6v?Ddg;d(!VZUi`sBp()HYCHSfWhFkqtKYDGtl7Imuv#!bUq^(82HI`TJG zEp9p4&qSf}_y2s6b?>zGx@_G$H!S*S^!ngMX?tgZb7Wvv;>tfpzvbqlg=cfVY;{bFHp^9AEPN%unv@{sg_s_5mrPXiKO7G3=4 zP#?1%uB#wBNgw=8R<&D8T#q%o8||iiQ8mokGb6=B`-TxebMdFmF>P|Ik`Y?X>dvOS z`sF%Z;5z6NN!XQrNGg$Q9Ja6xUbMO*W>@8am?_&nG*q#CnoPVUdkL_j^iJywIF1Tqucg1+Zr(7Fj3kYilB&J0FPJsm)Xc)#-g$2uzth}PKG!*K%Iw^oNS~)93dtrW z@Yqt_FX_rVW{X$fLe)DBOVb@QO-vu<=?sX~Ui)_&Q(~gu}_@n+uBsvF@>}xa+G44N&$vc*EEcYuDIrIbYW^n><@U} z=+2dG9hGf|*=0I~6~B&HhLH@;L#|VpJyud5P*CUzV=4R&1x4J|geCS-o2T>(F@Igw zO>K2s_u$pJuJ$bh40n~UC+>l)$Us3nzUF|9NJl{ z)pDIfFP?sfPeq?Sylf1)lq7N{h6{K z3IJ05HQEfv1K^Fdk8)O`ZSXbA#U<3WoVGrZ zlsEFY$C56#CSiBLjY-sO+7{Y+-wNbaq>9=PHxDi(k+L}F4LOqXiTC`R>VC%ZG&=*o z@r(5u9A1$ee`@K$8@uo@7OyFXs@gR^UH`)8wONyL*8(!GsstEq>QO%N`ue9a$0|HN zWQQd?j<5JqSkq6MeP8MZSbb9eIm1SOe?qai_w1>u*CL1Jlpgii_hKZ-sq59jW29tN zP%Cv0R-BC(e_W4yaBozC%zAZWAS(hP3X0| z-TBhmr{`6Z=RdA|r|i2i-Dmu~36}TfI{5y1YvhGki!Ns#13)@kiA?$vE_S(Y7rRie zUp7r3$5H#N5NERe$#STaZ4SPLt{V;tX3GwyA z;xi+%q}^TGpc?kYE2^Ng&s+8}*1GUT8PQv5M0wTkV}8|GSFKECjekmF7p zD)UTSTCckFeT*T;7X4RqB}R91@a~;m;3K4n+M$6u#z9wmL2q&Hc2SqZ^MfKSf9g((DlITcqH?au2 z2dJO6=n%mD#^%C9STq>F18X)Z93uf3a4|`U{PW5ubnx$x^d6gDAIWO5>>5~K4+p!y zDtHZk5LoHq=56nxn;%ZKZaJj2fR4FCEsC0K(g==d8buz6*i>uIhxE?eK^<<{2kEiK?cLkg0>BiMf&cj@=Q+% zo<2^w@zykTU>9xR=2q2>)7oo>$Sh5`paE*z)VE1{im($g2u3M^A!5GwCP5lj4k3(#j z?iLwjZCCwb1l1mCrHzqxA2p^W|5iPI#l}V8$zcq2Yzo-Y+-(aP3FOsa_)*f@t-Ah& z%kMr<`zON<^3^G9N3OCd@U5S_Jn=jUd-(ZSbJ-)3{TgN>q!L1R0NR}!$*3^(0s9w( zVB}{w%F)mQ1T&Ii(z){aZ<{_yKe$WJG{dMb{|?fw7Y*E#IMNsdT%9>nS80wFG;cf| zFdhNa181IG^GC2&-!009-`P_UUzDJ*I+$8Fr^Cv#_AAfxnuq&!5{fDkZs`X1`n9gE z^k6k;WFS2U*z7?Psv&&FhH)fg`s?NSg9s@?SAS}XB5WA_>URs|#tS@)>f^zYMVGn`5dRj zP+X>B$vKDfW;<0!x-~R;u1x6-1LDwNeTa^noFk_cUOxSk@~-VWq|fNELDHj9UlN7+ z1A}kGtE9(|GN$iA{WNFD6sp$a4uh%&!ULrxkY?ZzBo#eZq2Q5HKV%@j9E?lZ9wGiykwI4>vfh=HFyPoEPghqnpn5aHV47^XB~D>FU7gJ17K^miF-7d&bKK#guv}oSE6G z5@39^`zR$5l??wM$w4G>gTLAX^vs=r5`B7-YE5z;_rqfHwLRc1;7Uw)pyt!9d zqMEhQ;^R#v=oQQ!wRoGiq?vDyTC>68@dmquMX+C}5dtgNUFPl22ncQ6*xnA!1E~ML z+%XNv7bm~F4X8W^dYFG)Pp)>Zf4nb)&Tq>d7@ENEEQ2P>BX7*xXYWfkGh;Pno3|N5 zIXVPy_pNJdQ$Rn~rX$075GPf3L?uEO!|=n|9h^FnBYw;6ih@P6^<59Dy_TBQp7g(* zV>Cd8lF8q#WhI5H3$-$zFqm=TkU>b z_`TApwo6?ZkOTo21P{gtX2USLERlyRM5)$ecqfHj(uAO;6}ScnYc;gdv$ zn74#p9I;Q*V$D)CUB_;^Q!-_UGDIPQ8vIeq{7iUZ^A3{rtzhj|%UK*J^hdUw7h*ZS z$WS$-_(%Oto1`(1ipRj8*F5A=wF3UD;N4?Wwdqk|D10Bt1^%E9N{?X^7 zmyEjuy$8tLa(YM#^ASYoIf^O?R72qM5-5D8nP>o}11V4+Kw4;+aUHfi+;S~g&=mZT zu8pWC_k`ID+Lg4p{4_ZkJs<#xGhyCQXI_CQpEFcN80YcE33DBmd#)Octu#8LV=u+t zWL*CmRlD_O{Z{7KymA`b!IQJG4+Zx*m6NUvU`>#yCEr%?*UrKd=}BfX>`it;wQkjT zJw@JVFe4O)NTI6sm!Owi$1es&wzd}6iDha$8I$Qs8KfXlIbDr|o^ZAz5e!4)#!TuS9B zaOgaNl!SwL1BE*8vI8FWou;1yjMM=2W_^OaL~Dji$7oqwgDkP&%5(`t#N=wnez^YV zUo<2mtCZ?&Qfd+EF6Cece6g_p1?!X>z009d`IF+fuXbRhb&7ft5rG;bUFt6wTRPn~ zp=q`rqom(!KP2aJkjg9PD2hW!Z<2+h`Hh0UE*SIRFww)b2$vI^pBASfxarW_-s<<5 zr9ffZ+d_FR5L}$q^}?nSSOp%h0#Ka)9Q$@;cM>C&aLJacS^n$@NwMeYQ1xJoY!qG&y@zOkm)`U_fSoA+Au_WIuM3VuMA&(-8 zItjXmyZc@Dp_JX9+@recWu_9{c?O*^;vdzm%ruw-IGQEBX4 z)@zDv zIBJ@Kdgk!jQgfNcvS8}KbDb}|JNn5u|JL4q)(Okni(Spi3^lEC-OXytd*D?@Ld>ss zx7sv*x#r%s*)jFW4shHc&(ff+$sE>_doNb%QHAC#;bTc@zi4`Zy{;YgLx4 zOrrD_gfR&sQ5|hkkv!-Wgokv%iM~$4&69aI4;AE%^)S6=;YCkr*d;(1du0Ir0rp}T zSV2VX<*MO90W^4)Yy_AayaMYze3}=UeIBn=(Uh74i!_blnn?MmxNvy4!-^U-Z3#Vc z>CH@fC59UGj$tK5iy3@;ibTM-^w(#1LqS3S&5_|ZO6|DrL16=n;Jz7kC`$$;GrTE0 zSdAQU4EmL$}@v-abAOlGZ`SRS<^G z}TSk9;7Yz+q2{44ZOTFA7Pes#rbp1Q3p5XWqpFlOG4b%^(vWNB2J&2VX z70M%ibSlnS9hO;;pvfi$Y<#3Wh8LP*==jj!)gECT&8tvD!TfC<$POXZ>EN{qb3s-(B-hhq0G?9lPFx$NySweN>rr?&A=7d-whSl)rEd2~ z3hT&FYiwKBHhfTUANCq7-nYk;3Pl*5`4G!_Ls6B{#z%+{8M^ohNq5K9fiOoMDBb_` zg%FTFJ7O>fax$wh4ALaH?<`D(nMxPe?dqE~J@j4;&9buJ4jEKcQ(E}83?E}uOK}=e z%iEi8cAV#vhuH~_KbjSWdlsPnDam>^Uq*QRb!_^DkqUCp+72N(8BK)l6@XI0yRQBN^L&w z7Rw`ap^C!95aMONgY-KViH@)qSlDwEspe*iq?!ah@tpsrka%=bNP3VH6oeLVme8Yk ze9GPDL|QnDby3ChObN9~Cwc{MHnRg1{SZ5(QXV3sFF)Y55oi0QRoejGDS4#dyx14V zBW~2!T|7rEe|a8L=!|)AUArv074UPIIbE{(OH+U!4+Qs?eW6VM}#?K zluc#Stqk_W!a9kvXjI7PeXa;2zrv10bMA?7Kv5hKVJUIc>!mCynOOTWeEOM|(B>UB zm0xUvC(G_kuAs{ivvkA0jt0mXHB*}2_W7tad_QzApZ#G6(&~oSx8Q?Q;dzP?5Sq|R zudAU~iy0(|rB6fU#j%ZxVig014IZ}^@WaibNj8;}kft;fuVsg03?R-zuB>HpOcm5h ziIext7G8$l??zkg?QPz0$2?_STgE!KPyKS`LyzTB5)t3l0^#$%%E>~9(nj~p_XF3d z=!Zh%Eq60-aYKOW7?@iq?RxW(bm!32=eLTVRDJtkRaRQulQhsM>t26!tga~yHwy}D zl!u`dZZ`BN-D#M%-9s_Oo){$^Tbfxwv*~F#Z-X&Rd zCU}8}l5f2W9BIDQwl#1)n>^3!lCGJ~O? z>cZ3bT?L;l3r|z3h$tnT#D!loKt{3i0MH36O52YjUdh?G2q#kH1yK=}V^HHhoi;tM zufP!|CQP#O8xv{GA+7YFMaK31{aNe?os*qrojO`hX!ywGZV*m$bhl+uEi5`~`oupL zHUNLkGe1x6Y_Kp_%B*hfFWev~mZE>!SM8~JgBu?^ip0ie7l;42+vR2*rCBT z_R|yA{#tF%wd7QokuhH$iSFDOcr73=ToL^vdL5A1|f~Ekdhlsa=ox_v!CN|@Zc&e@% zO}uneHZos6qKfhrdnYQzH6_3&@`(u)7a?)g-yqXXmL^Vq_k&H-58_HubLdNj`eA-8EXpx+5RyguHHl+MUvk2()9eYJTDxUOUylu^QELA+t;)?=R!i&Fu~ z&TvugPSG9+&vTkIp}-k^#WwiA5ax*jVg4cOsma=0IrlgS|0?c(f&2>g$^u8C^a?Km zEZpEKMa+o7QJ9X4MjbV|M0i5U@uH3kjekrp3eDEr{)!;Qw~LVlh8iLqb2P#ffQW{E zg9XZ!0YpPl>n)fAwkM|?!ka~Slszh4M4u2nAKcMedi1i^XWV2Z&0~D=^g(?v#)KCl zk(CLMv&2B}0Bzu_f&%CcP${A0WrMq3T}ls1x`gmtu`R-sL&E(%wCgpL=MAoSNe|X; zk(R+DuU!;vm4*B?6CSB3rB;<7^}}?PYGI3Do;KvFj%Lu zmUA7cu6uCUjraq?b_$&gSsCb@YZ)*xHW9EC-$&HM63s8q=E;u-?&;1c9-RmP4jX3R zb^MAIikKgXrvjKSuUth*rYQJbKn~C#go^zu%N)94)*^j*N%N^N5BW4HSJ#Ke!})u% zGxD2{Z>A(*YgCuV>N=Y9Rp(z0D1i|LV+8)*5=nq>;j+-$0R-#?5V#OQ4R@~Z0#W7D z*S*vDpVqZ{NiIPHY|v{_0og{iW#Co1D#~uSm2*mk9-Uq^tb~&&|Q;`9%Q5 z|6}Ksm4oln-bCOsMfxLr*=bG5=2YArLP(u1FLV!J_eoO=`Y|ht+sO~bOvG86l;*Fz ze^@JYPeFJ}3L8BZ`0&kTLC+CV>Epq!Pq-^m2sPCcy1 zh3pgqqdBTVsL0PF>eI^x(|v@80wiaFdtU_#2j5?m_KsqU3o#w<1Z!+op1hMF83Q&0 zQqpgMSiU4|njw9PTv@{hiRml{RBWW+w#4}mD^=!jj$FlB8o&-ygbh<|)u&(OT{eMS zb0faY9?8Y#2ra+prU(>1BX^Ib=+9^g!grPvHG(5-{s`>~r!E#Q66QOaS`Y&kP%mwV z2*Uy_2T*7|25lmBv_we&x4R*IiiI=4r!Rm~@SQ^$TN&=W`M=&V6xi4D^ugPe5?cmQ z|9zh?F-JvA5`Rn39x+59sdNrQRshyO?XUaWy_03=43k?wt^(pL(Uw?9<|qQ&S+O^2 zMOl=M{7>n#L{08Fx|M-%feATH0)RRTJ4GNn(2s91Cn}c$ptY85dtl+!n|m^oE2d_%Rp7y<8-p zfW?ix1n+mW3!nv8mh?mR$-^1(KRJP`hsM^ZuY?oUSe}cLt?wMGiY8)_XwJ^zhL}$! zUJLZJ%v>Yu*C}I`@De#87I+pg&54#9N8XzP#8R9?mcVF3I;1)is)+DPM4n26p z^IdS4X`TI!cN4?lN7WwOY|6O`9JKfO5es4a_$_2nCX}t1`jG_ij`$HeO0oTjkPQwj zEhT78DDY#Dxe$17Iz}s#IJt_yeP_z|ZPd2Fl}J2rd7yZMUPYQ>8SW19ai0 zg~I0PhMD%DK;YD3a01vKj5b^@x`eFR-Vh08Ug54RZM`XwAg@}YPlmg8sfsy(RW&@1Iai^K|Nkmf~mipK1hMetNhpc7c_>C9pZIzaJnsWuIavy zrUpwiSO)KhVHv3zn1_qSDS+2DIobuIEnHTG%I-O{G-v6=418SkG@&muh$=9Pp~sldm}4h}+|Jw_CO{I^E%?&BTwgwuL6aupnd#VtnBqHz34_dFYUd_kuoh5?!fC=EuQG^jkAB z)GK=wT@)BY&at4k9wS0IZ!}ICc^u3&Z1_0qU>lAbD?l?Q6Iy_An-Hge=sPgUz!VsG zA@3maqj2d&0Y=d=zZG3Lh)y^rsLD?UiBdMlrEL(KE5??u#@nmm`tuldPfG|YU0LnoRIrpcr0CEA@MH`GRa+-Q<9)Rw4Vc!y#G z{^%!@=u$^yJ7OUCa-6s@kqM*~Mu)(~M66;VFDPaeU>@OPlNkcl7TFH;L%iqH^Y8Hu zGD1e{lLh^i`YW6O309>yGM`fX`}OI!x5LIkK$m!gba@1m?j74y#4~8HPX@tNhO`D+ z#eptk0S|9FjUg)`mILWO6_k$#K}0k9Q3+%b1qe_nrep#Z+_AyGhhqgegG*1$$=xXi z>98PS0^+a&p8Ist#Y16}%xKv;&pqH2x;l3!b2-z}=ewJ1UN@0lZA&IFCEIY^CKPAROrjWCjrC+w~PPAN`&Y! zUU5|7`yol^0V90~pBztL?94<;!}Jc{$YLB|4i+bG;+r z*nsR9X*v4H(%x7Q9*nmK2S&mqED7?_C;EY|^y;)bJ_kTvemKOC9)Xh!w&^jEG3ili z!{!?!4u=vE!EtScE~hu{F1uXIdHxPo#J-5CLA80Yi*-3oq%^a zp8;J~OoKUd|LSDPc@svd0I?hHKnjVd+C^Cf-guh(kvcX+a?_MMk|+Q?Z}Ne>oT>(^ z0E}a4i#iNZj}ZYE3HTMqK$2I!578e$E^oUdY-4CmfV{`*0*Wf5#s7z{FAwB8-`D?I zMp}g$N(qUYrYMmnVj4}fXgjAGQz$J{2&q8|t!PovCZTCFa~s=8QL;p2B2$_WB}+^x zLiX+VJa5gp_ndowf6P7S&M9Bt&*#0oUeD`U3N60!TK8nrvjQ^9Mik2-1PvpZljN2W z(|{GY);=s-+!$QCm~X&!7?+W7+7-5sXmRw1;NS)lA$9|R12o@g*om*90sw6-4~GsK zL^?c+^eh5zNQA*SPgtdLXYlx;i)PJvgOsNZcEKI{gA#VWRV_H~EODxu0XtVzZdOpk z0L8smRwis&Piba9s^dM+YYhRu$ zUMW5w%^q(l4kf`5W410iQOK7DSZZ#8(w-K*_!vYmE>;46x3yJ#mA%?%KtTdLwOoug z$KslBylSlbnnzJD zv!e@=NC5_=Zp?h#F=OWQN8H42$7;bxKedTpY_W8~*Z6oG4xB}>&?9q(zL&Vd@LRHc z#5|XQKv;B=>F^LNhE)Pw`LUvLiw1y|A%O`ud?r}(>RAdeUntHW@&aiqjlN8_t+MECN!x&+AMN;R3-y8A($eu5{c&=|&hC zX&b=4yN2%f50M1Z?f4JeVeHy?vEx;M=<@Mu z)z6WC89@`BGgvF#nkgo17{oGVjY(fJR;oCMIE};313y7GO9XLnUN49O=+h94l_JH0ArFyZJ=!>xa7dsOd;T1>OaA+SS9=#sLbv@t z4rTmePNb8t_(5?LD32%V$+-+GXAFc&aZ2F`!O@7V5}>UueC<=Sj$Ao~tu&(22lYG{ z0362(b{W>7r4|lvlySrYIO_-dW+X7rv~*UPtVMYti31;J*qG~ z<6IrV5Tcr77$)v#G{Nw6^gYW0K-6bYwqzR7q8>j{M2_J4L|a@Va(U^!$9b86eqW(v zRi2&mwV&7CypfukSu!Qes9U}ON#Q7BSThNKPDaUL<7#7X!qW2{r zoJ-m4da-v4jCf&#Uu-$|-|aLy)BLGz+xh2UA++RNAD7GjpnIYPZ@bg*?z>h!+sf2+ zDjuefM>_5Iw8aEES6NpOIK?QIrHjpE*6yN*0%3$v2NJwrR?;*;!-P&irR;;8Wa!?8 zULM>PQo*3li_iBx!1jmTpnxEH8;>*T6XE56PkNpR3%IEf0~*)Jrt2Rib;;zt>!VzQ zC;<(sgPYjQm`nk+n|=JoOpr-L(k)9;ZukDKXrbe=JPRdq!buDillsuJUZV&wQ*2XO z*1-NAl`K0eE~FGXKQ`0jpJg%7Gdz*Rm8?kcjmQ)teNn)PXpeo^NdYS8Xb}yX-kd&6v)0y`ym(F@}fEuH~^Zy zJY2Chj)aO)xS`gi@9ZwWdjyN6ZYiK{L%i$k30IM>OcN27wQ zzDqdbdK0xzZdv(og7?{0cERzhB`9;)Wql$GkQBvyKe8UMuf*i|zhXQ83*P|^HcDUA zdYhi$+`$RQ=Hl;y2^B={A_CzY!E*G`%+auz7ZZFIq+5((W)S=&;R=Z%9Z+uc>eS2} zmziP@iC5QvE>OY&JP1!xs8dky6TV`*`-zB1E-C#^36Cu*lud_P1eBo8bthzxZ!g9`JDpe145zb6!z>55Qe-tr0 zXIEdqH{Yz9ElKxF_q(|G*qw&kN1*f`$Zvkjr}S3Er*h4b3LI=9)YLRo3dW5vJ%scp zkqbtYQc$t5?(pYQcsHYyLEHq6%}v3f#WiW?ol$(TyE(;lDvnr0dwUbXiPhkOEoz1Z zkdjpJa59DnPh&R2#)R+{$ihdOM}s4Sp=qzCZ&R1=1Leq%7V4dq$6;1Jt$Z~sJjz^1 zw8G_8b-TmrWqbb10|eOZ<5%PGX3dI26t&tRXArS5-RHz{d-9c2(r7B#WfIm9BN=Qs zG%grQpnYPMyjVGOU^?nzY*Ks_(Jb8I=p`WRhV?47ND!0JnRrm_he2X&$4pLcIEcQ8f`}Cm5Yy%AB4s=6|B$2kZMPUMO?l-P*J;T24WtbRzu5L^1B2{+Oy(7R$88h`UBYbAwdMQRE8AO08@ix(X@!^o^z$yuA%4VvSuEeNdRe8$>9e`7y zEf?B+gZ+bIpavqzGS9+e=wuOy5KfeeMAY!F4mt}?i=ZelZ3~F95D#U!haAwho!BUC zb+FV4>S@Cv9QnkyXXd61q6B}7%;Mz>?4$dydf#L!7lqR1gj>Is@X zQ87h>%_u!WDrn3?#ZcW>+9J}H?*sRPK%a2G13chpg>#HM z2TuL??KvFB+ZDhgf6n%vNnxafgU(-Vp~kwwO_E3Qdm4WdG~ zRmDpr-;{+hYK4AY=9|dgNze!}6APb%3OuzT5L%RxgeO?fHts8i+7Qfo#Q!W&lHBw{ zJl2CnxHH9eL^&psr|=>PwGq;S@pTR-&(p1ygY|0;PFq!6L)3^{v;!Zuh2%t}-V^#| zg-?3(VSBF|S463qw-LNkH;=RV44RDdZ=%09SLml%6aOwJNtS}(fwxda{w~j~MVp|I z6_g;(ewRBU(!fg|2bipe0ZDkG0_O0HPXj2jZvi(xpa2f@Iq3_{g+Uc!T~YJ)N0#B6 z5h%p3&M75}wok;c*xt9XO@AjTViW?hs}$NE-hJfy&^GKd3gfVR zMY6&pyQFku>;Et`+Pt(sYH+$vcyZv~8(T_#h(2jOJ>-U!npAPIPWY9O9fD6#7nF?vYV;Oer1K)taU2^u8F363bIHPFAWQ^S0al{R{t zQWo7_+fa&=`oy2Y4h)!NBej7cQPz)ljx05?+xL7YCIQY%d`Azc8{gX6mehN@4w_NS zoxHwyQ;qd6#ElN_`&GOVn)M9xB;i5X6x%%=y^6v0*TKS0J!p1=a9l20<2`gs;cu{W zwSJNfbf!J9%A_0AfST{O)im@~j0;dQsk*uO#FXTopi;1FI#vuD*6Q{2^0>t!E|vf7 zz8GA!^?hj5XgpfFEz_C|4bZuI{qEk~gQLPuY2FFzCsq%&!*MBLCttX+C@v$#TyDsf zkTZeKaQ$qC-~vz7gwr?3X)ASo-$r?&#@+%r9=(ySZx3Aj-=qH>I`gIZ`>(WE4^P4pkvE0UUH?gYsHsP}$*MdfE4w~B!m9{|G$kdppJ1**Jp`=;@{ zzznTJ%|<8thLgq*CD?V5B=%++BU;<-+LmfPpoO8=PO6RDF<$M+(23?vDY7|1<8Yuw z_|z44Ae?;9re-7x1RG5W*%560L~e-rS6fOREV9u-`9${IGz4wvk}ZD=uZz3{nVOrM z%d(j!kI7Ze!_=~eN)ygKw|JOs9CX7;)#013taPWStw5f`lfW;Z=onb(+8^*-=air> z7twx2x2Y6|F{@0Afg**BSC-ngl)X8r#SXLJz8LoX_nZibZyDQH1H=a&Zcpl`_8}OL zsG_zTPJ+zK-{4CAR=wlr`Hdu>0XFFyyKEnu7BY-ge0xb}d zJ{_HDyd55;&A-Ps{ciFeKr*A@i^CLBIva~*Wni(aG(`=i*Q+Ic?+R3-gdi9RA-uh| zyLvt#BZU^Hg)OEtU+x&M*RfnLmp`D1#6Gqk`H4jYQ3XJ7D#}EV)SmC0xbMudGgc3? zjNV*%`uj29LkmILfs0b-=I*z#J)_aVTP7v&=bs!%Fv!w!t(V z=c7`_ZcW;HeS`E01ZyM5w_}4U*WKm+8&r6^AR4 zlhDZClF02q(DfH9v*r>fqHz!w8{7XGiy`Gj`ZpV}9R$(y>s}*FB+fns3wA1yka5z<tKxv8Sc15%eK{oIN#cxWz6zc#uM2Me;v8o8jBfXZUb@;(D_G&MDc-uRg4H>hrqcce}t$@1@i~EE`D?7ierc62q6!-Zm(H~;(zouCN8dM?D`Da0)fiAj>6er8`$D%l!@N$)qf8@B zMhFf`mZoTfm?+A%!Wl%5I2=x?uS=auYNB`()Ei700Oqkz zcur5gnQL80#G$>JnKCzxZh5v8dH?febxU~wTUqo~$!9%BrEKd4!E*HRzEa;;qn3Dm zB|U7I0_;jE4a@&B^t@c;wI?RI)uukVYa8F*tu?;NQOCnK;Da`$Ni=}wDMuy0?hN}m z3}XhbuNO?hcVJw^Ok4M{-zVUg`pi0^M6cL0?}$NaZn`+FPU+4$*j1n6uauEdp_w?s zw#^XAfL;4%Rx2ua8FvjBkn++oby(g02_7czM|HowZ;HzSHg;7xL5<6g?{RxH#>{Hc z!8MC7%=t0+tat9&b$$0H2rhD3XJOi_fxd0V*M30-HS(|$U#ZvvhNViYgGR#dq2`X`gWPOIlX1Bp`!o?dm11>?sxEmiLd$Z@TR za5t%U>D_yF_s1jdFAzuA^TyHhkD)W&YKkY_tPOK&9h@$?Fw5usfVI8`Md7y(IjDWV z#=B`qEQIz8V|0Bm;|TY~^;k7(N%BZ16d59K#w)R?dZ=A(OLFUZ$K+a=$kb#Zfv{J{ zdRwG@Lb>GPyqw(Vshr+|GXTKRBvCakzTsJtT zlr2pCGPZ43?8C)Hwh9j0u|ps1torz9;LS~Hu+40quGhN!f_HdAcSb|6x=Q`*f~$8I z-f+9}acn_!!&mCN3-=qp8262mFGMN#^@EI&yb7)6YH0p*^9;qWzoPIX1z z<;`D>_9WEZ(3?#!&Wv5ERKwj@Z&bByXt9et>l1&LlLT+0ACnsMcK+IS+&FR1zGGiS zIR!+k&4l}sr*h54Y@SrO8XHT2bY;6`om=Jk%h@Uf@Er4u&q-0$xYlt#EGBtxc*p0u zw}KK3OYJY)*@u<<;^?`*XqV~pt(9OeE$P@_-eC&P&!}gPE9(qzmAM+YbY#0cdV7BE zl!LVzQzu__MLgXy^{&;utxI|j(U$6WD#rz8YEOx2upMuz81c@#Bm8|0&2k_gxVfG z0-H4aPzvS5xNc4)EpWDiv~ye=-+9q~1#}WCE zxho$w?JC|+9g<(aR?ls8$oXZDKkN|UYVCOtJ_-??D#vTt;(EgI(!zqKXauGi!Nn;c zC0Wp8dforM_TbPD+^glUc9i` z_K==%I|juiwU?YI;^gO84iLz)1u)&ktdq$kbhtc`#oMltDiD++2pl`FBfv;j6}A2J zZ$aT8xP9Z9m?@AmV6Wzfm&rIez)3JRz)T>EbxibOOGpkI%zdo{!? zt$AEh+ba`?F!NW7hHaCN*^@utwc=w|Y(-x3#Hs}Omx{feYj#^X|FM_d;)LClURl2H+|dI{^g zKmol2a1#V|wxv@>I%(TpypcR(T=DL}LX-CDg@k~1sZLb9Q1Y^dO<;t>LZ6N;z7Ufn z_}+HwaWp#*Sub(Qd9Izs(}Ez3Sn|}gthXAPOUFkMV~Tf`s3kSDAGX`|eB5$)TVqW1 zxN&4rO7{3aN|Z4+4vBW#?sf>%)hfqZ9s3h}=O-^cVexQ-ms4(HY=?a1yMe83P#Ljt-dw#xg@{+#sowL2u{Va8_mmGG8bu~cp31y4=K~8NsoEP@A zM0H%h9M!0BJPoaR^6KrK{~WjAjzC-y1q@bKYLlEeGo_lb9E|FUxzt zi8PtzTKP})JkJI>K&m4GL!y>&+`2XK!19heEe1)=zYuS~XPKGR6mGcl@uCP zFg334hS{aMFO1H^*w9B@hV;;UfJUB{sn@^8QZSL)}3y*gVl}yinS)tw%A7wnP{==QNZo}pm zx>}(28JpcnZ(Ox&c;T3Sak9Q$USQbGS?*O^I`1C^MWZP-;l`qvlGKP5MYfqG%8ouQ zqkQMm|F3V+_QJY3XGr(BW&8Ngpz!bzr!~G~!<16vGWCPHA0NzVtWHCS?3l9ENv($J zU(fOgBBA29%v{(?tnr?*+O?)?BIkqH0FG$6e!| zw+gkDaZB8lmU;Trl&W3uI_U4Zt4sS#2@>gA>L=WeseK-zdv2}!p|rIxey@t$t&7-@ zoV_IrZ8L`!g$}OO`^{#`FB^}91(}|6K=`OZ&7HEg8)n*_nC*i(h$AQ(Z`Ol1qM0kvwUHL9!Ld>i^hXyD~gW`pJx0Lp} zyXtx^4^%Mm_NDy_4q6h?fzJ3t*g_N(ifkNiJU1Po8e%tz#sYN$WF0`n`4DlmL|Hf>ITmkADF;pS2JY5-6Js3X;`ZrH_ zf|7|VNqoQ^Z>wNM?ir3{O4MF<=a*k##fm!#HGHw~uHp%fz;E24!J)YO{uR7LzCV>& z=vDad>XX}SeVSMKh98Tb(usTFi-IMA_uduDWja_-d!*7`6YiT^*?r;anG2p*2e@VC zpT4`$`)KszKb^Ogy7oN5oNIPWr@Cu|3PmFrO_0B~*3Micr7(pR6J~V}M z7ML@CGg7UI4)z<-3FH));F$RwVMd?Ei=`sfum*J7i)`FXzADZgl^hZ{J?neN4PwmmUm05xOQ^Om_VhE zJx2q~*Gx0Li5#u(rkuU;3}!xHFQR|DIV;(1h{P(z{vJ4C4A=&2eqqabdNgh_Kwr=P zpR7&}9*NZoXHGCoe?!NsBk)dN2o?sa{>eGr$r{jCx1Ow|`aoR$9Juah|W zFd#-K{ATo`XpZm)*O@N8^ShIguC_K(Ca`1?g?EK4rw*rV{UG}lj&KBkjV$&Myx|Is zm$Pz;D_*@6{a*A6iKVoLwnTkL_pX< zjBk<0W%gr!#beD({iT=iD+@z9U^|(+;9ZB6bJ$!7g(|*P=B~x3>>c7WcE?`?quwS= zb=)i*Bnc5VUs(FNo&WPlG@8RYk4NT;1qFBRc89sQE`|D*FFBlYE=XM@WV~Y;@jN2wiX$VQbMz|1*ctORA?UWXlsl3obd1< z-qj7{Lx#4Ky%>oyFb7Av1d6s9i@_v{h6bYUEA;Awr5Pk0(! zzH4s3FYI<%59y0kfD(R^j6M9h5njX}>h3f^le|q31ScmV6cDYDD_~zX|T^TgT+HjRmVq!43~T46!9vrW?;-N z+hEUlwQ(7JhH?iyFGl<9KbuJTf7+?Eo8c;yc7H;}*1x*Mm_4s@*j=S&#`n>rwa1>(G`@Ob=>mJ8)KdGq$ zZTa{Z%3q*O$?ASLWS(n(iqa=uL{H?Wc%)pT?Ct`sxs@FQ1HXHJt|(%XnqGisZ{Fag zgW8l=ovYQ3$T!iIs+l!U_TeP2-<2csg#Q5bPn@TI5!?Q|Nq2_A<^NiK=Uz(K<9V+; z6zz{X_ofZ{OmES1zmBxI@>V7@Wqxc*J3S(xwg4aHqIEz!BJze*bKk+?!Ja+`{#tPL z$e>w!lYjkr?q_)$<+LM0BrxF5w!D<=R}?x(kZ znnwR0M!174(mqB?A+tK8riC=2mz|JQIlTEG1%S!)4!}Igfk5?TY=a<+xJ(;)8{iglOud-#W zhMV{Y0LqC&i0t;}fcz441e@bNx)(%|i_a6q) z4I<5|CsKQ$X74TI(W|;&M*HTbJc`Tj`$zVHee1?}-m5S$PP@CQ`F_mn+kkz)pOurm zxqH@4slP65M>xi;1h0Ly^O1Pf(tS`ips!BWGNW%^@bJpM!zUzeBjI1!);kNCI;W2H z8EYlDZQB#LZj-fH^)43S?)jP9s+X((cFO}PQon8)rq}jW>>}S?A*C*N!hMr^axL{E z3%v3sr)ujpFW38~B=qs~!zwx>y5~EFb~nEF%`BW+_UN4QTobeGQxnT}?wI*v{E|ys zDsu}>k1px_1~GV^8V++a_H?zKyyX<<_I7l{llq@b-zfaB0#ph$d1K!jga4DDC{tpz zs^`{hk9w`Ot6K#cRjmfB@q6%0BYXtwYms4${|=<;qiW@`4T1!bxv8uK*4a@pJ?WDc4nx;_ z2Ux1<_uY}#M_h+?|Bv_Gt9^J--!EBx)|KB4GOw5w*k^QeT8-TjKd;9Dlk@sk{`BfU z5)<}|#Du;>$#c64l$A7k+Z6jhzo%9o|If67h23qz2Zr|<{qNYi9tCrrKs{-i<|biM zP|T<{`$l?I!(kh`Dr&91)<1$a&4eQMjBIsTK7wz_s*_<3hUCRHg7hW_Wd(!urBf^Cgjsy zX;1lOrS8oJ-^w@oo^5Y)gG0Cm>2!k{FCp_|d{*=L-m{hoHnEDB!hDgDb}$LiToGYX zZU;SW>t3^%-h3+NL;2S!u9aTN6RsmG;k$C>#1Fp5JTtOJB7W*We?`Pg$oGf9+!)t$ z#IAB&{qE`8bi&tYZk>W!17*P4_Y)JBFO-rx*Tl2zUp0%%97zv6RkBJ0=-k`QEm> z1xE$@W&WM~CXQ)=I-~VORnjSbm)!x4GJB6qJ<^O{rKY5lA3u8!%Au_PE2WWupN+< zfB+NUokdk0`!{zRuKkCTgSkX{GyTD`k#R4g(6prW`wU?Y{<332`MD6a99&XI71#Uj zd7#kqUe6$Lpp{?l4D?;t^HOnl6Z0_&U3HErW5C1AYX(W3u(qBW)c7Nekf3?^x8+Ik zaj!zZ1AqDH8oh`nkM3u&eL7tpwim-Zo<84uEP4I20-%y=UV#|aiUQSX^8l;v&HBUX zDPP$T>8VBsSm5P|OyV>^ImddbR(rwvi(eG4ecybl+2e%uMSD0r)Xz39*uJJYu5hNC zoA23}*1TE+C-+jg-Of+zo{hf!1Np3F-46yVTae+`eoL`Z!KKS_nE!YdK6{TUR<^$I zmNW(wDy%n*Ihob>nOycd`(ztUdq>AG`Nmkyz?>I7#=V^iy{E`~JLA%_(xko9KRA>! zyZn@}#-y%@-^ZSYE{VfwSKoz3*oBo`_j#+j!bG zaDH0#HyuU#*Nz0;X>F*w`QpyWESXW`vr00Lj5)T@_U-xk+t<0ARe}Yk1Eh6pqTAm` z_85{7EUa1lQ}0a%Y5@B`9*c&9(mI}+y>?x*J=g)BsTo^GC$&_Di?}$=6J)lzM0Af3 zPA!W-+-^Ca^2jmGD4}GCRks}m^k@cI)w24P+dAS$)F|mnuYrC_2YHj6j2f5sI7u6^Wf6;UjL9NJ3S|L*x2%yhvj&P!jWDwUZ5rIY zNcWug9?VR)E$tm5LbRWUj-P7lt9R$f5tGjI1DZdqxCLQOVIUwM!>AZ! zJ`A^8osi;wHafmmGslG#t30`YE~KjR3tCqWYVIDWZ@)-otglyXmFK#fo5M`HpH7j7 zMNFVd#Dj{lD!I**;o(){{l|BoU%$6M=I+Uuy0~Nb-7fUMhQ8@iHS@-&o9H2DC<4q4 zVL+E3*(*b;PJjDUov0!8$LZhq3{XjOXqfZwh!O9)K8^low)8`Ftq)mq z!IJ0pReoXWyA530bzJgr6zzOzP#v%xf|}4=<74&{6M1#F}9a=n5 z8o&O;K`W~cpE*{$!Q66vgZ=TYh>tsbnmv4uL_5I2;Njw>S+|ESvL2mVo&8t#j9*=K zmJQNeH>75km3`rFfLeZnFidOsAHOx;E8b)}bj;9&X9~|>dgUYMwEvhkF#$gmL`H^B zu3bMpQA)y~JXI+2oVQvG-1>`!Y_@&OaQSs%Tlh^bJsTD{Id9`lD>Z$^ufs-X??|lr z!7FQAA4Zp_qUIbt>s@lT>Z)z))k!7O5(iEi4&?iSj>yIyJYkTf{r*?ghEZNuSMAvd zT1oqxkYBGE-jo_H{q@Zaj8Hj+DdkVcQuISN84epWbn>%VO7apJ3b8a1wQ8?xWFfD4 zLJ~4%RMbGJ#FdvuNaB+1HtjQ4vM&5hGjRE!xt7v<3vW1CN@rR;*)Tfy#Q3@`=|=L? zL$+LauxP=gr!h*);*`;Y*&EzY3{zR8zer}uovxKWuZw(7uiA5Z)$23UwoHWU43v% z=&w;yewFtMUGAKYD(jq~-ul4by=&|(_kt2{uQKIv1;032CyWZau_#N%QvLyOA}!~w z2zMHOPc?5NxuDrwq>+4ibp!yFhuTtjZd;W#koF`$#9jv7kGX|x$>=KMeHN=A;W?V2 z5t#W}aY=H^7Q_w~dg1Hy7{;xAe#-E`BTW&@9bq;#OY5OF|jxIqWXQ)r{XX%R6WWbgye1yLGe zir>~uVI&R(3CU}~pv9+Cs{wpwYc#xZP#^t&9_d6FePMzG3xrRhE#SWj`6B3P}+n8 zbJA7UckRYgA}dYvZY5 z8oxFA*Hx4GZ=jgNlNRDtdOHabJ?5xTJ3)@683-s11SC8i!ufdH(DOs^3T+M40pQXD zQ*HW7ezUZOthwVcc4R2(AUVc*LHY@Xu+#$z?C$I+Qr)g4Zt!)99PIIKduwU}#Oe`8 zRHklk%CgcwAPuo6eS9)scwnpqe$*?zb|CC+rol<&{Z3N&RQ&7{SpaHCL%>}Fx$WYc zOfq_~2=xPSQqJdWqUo*+xx$56J4b8k>z&fJL}0xjyjP;ne~Y;^Zx4<~gh3!p%+L!e z0!3V0n1yf4cTY#D^T6LIG{AE~9}G_{2%WHeC}zP|JsFB(odhkQeh_8DBLF7CpB@21 z9!}7q1k7Syi3~Brzd|W=HSyc%(}2fWEn5II%g4+Lf=jTi5o2TQ8&L|=I2v$IEpy$A zugkwwTx_oRf0aZ2{pqjc^l!#vv_%>t+)Y@10LzDjmhYA%@TS?MpTav1ttp-nkw}QK zVt-#qqc?0Hda(F(=lEX03de-Y)}L&Ur#8%d5On9w(6uDPD@9lg;ib0&dD)}hbi!YZ z=V@=^N8X-UPoddsA+y79J~S6_q_DiohpCld)#IJgrKOn$_F{79@O|(eBkY2TTM12 z_78mmGz|jf(S3tm**RJ8zd3B`x6%L=XV)ZK`d?wJC+8;wV$39{It~hq@E_Zn2p(u6 zNR=d`Hqr^pUZy75YR!)omGgSv9q&7p_I>Z9*tTzD3rL(TVc<+Evjf1m(C3 zfT*t33|$bccf36=4vK%txYvd~MW&c zWk!!fl*xjuDS27F_~il4>x4frRY*c1APxyl97$f{sX?W@pMEqLdeUKtY+ojO2&+$u zC}>V7d^@?>L6W7z8uT9nO3QX2>Y&~+#i-f;g8j{DwaEh?KU4PZ>gw*+>o!?3yF(%6 zaoF8ytS50EQ!+@cAi1YC$6#WNGGo^S99=j|gr1q)NzQD=!odibIu&8<$j6JTOEH6s zj^CrzC<=0N$Bq>f zadL!6n2}-DQ8|P+>u_RG`pS3}iu$qsaE*Ia!)KdIWh zJdu(ND-;6ZFmbj$A~G2;F#z3_dNO(hHq*fq;F%0_+;|iw0I68z?G#a}c?h2YZ(B#+sZvzfwUokWYu|)`Y$ohAM36Yq-E&b-&r9)~4 zS<%S>SK24r9l@uiOox{Tra0my^W>r_a4D?M_!xxR93n7Y=Cqb1Vfw~1&%{;UhJcK4 z>#|1?ey{Wjhf;)ua^e5do~28xY@EWJ+jFZev=G8Zn8KlK=zxXoV&NEzQN0+9VF|M3 z7Fs*EF~1~obSC8r*Et+6Ln06e36(GvY>C;@er+fm3BDUN$|^CtaoSDeV*STEEYBtA-0Zt=ak3aXrK3 zO_+s2f(jccapeh9T(Z&_r%q1`)Oy148Wz7}{5Xw72?wl$w*iYzGE$B@|9uNjA0k7K zA^P~hLX{;H)0jj?#Qc<+g-w5{cn?8jj6`n1Ku~IV$Gx6gbsyIkrc@DNM$iF~FL*n= zZ<;95o^RIqn7;)Kjd7)VQ~XUT%k&TiM+knk@VTeN7qkuqVPL^!Ofi04YJFXk61t8J zIs)OsekAx!EgF3i=*)!!8T&~{`RERXP03Ry4xAn6Dfrw_%WOntV>21WV|Php&nO%c zzDO};K!XF$7VcP~DcGGC77#Ki+c1jb(=CiF!X z`Ui@Zl<9gvDqrp*G6tIw;Sna{8%P18KoYxx5(O&4sTju86qF)f{B$bZh~EU&+Xo8h zY=rz4HtoKdk7_=z$}~EeH5eWt!snh>0NxF+WlK6AipqxtB2AG|Yi2>yfhtqn)8b?T zXNkff>_h4v#ZVKSQI&BMWz-ZO(}n*PPdZ8x>En7bi-y2Z&GjCJrd%+La-gZ8Hy>6? z_aLDH#h;~zq9h44rgoMwC}+H`-`X{qEK~GFLTAhq@9(Sb^(vo@PR-Qb(2RH!?gE;s zAfYDuG^{KpUR~;V6^=5Je+?dX{yp&PiTw68fYMI0#hTMYXa-Z>-ctU`t?z@e_Q_C` zio7NTHwY&UKSaIr4}TggbGt(c=bJ~*Oj5Q4swphZP!z9XeYGw7J&NRPdI&VCE4tAlyea=XfNTY6+_H#bwVc z5m0#1#54Ec-L>L|rJp)EU~rupT#AzioCR?^v;5N*kuPGK!R5gB5#83}k^wj(n z`0ENAd@485MhWh(T3WAKcSmI3tQ@=E&$afcWF2}o`n=A&EF+kBIUd_`9)oMgV`#N` z2=kFQ3AZ`eUvE6GBUA^%bD!c%PoB-VaA;NuL74A-E-!K2QXf`J#WpndK`9JQ_+!Ky%jZ2QtNW?+0qgIN`!RyW0 zU0ZOnm#PyB!hVOz-L0s~z2YKZQ#|_fkCBj#qg)^$P?xEy* zQ9FwO=6$cX4@{q&a&Pj7pW!UYb!IbJ2+H_Ec@Z;}LzJ7anBrm}Hyh{#!?l=xDP-{Y z%~C2Sr)t3*-6a=|AP?rm7etCfz;uy&yKi`;my8>mO( zRUy3OsVbu~fW~JcvW0;=+<(sHtORneFQ%@Ymz=0dD6S^=6xWmZ{b(}+6{JFiND=^@Zm38y>5pH6t9vcEp16Q$w8r0@ zdiv{kpI>j}1qhZsi+*aC_KS7FKTJ6>j2G1Jv4BO+%uMt;DOgWUMqh<3H8(`u{vy%=&V<0ef=m!= z^><7a2t=rt1PlP~6Z{eNa)64&=K}-c`YQ0uGH6HS=ZbAfUWW=(G^7tC<)u8p$;U{B z0(!eq^@$Shs$J(*Q9!a-;yK%hQ>v&L$q~d4w+e^(!Y1IE!UO_6n_8#Zd+b z4#^*&$5w6l1|H`9LBVgX}q`OK06aqXHzYBH7pBgja9tZQ7e!XS>Iv*UK7sA`>VN*{JP-tPv*Q0Qm-grEv zC`J~AhLm7>v!10-k~m!X;o#Ylk4-mk@q%z#OKcIGM(u6$5i$;|Ir$|^|!`_PRdGA){*#q8(%}2S;On}Z;xm}2oIU*r7URDcq+<>VoPK? zDxh(u{m^&;YXT;*Y}css8zO_FALttG+qn-ZLuN70okFmer3uVHxTdjQGEfT*r2{mM zQ5G8f?o_&IX~m&pU}hrz3JNXyUhtd|WeUSk(M3?uh3*^1?dS{y7+3<=F*E_2%YIbb zY#mucH0{hj?T9dmm3%t(oMojC8;SmwM_JKA7M_@^zr;5I*r7k9-$f`F7af^=^x2d` z&Sy{%T`WqAR~Z}#ahAn zg9}c$cfziR(4C?b=MjJ|9AGvvAHWv&8=WBW8L%%wc<1dM0~r;JW0lP&c%F3XfMX%j z@H7JXYB|UW8c$C#RxrF2c&`3!8Z!&W83rM6LIu|j%v2!YxNM%<VPfpC;J+1a# zg%`#KiwMFpGC@lmXl#?X)#>NLxdR$g z7D@;gmrchp&ggG^W86fpz%E_!+CxHtZ4e&?mcO=0kqJZGci0#7d_wrCg{*|+S@P{! zr>!l*5ENHYm9Ut{w{0g_!poU#S8?2OW%Jo?o)ftOX7PQal`wCVz~~K*P`Lkj3UaoR z+CW=woWlSB82+j%a+|T9;PWHEG2((ChL_?VCS??lk;Nc7c5GFr1VHwbN$j2zC5#X8 zb0p}BXBij!hdX|4G5}7*FtOuV#X1Ey0097ct+P_c+lV$~heF_jU@`sjP${Aq1jNf8 zf84MCTfEC)KnZRus&J5qCtj5Z*o^VopDr3B;y^S_J$U~CdTt?Cw)07X)6TH!*#M>5D`Kx|6+xuEU9#?k8lJ$wE;I0*f9I4KX#5~`!7&9 zXkr(NHB5w!B|l9Ft4N$S3WF(HY4?#KazkJ_0n1k?5U_3`QUE6%^Z@$B(y0wLcc^0c zu0h+kfM)jDXr329b@)JVYGOLA1@hZmrNosd9-JAHDGy??e}#RY9~%@bU=9Za$j3K^ z_rxlmX9DLDG^YgGg?1I+R|u$rh2b?eDpm?0C>(&ONI*Krv8L^oCJcOeUhsNDbq#-L zqP8~C=7Ws+Kw&?^1 zrMDept5_&U8?sumaTIWpn2bQ5EgCvb<(=W{iz|o!su79e^g0aYh03!Avy$P8c$w@o zMK!|HfG0I8UYPw0MFEObOhE~zDrg5H1Ow|>V43`A>~f?D5e4Edgn=SpIDS7~KBD}l z+=&GX+bRqZm>DPjJ@ydbefkG*b}`@99309)ARzf9uH;gJt+)2F1)9QG|C3V*Xf`-M z0$-&%I4+o!qM@lc=(h=lPk)F8sm@5a*n-Dqg;}VD2E4jRb-XF{b0U0dVXU>?tQ7Z* zhzw+$2ry0p6)ergWIgTKXnZHmAdnG(_k{HsjC~Cv4Vx>09*fBymZ)phv6;*t=Hb-8 z18^i^qpE0Y@xv@-Y~+07ASgE`9b}mebAI~T(kfVzZUj3zOlz7&C=wJj!ctop-!adN zgBH|PQt51>T`@OUC)ulQJ3>0ESr>{N8b(v-fXBp}gzp`P8Z=ayMv`FM_8jYBzeo@c z@)K;6!nE~OWoka#5=vWC<~gy?U?>0{1s#MFS3+hpG@XzjYddOf@WkuBqRp7XQyY3S z!W0rj5FB0HxG=xQAr5i^tZMKGzukmRfy;^Cf>mJXKtG)F7P6m;6q+YVXm>WWloId4 z8L$Cn%vgpIY=c3eHlO={QEW%_H{?cc-w{~DC+CqdAhJd2tHUe|&Moc}Dr9m`zk7cb z1_7+%5K7m0&871iO9rIqp|B(-3>1!H#iE@O_6DNWrFV@0uW?oeISZ8h?%iX~>JJcC zgS#4eEg43FE~j?)u^Qkaa%s?6b89g#8VA8da5;*n@#bYc!{Zm~^ES39+p)}~(IXM# zV*5=~1S~wKTGd;2CJL1uN^_Oq!82XiO8Adqslr|e4G0=R;W|SHYBmeZy}_Bo1055y zhmFHnX65;L4qIFPxL{YSKWGN6;4YwzN7apI&D-2%LLe?vmOG!|EKCGJzr%Q0pzn$q zqK(4p#&@LQzb9J)JR2E5XSQSjno!4Jc`RK3hBADnf=^h`YzYvSfNg~#BY}ZGzrQG88IL7)uIBiY6N9HgX`ZgJ0tfuVt#V ziz;(!-d7BZR=jqs3{9ZrmAkV|@Nx`W23MUdq`2VrYn!4tK>oZTPJB{AQSa=4fVs^U;6T!;r$+=O8cs zqZH&ibM$GDIN>G7c7G9@Cx|PkrG*ZNF+oUlAsr4>HU>oSc)}^K4bC!hw<3No=*}zn zK+6r5ZaGH5+#sF+${-OHMu5J3a1%opa@^~kgK(Q~gI%W+tQk2(MJ-S04b==@O)wvd zKzMn52}3}h`q-arLGgD9sOVvz3eFg?RW#)+s^}e0Sd(<0o#-9#VK=>ehFDBu^TqxE z5co~6WjNW^mXn&rOXY6ci-!adM6?bA8qwQ#jP${b?F_1X|A`rcDcX(?AIqUo`5px{ z*ic;l4K;8?3y*$BUVkKI;M@}`b#&60E#T+{BuFR9Zbmr^e#oSQ?ALGt@R#L!!RtH8 zs1tT{W-Tks?h|cu@3&Nw1fR(2!CSPMoVXIG2FfnCF<9gH(fI#pc7lHZ`wuP!93c99 z);_keIAZc-&6V^5P-$mX-YlA0Gk=Fiym97kz4NwE|IIa=mj; zB$#!H>_q(`H=c2?iaHW+hcpV5@>e0SrNg(Cu$cucPQD7KS=1aHT8Cs3UR5HZm-xhD zWz=D)c!krPfmySGz`HZ`Q2-DBcj-QHHYh_*5OC?}Hg)aKQ2avW`m1%X4sf zp2!*ud=ODUSgw5ZxcW$~09oL%&3G*yIULY+(FG*6cCE+sAej;+iTmI4 zNDwpn#Zi5qOhA`#AogsuHG=pw7fdpF_me`ENF|_s9=HI5**nLFJHSeCh+vwPoZa)C zaNr-;6+f;@%x=-*@HiDMC1Gx!0`5fqXrRJMiyGVZ{bublu(4y^QDo~5q+uO<2NoIQ z4RGBE%$}_~*I38iaqUvP<9}P7ClbrZX*2kBz$XqWM0Z^NxKzZ9?NiS~j+`cuK=H!S zX~m>f$dQwosFBsYG2vW6( z-dzZaHlU|dR&R#X+Euf>>%Y9-a(D-vcH!x1{p8xl>4|~AEM5fbSf>!96-KazNIazM zG`zs>8tu4XvnNH1)kS@oe?qo)<%IEbxFWI703p{8gLn=ls&vVhTBi9o5$mNmQipC!8}WJm@#&! zfjkq6$Qo>CFi1P#r8|HGYo-%oX+m4r%}@x|GxsK^NaHL4$_kt389r63e2-n;R=G~E zP4$V@M12Icp0amz4|)H7is+#4uLmWI1^hD*&ew<4o(Toj8p$|RAc1J`JHZ&1Ux;kO zv)y ze&8IK!x2GH;Snc#!wGgx6U3M=viD776Aiah@X5xA1|bq8*aEjP@Z{d(hm*9$$(C>i zqv<(znh7T4g5(tv*}Z|1c7Ym`dG`dfQ5J)*xDIJ^&tZ&2U|z!gSR{Xt0RqfeGcd23 zbF^Fk$VH@$DYqo24kDuuty6dIYWHBRu_p_~?iaFrPh&qx7#Ia$zf>*EL=5p<_zt_2v_Z2M4KR zxGSeJy_g+%HzfPx&<2&Kw-O(8YNEzpTXO=dbUll|8JFEsw06lVawm@O$#W>a>v_`V z3uZ3Y{dqpX9+wg>Q``6dv;$wKTy(5lR)znts^koo&F87O%cNMFtQ6)6?>Z zY3qGrEcfl+-7kJwrj_5US$``rF?Xm!IarPF?k88Z&Q5AW7Wn!qKVS{`LS12FB!>8RcjFu&<} zY|HbqTM5+%0W0O0RGwU8wyn|Mw>77%^GcLk!`L=lKg$%njN$G1n_-OE&Y`8vQyz_( z4FjH!OJM;SBG9C< zAu`?h$Yx|37vzRliR6sf#<#H#;SLz#y*2TK;oz*P;ia!{V!(JO(oMHz zWEKq&-EMfN^Gs~2&M`yKM2i2J+!@DG+Q*N3_{JeB$+g!c;jhqNs`BDjm~@}E57FW{PLsDG5Q+yx)&#pEwb}Cw92z%)hn0YPM4(GdT%bh zq}Mgy(_U_P<br~0QgzAsvyE`P(pckWqMtmQr7<)Dew*L-+2 zyp&YVnvbKH$c$Mc0_U)T*O|IvO0TSV;VZAhHj_1_E?pnXfy7S=UK3S37X9*>ti8F$ zR5ky-94_Q|=3Q3DTwv{dPW(OL?!l{{^JoBnB6St4#Sdh-a?U|UCrN`=92S$aK`yU| z^uUbHA5l%B_}j16tr86Mn3LYvS~X(4_25a4CLX^kGwe643>I%!i(Gp$0chEb^?*P`~5xDJh zMZ0Ib###WktZ+u0SJfmotu;lF6^)tUCXH4e?rB>_C-3;m>yNrrSuEP9&N)*}dOd!K zjxT!e*8O_hCdczhuDh!}>bobO4>mV33r=$An^SYO>d-3R5wBCLel_y$-PJHDug1Mb z@5j;eD#oq6<({)*Dm=o_7fdgkUe%r(mVCGTb?vymJFl`1)i2Xc?r1;aR$kI&eCO7Z zF?tui>d;{IH#K}g_L&1&nB^bZQhl>=jPAId!H#vA$#swJBpDB(!}R<4Wq#^DM|*nq zM7-@vaxINaNGtRA-{xLD^Le=6s6UIneo;%VZx<}@gO^QjzYnUj zeW!TA=fVDD569ewvR;)d`Mnu)cC}A4Dsu1be6qsp_rm%K3p6we$JGBitE1^w-0jrg z&U#E0UZxX~ ztjDi#!ZR1OIninbAGGU0U&1G5T~_b;fyFzpY0NW#JEK|ZkG+P~xWJUU>b?Kk`nm#1 z&NE+lj#q*7N^uR2J(fH?S#do+35m>lf9u5~9GBeouW>gswIc&lE7zLlQPw;po7U^K zR+{uwhCh9-2|Izn&1-Il4Q^2i({mfE@D;r0BYyvGSVQ~5ri`#LX|0>KHRv^}-9jS8 z=aOu<%DHI|1J9-%Z22+AVK84I&WuiH$?o8AlOZUU$6F!~a&( zttCERA$_vUWOG$sVp)P$x{GO5!Ow}(`a{Oo8XYh8%ykTWzB*4uBO>SNX#UD0ZTpt} zJs(A47Vv!_0h3cJCllj_dBP?Gt$k##5DXQh?H@~>QB;ZpgohAEuk3OdJxcO{puiCA zCLj~=3MWV7E7I4or=v<4+q#aSGnMjSM?BP#h7X zRb$CWvNCE_5jE+GCiuf9m)1?3B91qpoNsLr9F9znjy}DxN-ER(YGgh_g8%L?|8;ov zjeKx3`EA;ZWOtP}w-199;M|$&5PuFmY2RqM2%q{NTUYrXfEX_TNpjApCB|f2W9F}= zGynvF$|I5%(Z3=E7%nr>6JT?xxMS^vORi>^9;3PufqsYw1Pd(c>8#Qcc)C(0F*_y(F0+n_0#Y| zV<`}NPu=T*Di&hitGe7+0IPUPYxsof3ExTWpcWX62Z*H1-*Lc9gZ_xc9fK+HrKpjHk zRPM23?UDm=$&1V%_Q#5-uqREi_N> zQ!@v@o7zS;4^v^IPh)JGb29H!{2s_m$RD+UX&-n595osT9jyv*rp+5syh(;7AblMhbv=rbw*m}7QN&Gh3x|GERxbAY_m@qHau0IDeXwnCJc$1xedmcO4C$>R;4{>OejjHO_@8KPK);Xz1~;MJ!9Oz|G$sN znLlV!=UnG=eLnB~dOexnl=zl^6wT91>UFVRW*8EArqQ)2<=xwgy~Q77Y3PIuAD#XG zQ;Zo+6F2PcuH4!4?>jDCYSr!uErFsclSpKM)AGl7&nJP3DhWHvTOMYoRq%Z#tWgM; zWG&OqxMQU(rINL@2s#WaB~P$CpB)IQE|RZEn?IIX4n3Wn$vc*=j$Eh&IgCb0)g^0Z<}z0Yvw>R#4j>zY=psQo?RV;;(ue=jZoiJS>4 z2_|BA)E>{-1o_ylHVl-=+o=M(z>*M$CPy9DP*Wx0f(Wz|3#Uosv4Ui{>fvoJ5tTY| zt`OB)c$}jLeaF?%!O`MchAjlYQS$Z%C-E9X4bKy+nzm1Jh~X$oXerBZOIv-+4ej-~ zmdXidMJCB|FNjopC3|5;o}skVB*di2T0!`6_m#?cuSv0z;p1g(>ik~YMiou#eVte! zU(1qQiGQAiKl^`P!nI(@a`Riat0ibEK!ZZe-0*aykDX`?%Ur`!B8qn?G_rDoqEEpZ zKo@YSlSL;5-B~TEMW6Jnh?a6;THIh<+uTAZci_S`AP%=Wf1<3o|Dof@Cx*#Yuv)dJ zB|KnBXxT_E2y6R_B__{OlVQHNxKLbVY91iIR})FB|MI@ukGOH?6J&#@;`1enR7?{Q zb^p@xT6EOPPZ;f0?`KfdR0?xVu@}r(VS$rHlSO={&2;^-QWE|F!$4~J9- z)-2R%Eg9W<1A$<%@vAd6*N@CjpDHuD7y_ZGI=t1NSe)g|;xLtoHkI&EkHS%-rc-P6 z+a@as7X0(aw~Tr)nkr-adLU4F--x#k*J8*_MmhYW>IOY35=uf(@4e`#n2I1b{-0Hl zTXws}|NNo<{`gm%29Un3*;YtWs;)8D# zu7q$UK8_NEdqTJ;gnL3*$9!)GgcaQnhe5a!!jyph%y5HJPR#M?aH@ms>AdxMkGx z8Q_EB|5RyZK5tKA5FRWXR@!Yf@8T0YKCC3>tPlqz^sSMJ&*m-o(MZ!OW7;wiL15fI zHG#<${GpnG%r1F2YcM+9g*JvvE1@MER8goWJ(Ue5 z&L!7RmmRe2IyW)1>Sw`_c~j^comq(eb+d@qPRWy#PEy55?eEPGdw_L`F{Dfdk_m5>kGs$WXF+Jo&Pi zvRw>x8Dcr#R){OBLPt;0Ts9V@J`E{Ser4fHz;z)zMk1@KQWl0^Fn{V>Jhq(y`V2m% znn=u^JY1D##6IA-@dMXI@E57ZVj)=W&TEddAWN|4QlH$jANYIj$<5%_i?#M|PCBGt zXVtA^7^y4>!2{*_U06jHUWOGHj+b;$sD*B7IizLUTohOqS@ZB{_4ru4-usW8kqTv^ zKR?+g9u-ANX>F)`{H>AND+b zKOS)O3Fr5&pKAxzxDD~B4jPJB;9Gc-4` zdBP5y2r?! zKu*7Pi`~mDuLh%D9(>@G3Z-|!(cIp_op*mJCu$r+bb%7u-Q-Sa+(%k>(2fGC z2Xccc5GYh5%Nenl1pp&Fk+ z@C|Yd451hp!b=t6&@pk#X?25wE#Q2CMQ+(o-JrzH&Iine(|I-#x`X%2#cqS~O~49b z!Ggw}(swE)?=ax2pdvuKWdX^@2JL&>3lXcCBnwVAefam%ckXAE`fBZ4GP3Xal$&Kw zkCPBwcHet*s9t9zHoHefsjH0t@Ziol;p2bakaQl*%J~G^c1vDfKui>R^RiTD1IRK? zCO)GJxKjm_tDWPm2*I-q>V&}jwENT)DBcH66pE}6(-yokGm#AnBfVkh$({vs))6x9 zmBUcTfa>8)NN`7!q%Wj4rm%;0)VP4JPW0AaZoDXigf!>zRYfc!HUpp%5DfL~_J0XE z!BpbIsu4A58N}vmP8)@K&Vn8XVSm1YPz?04*zBQ!lu>m8$Uq!&&bFkd=1o2|rn?UG z+WWJwnH0T?OgZLd8U`tr1v~xz6RlJ0BY8t{IH%?I_z93zhi<20F~pLfjB_mRIw7E} zp|b?h+Y6jYIr`lo6!SrX#`3~mz{W}O621$I8< z-W=!^xlp5o%&`t6)90VS+TO&E0%9K31#n9ybbdB~bgD}r7?vWq!fEO0mGqlMo~8hI zzk4^`1l-MXXoGqPrh62TaO0!HMn^WUf7$)VSyE{K?i|VCnJ%`gdkJE(8uWv9%S)&U zR%XOi`a*bkF%*Ay0E46hPOdb>M$|-zY7ze8g(dA!a8|#BIf94}6lG1txdrR|xz9XCSKO_tzAeyB3A(|S z;jVGw+XO~ED^RLT)%S7hbWl>|?9U+m$O?$sE5nLHOP=@>B$7;P`$KvMHp(ezn=gSB zCv?h}<1FEH6zM_4T5fP~JiR~5L4F-@wKs?x8Ue<9B)tOhw~K-3@wrm%vjOTXv4{}J ztO6_uF>!iJ)1=uMH`7T&x1(XaiDWG_pA8dmjv-5_Sn%+nUV9I5xFI9o3>%mA*fLxE z+SI*C;2doy6==}rBw?OG=E^N44k7f$mynhG*wO$o^LI-lnLJ>h5C73M&c5S>AK09R z4Db>Aq_+e3M-@(-K1U#ileUHNVg?w|l#$0y(_DHv_&6i&O;$jqO z_^PI7R^-Aw0IsCzZFh_YX=iFgQL6xW? zqU;n1kVWkyhY!YW{66>F`Fhglh>t|?5rm~^i(xsOHAyba}VXpKR7n0FsW z*}g*|S76`=oTO$F5vU>G7+6AHLeOP0YDuh{aWET1jqbvmHJb-Ry$I$6=gx*D5S4nZ zc5;nNt#r}8$QyH)FDl(*vC;CNZEO{rmG)-*qpfmh@-DV<{rJ5j%DnCkJPL7EQ{VuW zpGV8dfLCVQKX;*R9z`F#mB2WAtnZ!2-MtAP57beyw_kB@M}%wLm~<_zBcKW%&hip^ z`f_ABr^7`Bn)AGe3ay8h!k0w5rd~W72Rm#%(!%W^!)xnCODty6ImZNwMit*{^<>=9~ z5Qt7}(g_3B20sf`j$_eWStzmcr0D;>(}lAK&_7wopyTK zc5CpCpf;%rG!w4`?~6c$Fs(xk(j$_fGexRFvt2Hr@xr`oGVB@&WgQWygdztz0KOjZ zOJkh24||888H?lNC+6Uu#n5uLf_K30#~Z0XykfvbKu%|`A7G;Sq*9$oJXzcypu3>8 zZc=g0@~F8J^mq;US*MBV4N{k5{GW~G|F)Qayk)2MujBzGO`YXO+)pO2y0H0nYb=}n=fd0+T8oZZlKz68TN7jRQ;@gsb zTn-QoAqZeJHFS_jAgZXcWym@9f@$^;RU=?7dDYD!6)mJMXPMh=`3kv+U*32>Wr%s& z7r}wr;CD9QEcCtRj!aT~1)=eo^tSo5_ekP%K~~wVEZJ+~wY9a-&7)u|r(t4F47V@B zx{bi(tiFf*A;p|mr*ke=2fV3PR^{M~j5mJgY;tOF_Uxo>DdW`yb&xeH6JMi@uhcGH z1_c9RRgkbUg*w^h2~a3Jv}=5x(~=wQ3s^)%eBl1H%ObHmwZdg~=8Gd5_8sHw%bXRv zzQ$>^)zgFID{J1h-+E`q$14QEWx1p!khN+-I58FIyGsa`P6N;oO@4Uu`U8i4fkZ#R z(MU|g2^oZ-6H6a(o~URIKwWGB1@B%zBB~X3y@&1NaT`q>j1vV?i5+Kts-BZLc*nkk zu}L`E&kY6P)~$!_d()RKTh@SBac0W!^OT2(;G((${CY9Jt%@EcN+|F>A+mNpSb1`nEOD6~n z+v*3id67@8j+ZdzgoFZ{#EcuVZ#&-EDFBH337rO6evci0JPO!2 zPQk?lHtfiGX+SiUH3-f$2#qVii-gZI$tfV?-$_Q@59VtrL=!dH`GYz9qXJ#~uE~#W z`-gd>a}XPI;0Ud^-qK~~BbksyX~VPl3I%SCl(n^4hCE#;jat0Lx@Zy(xeN((iZ~rB$?~lZhapbsSpaALRbHawRJOC%}^wZeWa^Bk) z+rZp-;UXb&>B@xd6vmJ?KRgs<*L;b(kY<}iR(TK7tKQoB`lgPKFj7x{3E1v36kqUd zqSO$UYKbcH^B7n$sVw++5z_r82;9Gnp>G#?<;o%oD6Wv${@%+8+NkjZ4}OBfYB7?iNV zw+6-e`D`4iL@ri(umgb$p(<|DEthXz2E3)K4^oGRxM@2&L8a(ok-WO}@Nd!(Cg-S9 zD2?L@aYo>CkT8L8M=;H1{O#i5A#L%Co@cJ-tmT0thH{#Asyh9@Ne{~~fYlyfFkL{Z z+MZ&zKX-T|Vz>RaI><^vAXdpLFvv1W>{^~!JPphb_#U9<`xrDvJusOe6XlM8qperJFaHVE-SB&4D;n&&v1;tC4pEZ+6J?e}X&v{vB%q@;g; z?*3lgj>k8QBlVA_>93O=96X7m8tDf!U>xn&QZOS_H%S-2)Qn$3{-2r~h?BRH4Z3&} zXmTxdSM@Fo*KvncfxWk{q;pW!I~#)fK~QW2fau=63pFpY3-J|7EFf5dZs3NSyN`ts z2p^GXAYgu;CLw00Y{mI=NyPYb3YJ(LM5wxiuZ}bUwxKn${#8#u;+wg^{Nw{^dkd&8 zcK#R%5cDeAdSAMo+&A)7(>NFqEm`mnX_I|UE;v-YYAL=JR5Mwf>lC~(B1Ocr-{78? zlddG;RHW}5aJ-Pi_hF9XXrD%`)aIByEN#L&ZG<#yOP|0Q`BdZ(c{~M$i4PNV^FVEx zY6n?nAMxlqG z5S6+&`oW*}0_o}^*A+>1_2C+Z4+fOIVvwUeF*S)1U48=qrY`8vUWbaOn?=%jBC)WQ z1e?yG)>P8wU(W<7$F00fK%78nHZ&NhhUzVg?XpU+EW{$KBpc(!;p9Dis%H=Px`yWs z_Cx}ilP2I-iu-_cF{W(N(k)UF_hVdg?#POwR5nB41r+7<6cOCjA*2mcGwTN?y?eLV z6QG1npTyqfOijhuH;3QG%(4;idC}1iCIB*Pnyd{9n@b^#4-1WrBT`wRZ_k2kA z&$VlM0G1xy1NCK1r-A`fx>S}c%SN3RMs+#<6|$?yss$ijI1;&HMlMpoN-7_8 z?M-=CmW8Zu8RJJO{Y07Ww&NxB`nPZDNjz8uw+Cwtm>qs}0TX0Xl*R?62Qu|Im&Bda z!DJ-T8xZrH<<%8?elY96TQ$0a3x@Q&A8eO}#!YjKk5f+Y6?oAAdB;vFFgFE@R|+QA zST$sHK`hEMSvUr>C~T_4hCmSub~xI`|A|ctFgVpB?>&~_iT!XSn4P8_BmB|&bDLRV zl6x0ncRDR8)@ob5--*30Z4Qxne=YvI{rn2c ztisc9d4e6tk28PH%q%95bd_xYJ`~~FgiHHFf zQX>pwJO_F4zEH;&?9eu!CpP|b;Wc6$z!<}|VPQ7LQuYM&2`w19A5?z@G^49ib5whl zK}IFYO9oUxK_0LOg-Fd(3U<9lhSp*mFjCjPMnDm>sFHy|taO-3k7M*9$4zaUr ztzjnQp;21w14L*CF1~(Z5XztKdc_;vplONSLjdYeB2q1R1!yaVt%40|jM}T~*sfg;YS;{bqpv6ofiG0hkbKZ8^93>qTHP zQ7KX7t?L@X3xed@07S6oW#T>&vxPtibaNg8pJ!$Y^w$PA3m4Ij?TI}z{90WD^pv_+ zDU|@NTJ@M{Gl6|41I??af%p7v`w9Z&1rz<<0VHUWwf1{{!+!!Hh;~&2agN^0NZQ&k z_|o{CKNLEBQ+l7HaLjdD(uTaX9lFS91i>$$YLCx$7zqR67YLWU4~_?s!ek}@JQ~YI&C7BI7#47;^EH&DHQndf;s~}g#hfC(w zfLRDr4&{4f2zH<##mDczdj_djDHFND7&sS4;q4u_q5pkw_agN%yYH6;4Y~n(GQGU0Pc^kpS|5VD6)8eC(dR*g6M#U-T-Ei}~eg4DjyQ z#O)onDLwyTJ+`HzhBB0yPn!NwXS>04`(sG4XZTfn`zxU^tfODg5usO z&9NBZM=V6L>5@?GG#1@x zIa(#E6VR}VLb#<{eYq7C!h0N$Gb|8)!-b9i<8B$+X_>ya^xII55BoMf^wVA14?(vF zDs>OZ=XWCNyn;9Zy}c0Y{&UtZ4RX8gaj4EuX$93=5;iXYx;PZIL8&2R;GcW+f8&7; zstlV&$4Z^BwN#`X6E0c6lBt-)W z1{^QZlrr)kkNyzNzJpBm1~1~>C}z^(A1=xC11N&(LdG;_C{KE59&a$)q=cfOq-nL- zy!}!9K0Cy+FY4dS4x5K!aRL}5`QV%OkhHjN+VV~zGhUUz!kQwfFQ`x~+2$wS!R8ZM~iX~;x zeK&L_ z6=SNKWFGchVl4rZ;iXN$T7o48V|l0*cB#h~4Mx`~NprykosD$p`}FUh-bd$*DVCzx z*FI&FeCQc8jT*s#&-5G_^3l_4C5YQJ>TiHW00eh^J?`geRML@K!^AWF4L7&5SS^*B zR=}FDq~)3ZwcdgiDP6W%h<@17N9&rwv)4hzPW$WwCzKMsl-b06;h7ZZ|y|kbBysy z49>I_b^F?W+!_fceF9^bMZHf%1C%YYQicNNfDzl*=TkcK=4yr}UHa_OlPo<2i;-Qsj zIX3kkD5_@a%qUFsHlmF46`1yem|2J(Dv5TGSEmBO>FD8_=;nR(t#631ObrNy`u;1t za1`yTuj(SK1_iB#+{SU~z98uaY-uAORG;m&&stts45tF*!De8gk>fnF{k=CIJm37{ zfY-%eu5TnKi5^kY6ywHb8kC^@`0Y8g8e@8VAD9KSn0>MSC78k}Lf6)|%|WHQFWr9d zFhLY!{%9dVocoR5&Vd ztfg*mh-8Hq9{2gI$l-@geHYB((os9o+dY#&)xj*+yB z1%RiWqE&CAVE)z<+1eV1_@^_?%q;UT0fPXh755)3y*I?#n}xW#4OLbfGKVvOw6!BL zINDtB$wZ?gIVo$xqb_ zLExEb*Er2C3bh6pEoKuk>ERir{_oIkVPsoRV&?5X}1O^!pw$t9vX*S4- zW*`w+_3hc8p_hZQ!ddtl{_aca=u~dp|7)XfJ-Ql_L7ubzxXTVAoNpr<@#FjctvN6u z8DvQUFEjl<3N=B`oB>aaHDL@Lpi9;r=z8x(1^w_54qweb6 ziEd&T70rRzYBX$7kzOquA0O}cu+2{eNUGg|qd@@D_ByBAZeCAC&NU|8%~@z)D1t(W z+T&r$I>4gZLUa_MX=lSFB;=vxh8J>@UuQQ@Vf{D-_k?(zRjaQxSS4#|mNYi9c|E&$ zgeBaz_yXCvw#xd~k(`s>9EAX!>&sMqagmpxatD^|9_CUoHG(2WCWX#LMIjWpRkW;< zzI%eYgiP5x?mAn>M$&+y$A0QgATMVD(xUFs8A=6nRZwGcAJXaQR8i^_pYI9muejpq~}wTD}ckv^h0T4a&fd$GYTrkeaCo-V4w1O>wMZ?3}IGc0w0~i_1CF1 zS4xHDPR4dUzm=_hjcR(94C+cdYD(;^eDwtp0yNIeu#Cx=6Q|~s4>C}B`(le!GDckAj(bK2|Bmn_4Xwekx-i3h!u4`Su z;1x1qWb>$>fLj8@3Fc`~n+e8;MzFuu^f%k9Ps^w~OAPCKZ)b)&qQu#(*vo0{8i)5h z2JQ+o#KE{`4gcSZwI_+iauoG8$<=k)*jpXi;m+&6yUc8*q5k||`_3sjA9>w)*|&Qz zra*BXCx9%OC#dX1cgC2^x}*~n7!d>;XkiMRWAQarQ?Fp62mv?3h3_a^p$K+TL690H z%pkiRxV{JDiHIjbgU6_g*!A=lsvi)39FZ748b+C@f($k|F{~uhw=!?s*X{#-}%x^oFRSz6?}+aPNtaVJXrYk?(a3y5TwDTu&EA+a%G`c z``sw22T~h=DAX{j2i(P!!>O$ZhXe4f8Aj;xX+(!wIyAz8YOgp|XN-Uk*UaPvXi$zw zuM&CGl z3~g=6H#P;kwhM^W^Ki>5NZn{$3UIG^{yZ<9f3JX+a`S{}-^X*n@M3xzlFavj^zCnr zXVHaGjWpV);wm;1CK3zf{2yKDLg*e!j!yLCNi#&fAp7pTdni0iS@qXMHJbG}fF2>$ zk&z#*MrVgcTu=%mQX*((K-0r(Q*{_VjQlg$�qOp^-`;*kv@0NTsJD{EenRzX%mq z_Q#ZpQ9Y!!Qc~#m7cq|l1d8mg-Pyx?TX*O5pjiXlMM15l)y z0@xiP3xUIe*+;32nKMKv3g0$Xp+rf;tG2Fg_nn_p);l~+KF6TIhesfwPX0sX^B`v$AQviQKsUA6#A(-I4Wu@IeEpBi$C6f z@WhzcUu{xZKM=HdLF;lUbA_N@PN{ygqsD%hujj;K)COnpm#6qpfWW%w(9~dwyWSko znb<4~M7`^!F!z^jC-%kq@AyaI`+84H03%nKXyVfia{RY&#MB z@14m%q@(#CBQ9_8(^fZIH`t)>Sq_)fan%?Ebbsm$!^w;gez_hPHK3WKEKni8^<*ZN z)(gd!OX_!%=cwXyV6>N=kNZdHX*8$MRT~?oWK!ylNhhA{r5wDeKedw3Rtu7Jkxoc2 zP2`44;;j((i^#CM{V|3%HXn&wxcCB#Ox)$=UYB6Nw9mGcV-9NIrnOVAYv3f+n7r2I z=J%`2mmdk4!tYQlY@PmoDDuWR(I6a6M3Si+s<&X0QxC?`)(w}J_H2^DcJ!IqY}_+M zD%?dn&+6(LsPi#!oCVaN+X6jgH`*{)%xH#dho z*&C_mt`lE5Z!T)9sAAFpP#05^3LH+SKJJI-#^1O1M>5Y8F574h?DCSaN7N@O*S7vR zdZg{)uJ79y#P(g-t)6kl6ea+4!?A&F3^b!Dwa!$TuV#WJrZ49Gamf5?l@-^*|x!F z5-Z5lY>xb!C*SR#u&9|5Y2es z#y7!VyEgT1^3QhCHMlr#B{QatSyfB}V+PzxZ0s*8KGFy_D{VV>*~Z=Kn&%=15;DBQ z|9I^mM5;z%!R^UZ#>CKBbt6q6RdB{uVX;7--@g@@>lOrf~|- zP0O)}H_9+7KKci(Uf`L?e}DUG)!6C0U78^wD>G_CH)6Dk`BPP4{n}kG`=``!W?{mA zL?ac8GIEdp>Emg*f6Uzygn4w;YWcL8)mdMNS`1lFXAgNOhs(tsf6;NMaH6_QP+zW< z?C1{jrnTy;;t#}gzEg|oOT=9Mm*!1>9YOhX-v;Ue@zoP}Q7w5OalNwVlTpWyFXH_* z_3)05sV#i+WvJskIz8kcjpx5=EBrXR&U>xcIo$UPyv2xhs2EAf{- b5h&jCIDg7=Q?5G;|5>cvXqvp*A>{u7>CV+M literal 0 HcmV?d00001 diff --git a/docs/localization.md b/docs/localization.md index 2f9e98a676..96822877a3 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -20,7 +20,7 @@ Whole sentences that the layers below word — what the accessibility checker fo The Ukrainian catalog has 2,530 entries, written from the meaning of the English labels using Ukrainian computing vocabulary. Commands use infinitives; fixed-count messages use count-neutral wording. Plural-aware messages use the three [CLDR Ukrainian integer forms](https://www.unicode.org/cldr/charts/48/supplemental/language_plural_rules.html#uk): `1 сторінка`, `2 сторінки`, `5 сторінок`; 11–14 take the third form, while 21 and 22 return to the first and second. Calendar dates use genitive month names (`9 жовтня 2026`), while headings use standalone names (`Жовтень 2026`). Home recommendation cards grow to fit translated text; the toolbar search hint truncates before the shortcut, with the full label available on hover. -The Lithuanian catalog has 2,798 entries and three integer plural forms. Calendar headings use standalone month names; dates use genitive forms. Calendar, inspector and password-dialog UI regressions check dynamic labels and preserve document values. A native Lithuanian-speaking contributor has reviewed the translation. +The Lithuanian catalog has 2,798 entries and three integer plural forms. Calendar headings use standalone month names; dates use genitive forms. Calendar, inspector and password-dialog UI regressions check dynamic labels and preserve document values. A native Lithuanian-speaking contributor has reviewed the translation. The Scan & OCR review strings added afterwards (M10: the OCR Verify screen, the batch options and the engine choice) were written without native review. The Vietnamese catalog has 2,798 entries and one plural form (`plural_none`), covering the current catalog-key union. Preferences theme `Appearance` uses context `theme` → `Giao diện`; object/comment/signature appearance stays plain `Diện mạo`. Form fields use `Trường`; redaction apply wording states permanent removal. Calendar months are numeric (`Tháng 1`…`12` / `tháng 1`…`12`). A native Vietnamese-speaking contributor has reviewed the translation. Status stays `partial`: no translated help, and engine/OS details stay English inside a translated frame. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index f71546b731..ec144d3a07 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6683,7 +6683,7 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize", "ocr_status"] -evidence = ["crates/ocr/src/lib.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/lib.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/lib.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/lib.rs::model_discovery_keeps_legacy_only_installations", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] +evidence = ["crates/ocr/src/ocrs.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/ocrs.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/ocrs.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/ocrs.rs::model_discovery_keeps_legacy_only_installations", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] notes = "ocrs engine (MIT/Apache) with CC-BY-SA ONNX models fetched and SHA-256 checked by `cargo xtask models`; within each search directory, a complete ONNX pair takes precedence over a complete legacy RTEN pair, incomplete and mixed-format pairs are not combined, and directory precedence is unchanged; the desktop release packages ship them (#103; packaging/env.sh stage_models), the web build does not yet; runs on a worker thread with progress; one undoable step; pages with text are skipped unless asked (ocr.skip-renderable-text)." [[feature]] @@ -6708,7 +6708,7 @@ status = "shipped" commands = ["ocr.recognize_batch"] tools = ["ocr_recognize_files"] evidence = ["crates/automation/tests/it/automation.rs::ocr_recognize_files_writes_searchable_copies", "crates/ui-egui/tests/it/ocr.rs::recognize_text_in_multiple_files_writes_searchable_copies"] -notes = "Pick PDFs, then an output folder; copies are saved incrementally under the same names, in the background with progress." +notes = "Pick PDFs, then an output folder (the last folder is remembered and reused; Clear asks again); copies are saved incrementally under the same names, in the background with progress. Output names are pre-flighted (a collision aborts before the first page is read), files that already have text are skipped whole only when that option is on (skip_text_files, off by default; Force OCR overrides), and the summary counts recognized words, the skipped bucket, per-file failures, low-confidence notes and what a cancel left unrun." [[feature]] id = "ocr.ocr-languages" @@ -6719,8 +6719,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] -notes = "Language choice exists, but the models read only the Latin alphabet without accents (English)." +evidence = ["crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text", "crates/ocr/src/tesseract.rs::language_table_maps_and_falls_back"] +notes = "The choice feeds both engines: ocrs reads only unaccented Latin (its model table holds English); tesseract maps 12 allow-listed codes (the language enum comes from the engine's own table) but only as far as the installed language packs cover. Recognition quality beyond the tested English set is unproven." [[feature]] id = "ocr.searchable-image" @@ -6731,8 +6731,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] -notes = "Invisible text (Tr 3) fitted to each word box; the image is not deskewed or downsampled (exact mode only)." +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/engine/src/ocr.rs::searchable_layer_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::preprocessing_flags_flow_through_the_job"] +notes = "Invisible text (Tr 3) fitted to each word box. The page's own image object is left untouched; the raster the engines read can be cleaned first (opt-in rotate/deskew/denoise/binarize), with word boxes mapped back to the original raster. Words beyond WinAnsi are written in Type3 fonts with empty character programs and a ToUnicode CMap, so extraction reads them back while nothing needs a display font (no craft-fonts input, and no character is ever substituted with '?'; those substitutions belong to the editable output). Missing: the downsampled Searchable Image variant (ocr.ocr-downsample)." [[feature]] id = "ocr.searchable-image-exact" @@ -6753,9 +6753,9 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["edit.edit_text"] -tools = ["text_edit"] -evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once"] +commands = ["edit.edit_text", "ocr.correct"] +tools = ["text_edit", "ocr_recognize"] +evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once", "crates/engine/src/ocr.rs::editable_writer_refuses_empty_and_writes_visible_words", "crates/engine/src/ocr.rs::editable_output_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::editable_writer_maps_rotated_pages", "crates/automation/tests/it/automation.rs::ocr_recognize_editable_output_writes_visible_text"] notes = "On demand, per edit: in Edit text & images, recognised text (the invisible OCR layer over a scan) is grouped into phrases (words across a word space; a wider gap such as a table cell, and each scanned line, stay separate) and opens for editing on a click. Applying draws the new text visibly in the scan's sampled ink colour (or the chosen one) and first covers the old words with the paper colour sampled around them, so the scanned word no longer shows; when an image or form is drawn after the OCR layer over where the edit lands (its old box, moved, rewrapped or grown to more lines), or an inline image or shading after it (a layer written under its scan, #3973), the cover and new text are drawn on top of the page instead, so they show; the phrase keeps each word's position and width and grows rightwards instead of wrapping. Turning Edit text & images on for a scanned page (an image covering at least half of a page that has no text of its own, once per page per window; comments and text boxes over it do not count as text, a page with real text and a background picture is left alone; one undoable \"Recognize text\" step) now recognizes its words first, as Acrobat's \"Converting scanned page to Editable Text and Images\" does (English only until more OCR languages exist, gap 7). Still missing: separating the photographs on the page from the lettering, matching the scanned typeface (the layer's Helvetica is used), covering on a textured or patterned background (one flat colour is used), a scan painted by a pattern fill or a filled rectangle over a layer written under it, and, for text drawn on top, the clip, optional content and marked content (tags) that enclosed the OCR layer." [[feature]] @@ -6779,8 +6779,8 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/engine/src/tests.rs::recognize_text_reads_a_picture_on_a_page_that_has_text", "crates/automation/tests/it/automation.rs::ocr_recognize_reads_pictures_on_pages_with_text", "crates/ui-egui/tests/it/ocr.rs::recognize_text_can_read_pictures_on_pages_with_text"] -notes = "Pages that already have text are skipped and reported by default. The dialog's \"Also recognize pages that already have text\" (skip_text_pages false, also for multiple files) reads pictures on such pages (#1028); a recognised word at least half covered by the page's own text (an earlier OCR layer included) is left out and counted (already_on_page). Missing: recognising only the image areas (the whole page is still read, so it takes as long as a scan), and a word in a picture that sits under real text is treated as that text." +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/engine/src/tests.rs::recognize_text_reads_a_picture_on_a_page_that_has_text", "crates/automation/tests/it/automation.rs::ocr_recognize_reads_pictures_on_pages_with_text", "crates/ui-egui/tests/it/ocr.rs::recognize_text_can_read_pictures_on_pages_with_text", "crates/automation/tests/it/automation.rs::ocr_recognize_files_writes_searchable_copies"] +notes = "Text pages are skipped by default; skip_text_pages false or Force OCR reads pictures on them while omitting words overlapping existing text (already_on_page). Batch skip_text_files reports skipped files without writing them; Force OCR overrides both skips. Recognition still reads the whole page, and image text under real text can be omitted." [[feature]] id = "ocr.rotation-detection" @@ -6788,11 +6788,11 @@ title = "Detect and fix page rotation" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/ocr/src/preprocess.rs::quarter_turns_are_detected_corrected_and_mapped_back", "crates/ocr/src/preprocess.rs::realistic_letter_pages_are_left_upright"] +notes = "Opt-in preprocessing: 0/90/180/270 detected from projection-profile banding (conservative: under 32 px, blank, or without a clear margin it is a no-op); word boxes map back to the scanned raster through the exact inverse. Correction is proven at the preprocessing layer and the flag flows through the job (needs the models), but correction of a genuinely rotated page through the workflow is not yet covered by a cited end-to-end test." [[feature]] id = "ocr.deskew" @@ -6800,11 +6800,11 @@ title = "Deskew" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/ocr/src/preprocess.rs::skew_is_estimated_and_straight_pages_are_left_alone", "crates/ocr/src/preprocess.rs::deskew_inverse_maps_off_centre_points_back_onto_the_source_line"] +notes = "Opt-in: skew estimated over +/-5 degrees by a shear-scored projection profile; conservative no-ops (under 64 px, blank, below 0.1 degrees, no clear winner); boxes map back off-centre-exact through the composed inverse. Correction is proven at the preprocessing layer; a cited end-to-end test of a skewed page through the workflow is still owed." [[feature]] id = "ocr.despeckle" @@ -6813,9 +6813,9 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["ocr.enhance_camera", "ocr.enhance"] -tools = ["page_enhance_camera", "page_enhance_scan"] -evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] +commands = ["ocr.enhance_camera", "ocr.enhance", "ocr.recognize"] +tools = ["page_enhance_camera", "page_enhance_scan", "ocr_recognize"] +evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did", "crates/ocr/src/preprocess.rs::denoise_removes_specks_and_keeps_dimensions"] notes = "Camera images, and scanned pages through Enhance scanned file (ocr.enhance, page_enhance_scan despeckle): isolated dark pixels (at most one dark neighbour) turn white, on grey, RGB and 1-bit scans, in place (the OCR text layer, comments and links stay). Missing: as an option of Recognize text; specks larger than two pixels; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans and scans inside form XObjects (reported, not changed)." [[feature]] @@ -6896,11 +6896,11 @@ title = "Correct recognized text (low-confidence suspects)" area = "I" tier = "P2" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.correct"] +tools = ["ocr_words", "ocr_review_apply", "ocr_dictionary"] +evidence = ["crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_uses_recognition_language_instead_of_interface_locale", "crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_limit_and_normalization_failures_preserve_file_and_review", "crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_undo_persists_and_failed_undo_keeps_history", "crates/ui-egui/src/workflows/ocr_verify.rs::rereading_keeps_captured_language_when_the_draft_changes", "crates/automation/tests/it/automation.rs::ocr_review_corrections_keep_boxes_are_guarded_and_apply_once", "crates/automation/tests/it/automation.rs::ocr_dictionary_updates_are_atomic_bounded_and_confined", "crates/automation/tests/it/automation.rs::ocr_retained_reviews_have_eight_document_cap_and_close_releases_slot", "crates/ui-egui/src/workflows/ocr_verify.rs::accept_zero_words_fails_and_a_live_review_applies_once", "crates/ui-egui/src/workflows/ocr_verify.rs::legend_rows_carry_the_classifier_thresholds", "crates/ui-egui/src/workflows/ocr_verify.rs::suggestions_are_near_dictionary_and_page_words", "crates/ui-egui/tests/it/ocr.rs::ocr_verify_screen_controls_keyboard_and_accept", "crates/ui-egui/src/workflows/ocr_verify.rs::accepting_pages_one_by_one_then_the_review_applies_every_page_once", "crates/ui-egui/src/workflows/ocr_verify.rs::rereading_a_region_remaps_marks_and_selects_the_first_new_word"] +notes = "The OCR Verify screen: page list, source page with confidence-coloured word boxes (click-select, drag a region), recognised-text preview, zoom + word inspector; corrections commit on Enter (an empty commit removes), Remove/Restore, +Dictionary (a per-language user file), per-word Verified, suggestions within edit distance 2; F8/Shift+F8 walk the low-confidence suspects, Ctrl+T marks the page reviewed, Ctrl+Tab moves the pane focus; Re-OCR of a region or a page; Accept/Reject entire page; a real Cancel keeps the pages already read; a stale session is refused with its reason; review edits are undoable in the panel; Accept (searchable) is one undoable step through the guard path, and pages accepted one by one each apply once (the guard follows each accepted page); a re-read keeps every review mark on the word it was made for. Headless: ocr_words (retain_review) returns the words and a review handle, ocr_review_apply applies corrections at the stored boxes (an empty one removes the word) through the same stale-document guard, or discards the review, and ocr_dictionary edits the dictionary file. Still missing: suggestions, per-word Verified marks and region re-reads are UI-only, and Acrobat's suspect walk across a whole saved document (outside one review) is not offered." [[feature]] id = "ocr.adaptive-compression" @@ -6932,11 +6932,11 @@ title = "Pluggable OCR engines (ocrs default; tesseract optional)" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/engine/src/ocr.rs::selector_names_the_missing_engine", "crates/ocr/src/tesseract.rs::recognize_runs_the_installed_binary", "crates/ocr/src/merge.rs::rover_vote_always_runs_both_and_degrades_without_a_secondary", "crates/ocr/src/merge.rs::confidence_weighted_runs_the_secondary_on_an_unknown_primary", "crates/ocr/src/merge.rs::an_unknown_primary_word_yields_only_to_a_confident_secondary", "crates/ocr/src/tesseract.rs::an_invalid_configured_program_fails_closed", "crates/ocr/src/tesseract.rs::list_langs_come_from_stderr_when_stdout_has_none", "crates/engine/src/ocr.rs::automatic_defaults_to_the_confidence_weighted_strategy"] +notes = "ocrs (pure Rust, MIT/Apache, CC-BY-SA models fetched by `cargo xtask models`) is the default; an installed tesseract binary runs as an external process (fixed argv, allow-listed language codes, private temp dir, timeout and output cap, a user-validated PDFCRAFT_TESSERACT path or the PATH); Automatic is an ensemble (ROVER and confidence-weighted merge; a failing partner degrades to the primary with a note). Partial, because part of the evidence is conditional: `recognize_runs_the_installed_binary` runs only where tesseract is installed and skips elsewhere, so a build machine without it proves nothing about the process path, the absent-binary paths are exercised only on machines without tesseract, and the process-level tests (timeout, output cap, minimal environment) are Unix-only (fake scripts). What always runs: location validation and the fail-closed rule, the --list-langs parsing (stdout or stderr), the engine selector, the merge rules and the default strategy. Automatic defaults to the confidence-weighted strategy (Primary only would never run the second engine)." [[feature]] id = "ocr.ocr-web" diff --git a/xtask/src/license_check.rs b/xtask/src/license_check.rs index c845101db6..0231b4e853 100644 --- a/xtask/src/license_check.rs +++ b/xtask/src/license_check.rs @@ -54,7 +54,7 @@ fn validate(root: &Path) -> Result<()> { let docs: Documents = toml::from_str(&text).context("parsing LICENSES.toml")?; let attribution = assets::load(root)?; let mut problems = check(root, &docs, &attribution); - let ocr_source = std::fs::read_to_string(root.join("crates/ocr/src/lib.rs")).context("reading OCR model filename constants")?; + let ocr_source = std::fs::read_to_string(root.join("crates/ocr/src/ocrs.rs")).context("reading OCR model filename constants")?; problems.extend(check_required_ocr_metadata(&ocr_source, &attribution)?); if !problems.is_empty() { bail!("source licensing errors:\n{}", problems.join("\n")); @@ -72,7 +72,7 @@ fn check_required_ocr_metadata(source: &str, attribution: &assets::Manifest) -> let file = declarations.first().and_then(|value| value.strip_suffix("\";")); let Some(file) = file.filter(|file| declarations.len() == 1 && safe_path(file) && !file.contains('/')) else { bail!( - "OCR constant {name}: expected one literal filename declaration in crates/ocr/src/lib.rs; update the source validator if its syntax changed" + "OCR constant {name}: expected one literal filename declaration in crates/ocr/src/ocrs.rs; update the source validator if its syntax changed" ); }; let entries: Vec<_> = attribution.fetched.iter().filter(|f| f.file == file).collect(); @@ -407,7 +407,7 @@ mod tests { #[test] fn required_ocr_models_cannot_be_missing() { - let source = include_str!("../../crates/ocr/src/lib.rs"); + let source = include_str!("../../crates/ocr/src/ocrs.rs"); let original = assets::load(&assets::root()).unwrap(); assert_eq!(original.fetched.iter().filter(|f| f.kind == "model").count(), 2); assert!(check_required_ocr_metadata(source, &original).unwrap().is_empty()); @@ -420,7 +420,7 @@ mod tests { #[test] fn required_ocr_models_cannot_be_misclassified() { - let source = include_str!("../../crates/ocr/src/lib.rs"); + let source = include_str!("../../crates/ocr/src/ocrs.rs"); let original = assets::load(&assets::root()).unwrap(); assert_eq!(original.fetched.iter().filter(|f| f.kind == "model").count(), 2); for model in original.fetched.iter().filter(|f| f.kind == "model") { diff --git a/xtask/src/screenshots.rs b/xtask/src/screenshots.rs index 243d8b0bad..521d793f48 100644 --- a/xtask/src/screenshots.rs +++ b/xtask/src/screenshots.rs @@ -39,6 +39,7 @@ pub const SCENES: &[(&str, &str, &str, &[&str])] = &[ "README: layouts", &["--layout", "two-up", "--mode", "read", "--page", "4", "--theme", "dark", "--notice", "off"], ), + ("pdfcraft-ocr-verify", "OCR Verify screen", "README: OCR review", &["--size", "1680x1000", "--ocr-verify", "on"]), ]; const PDF: &str = "dist/demo/pdfcraft-showcase.pdf"; @@ -109,7 +110,10 @@ fn update_attribution(root: &Path) -> Result<()> { // The home screen shows the ArtCraft mark: say so (it is a Storyteller trademark, not open // source; AGENTS.md §1.2), so the screenshot's licence is not read as covering it. let shows_home = scene.is_some_and(|(.., args)| args.windows(2).any(|w| w[0] == "--home" && w[1] == "on")); - let contents = if shows_home { + let shows_ocr = scene.is_some_and(|(.., args)| args.windows(2).any(|w| w[0] == "--ocr-verify" && w[1] == "on")); + let contents = if shows_ocr { + "Shows only PdfCraft UI, its Lucide icons and a synthetic scanned page built from the embedded fonts, with a real recognition (the OCR models) loaded into the Verify screen" + } else if shows_home { "Shows PdfCraft UI, Lucide icons, the showcase PDF built from the [[fetched]] OFL fonts, and the ArtCraft mark (Storyteller trademark from docs/brand/, not covered by this licence)" } else { "Shows only PdfCraft UI, Lucide icons and the showcase PDF built from the [[fetched]] OFL fonts" From ce3857c2e5ec61dd6a5cf8565c7e699acc7618a7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 20:55:45 +0000 Subject: [PATCH 62/85] sign: classify changes by signed object role Judge each object changed after signing by the role it held in the signed revision, keyed by object number, and compare a slot's accepted generations across revisions. Adds `Document::accepted_generations` to cos. Co-Authored-By: Claude Opus 5.5 --- crates/cos/README.md | 4 +- crates/cos/src/document/objects.rs | 24 ++ crates/cos/tests/it/generations.rs | 35 +++ crates/sign/src/pdf.rs | 3 + crates/sign/src/pdf/changes.rs | 168 ++++++++------ .../src/pdf/revision_classification_tests.rs | 207 ++++++++++++++++++ crates/sign/tests/it/main.rs | 1 + .../sign/tests/it/revision_classification.rs | 158 +++++++++++++ 8 files changed, 536 insertions(+), 64 deletions(-) create mode 100644 crates/sign/src/pdf/revision_classification_tests.rs create mode 100644 crates/sign/tests/it/revision_classification.rs diff --git a/crates/cos/README.md b/crates/cos/README.md index 774f2c4367..9f5fb56725 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -62,7 +62,9 @@ another. Edits never touch the original bytes. not expand compact xref ranges; the winning xref entry remains authoritative for free slots. Compressed objects require generation zero. Explicit `set` overlays accept only the supplied generation; `set_existing` and `update_dict` retain both accepted generations of a - damaged live slot, including through snapshots, and require full rewrite. Headers beyond + damaged live slot, including through snapshots, and require full rewrite. + `accepted_generations(num)` lists the generations a slot accepts, so code comparing two + revisions can tell whether a reference still reaches the same slot. Headers beyond this bounded probe are not additionally recovered. Materialized primary and hybrid rows are diagnosed before folding, so a newer row does not hide a retained disagreement. - **Tolerant, never fatal.** The lexer works on a byte slice and a position; it allocates nothing diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 8d9917d749..458840de3a 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -175,6 +175,30 @@ impl Document { } } + /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), + /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged + /// file slot (or a repair-aware edit of one) also the generation its object header carries. + /// Code comparing revisions (signature validation) uses it to tell whether the references + /// that reached an object in one revision still reach the same slot in another. + pub fn accepted_generations(&self, num: u32) -> Vec { + // Mirrors `reference_matches`, which stays allocation-free for every lookup. + let mut generations = match self.overlay.get(&num) { + Some(Slot::Set(generation, _)) => vec![*generation], + Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], + Some(Slot::Freed(_)) => Vec::new(), + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() + } + Some(XrefEntry::InStream { .. }) => vec![0], + None | Some(XrefEntry::Free { .. }) => Vec::new(), + }, + }; + generations.sort_unstable(); + generations.dedup(); + generations + } + pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { if !self.reference_matches(r) { return Ok(Arc::new(Object::Null)); diff --git a/crates/cos/tests/it/generations.rs b/crates/cos/tests/it/generations.rs index 3d56628c93..bbe2ff0b33 100644 --- a/crates/cos/tests/it/generations.rs +++ b/crates/cos/tests/it/generations.rs @@ -473,3 +473,38 @@ fn restoring_explicit_sets_removes_old_generation_aliases() { } } } + +/// `accepted_generations` lists exactly the generations `get` resolves, for every kind of slot: +/// a clean file slot, a damaged one (row and header disagree), a repair-aware edit, a plain +/// edit, a freed slot, a free row, an object stream member and an unknown number. +#[test] +fn accepted_generations_are_the_generations_get_resolves() { + let agrees = |doc: &Document, num: u32, expected: &[u16]| { + assert_eq!(doc.accepted_generations(num), expected, "object {num}"); + for generation in 0..8 { + let live = !matches!(*doc.get(ObjRef::new(num, generation)), Object::Null); + assert_eq!(live, expected.contains(&generation), "object {num} generation {generation}"); + } + }; + let doc = Document::open(contradictory_entry(false)).unwrap(); + agrees(&doc, 1, &[0]); + agrees(&doc, 3, &[0, 1]); + agrees(&doc, 9, &[]); + let mut repaired = doc.clone(); + repaired.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + agrees(&repaired, 3, &[0, 1]); + let mut narrowed = doc.clone(); + narrowed.set(ObjRef::new(3, 2), Object::Int(99)); + agrees(&narrowed, 3, &[2]); + let mut freed = doc.clone(); + freed.free(ObjRef::new(3, 0)); + agrees(&freed, 3, &[]); + agrees(&Document::open(contradictory_entry(true)).unwrap(), 3, &[]); + let mut doc = + Document::open(object_file(&[(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")])).unwrap(); + let member = doc.add(Object::Dict(Default::default())); + doc.update_dict(ObjRef::new(1, 0), |c| c.set(b"Extra".to_vec(), Object::Ref(member))).unwrap(); + let packed = Document::open(Arc::new(write_full(&doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap(); + assert!(matches!(packed.xref_entry(member.num), Some(XrefEntry::InStream { .. })), "the member is in an object stream"); + agrees(&packed, member.num, &[0]); +} diff --git a/crates/sign/src/pdf.rs b/crates/sign/src/pdf.rs index 75b42f9905..8fafff7f01 100644 --- a/crates/sign/src/pdf.rs +++ b/crates/sign/src/pdf.rs @@ -45,6 +45,9 @@ use validate::{DssEvidence, MAX_DSS_BYTES, MAX_DSS_ENTRIES}; #[cfg(test)] #[path = "pdf_discovery_tests.rs"] mod discovery_tests; +#[cfg(test)] +#[path = "pdf/revision_classification_tests.rs"] +mod revision_classification_tests; /// One signature field. #[derive(Clone, Debug)] diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..81467542e6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -7,7 +7,11 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< let Some(old) = old else { return Modification::Disallowed(vec![kind::UNREADABLE]); }; - let old_content: HashSet = pdfcraft_annot::page_refs(&old) + // Every role below is keyed by object number, not by the reference as written: a damaged + // file's slot accepts two generations (#1143), and a page signed as `3 5 R` and edited as + // `3 0 R` is still that page. The loop over changed objects refuses any slot whose accepted + // generations changed, so a number names the same slot in both revisions. + let old_content: HashSet = pdfcraft_annot::page_refs(&old) .unwrap_or_default() .into_iter() .flat_map(|pg| { @@ -21,24 +25,34 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< _ => Vec::new(), } }) + .map(|r| r.num) .collect(); // The signed revision's document-level XMP stream. - let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))); + let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))).map(|r| r.num); // The signed revision's XFA data packets: filling the form rewrites them. let old_xfa_data = xfa_data_packets(&old); // What each object was in the signed revision: its pages, and its trailer /Info. A changed // object is judged by the role it held when signed, never by the labels or keys its new, // attacker-written version carries (relabelling a signed page /Type /Sig, or dropping its // /Type and adding /ModDate, must not reclassify it away from page content). - let signed_pages: HashSet = pdfcraft_annot::page_refs(&old).unwrap_or_default().into_iter().collect(); + let signed_page_refs = pdfcraft_annot::page_refs(&old).unwrap_or_default(); + let signed_pages: HashSet = signed_page_refs.iter().map(|r| r.num).collect(); // The signed pages' /Annots arrays that are objects of their own. Adding an annotation updates // such an array in place (the page dictionary does not change), so it is judged like a // page's inline /Annots: by the annotations it gained. - let signed_annots: HashSet = - signed_pages.iter().filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))).collect(); + let signed_annots: HashSet = signed_page_refs + .iter() + .filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))) + .map(|r| r.num) + .collect(); + // The annotations the signed pages listed. One of them listed again elsewhere was moved or + // copied there, not added. + let placed_annots: HashSet = + signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); // What each signed page draws from, by the role it held when signed (#1787). - let signed_resources = signed_page_resources(&old, &signed_pages); - let old_info = old.trailer().get(b"Info").and_then(Object::as_ref); + let signed_resources = signed_page_resources(&old, &signed_page_refs); + let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. @@ -61,8 +75,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // while every comparison here finds the signed object untouched. No writer has a reason to // add such an entry after signing: it is never a permitted change. for (_, num) in doc.xref_oddities().iter().filter(|o| !old.xref_oddities().contains(o)) { - let r = ObjRef { num: *num, generation: doc.generation(*num) }; - let what = if old_content.contains(&r) || signed_pages.contains(&r) { kind::PAGE_CONTENT } else { kind::OTHER }; + let what = if old_content.contains(num) || signed_pages.contains(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); } @@ -80,6 +93,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + placed_annots: &placed_annots, signed_resources: &signed_resources, old_info, dss: &dss, @@ -88,6 +102,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< if same_place(num) { continue; } + // The signed revision's references reached this slot by the generations it accepted. If + // it accepts others now, those references may reach nothing here while another reader + // follows them back through /Prev to an intermediate revision, or to another alias, and + // shows what that holds: the object compared below is not provably what is displayed. + // No writer changes a signed object's generation. + let signed_generations = old.accepted_generations(num); + if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; + if !disallowed.contains(&what) { + disallowed.push(what); + } + continue; + } let r = ObjRef { num, generation: doc.generation(num) }; let new = doc.get(r); let before = old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)); @@ -124,9 +151,8 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // rewritten by every save and is not a change. let current: HashSet = doc.object_numbers().into_iter().collect(); for num in old.object_numbers().into_iter().filter(|n| !current.contains(n)) { - let r = ObjRef { num, generation: old.generation(num) }; - let was = old.get(r); - let signed_role = old_content.contains(&r) || signed_pages.contains(&r) || signed_resources.contains_key(&r); + let was = old.get(ObjRef { num, generation: old.generation(num) }); + let signed_role = signed_role(num); // A signed page's content or resource is judged by that role, whatever /Type its stream carried. if !signed_role && matches!(&*was, Object::Stream(s) if matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))) { continue; @@ -142,13 +168,14 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, - old_content: &'a HashSet, - old_metadata: Option, - old_xfa_data: &'a HashSet, - signed_pages: &'a HashSet, - signed_annots: &'a HashSet, - signed_resources: &'a HashMap, - old_info: Option, + old_content: &'a HashSet, + old_metadata: Option, + old_xfa_data: &'a HashSet, + signed_pages: &'a HashSet, + signed_annots: &'a HashSet, + placed_annots: &'a HashSet, + signed_resources: &'a HashMap, + old_info: Option, dss: &'a HashSet, } @@ -170,7 +197,7 @@ impl ChangedObjectClassifier<'_> { } fn resource_kind(&self, r: ObjRef, new: &Object, before: Option<&Object>) -> Option<&'static str> { - match (self.signed_resources.get(&r), new) { + match (self.signed_resources.get(&r.num), new) { (Some(false), _) => Some(kind::PAGE_CONTENT), (Some(true), Object::Dict(dict)) if self.keeps_previous_entries(dict, before) => None, (Some(true), _) => Some(kind::PAGE_CONTENT), @@ -183,32 +210,40 @@ impl ChangedObjectClassifier<'_> { } fn is_signed_annotation_array(&self, r: ObjRef, before: Option<&Object>) -> bool { - self.signed_annots.contains(&r) && before.is_some_and(|object| object.as_array().is_some()) + self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: HashSet = - before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - let kept: HashSet = now.iter().filter_map(Object::as_ref).collect(); - let removed: Vec = was.iter().filter(|reference| !kept.contains(reference)).copied().collect(); - if removed.is_empty() { - let added: Vec = now.iter().filter_map(Object::as_ref).filter(|reference| !was.contains(reference)).collect(); - added_annots(self.doc, &added) - } else if removed.iter().any(|reference| self.old.get(*reference).as_dict().is_some_and(|dict| dict.name(b"Subtype") == Some(b"Widget"))) { - // A form field or signature widget taken off the page: no permission covers it. + let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); + self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + } + + /// A signed page's annotations going from `was` to `now`, compared by object number. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + let had: HashSet = was.iter().map(|r| r.num).collect(); + let kept: HashSet = now.iter().map(|r| r.num).collect(); + let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); + let added: Vec = now.iter().filter(|r| !had.contains(&r.num)).copied().collect(); + let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); + // A signed annotation listed on another page was moved or copied there. + let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + if removed.iter().chain(&moved).any(|num| was_widget(*num)) { + // A form field or signature widget taken off its page, or onto another: no permission covers it. kind::OTHER - } else { - // Deleting comments is a comment change (permitted at P 3 only). + } else if !removed.is_empty() || !moved.is_empty() { + // Deleting or moving comments is a comment change (permitted at P 3 only). kind::COMMENTS + } else { + added_annots(self.doc, &added) } } fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { let previous = before.and_then(Object::as_dict); - if self.signed_pages.contains(&r) { + if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. - page_change_kind(self.doc, self.old, dict, previous) - } else if previous.is_some() && Some(r) == self.old_info { + self.page_change_kind(dict, previous) + } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. kind::METADATA } else { @@ -216,17 +251,29 @@ impl ChangedObjectClassifier<'_> { } } + /// A changed object that was a page in the signed revision, judged by what the page shows + /// rather than by any `/Type` its new version carries. Only its annotations may change (see + /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped + /// `/Type`) is a page content change. + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + match before { + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, + Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), + None => kind::PAGE_CONTENT, + } + } + fn stream_change(&self, r: ObjRef, stream: &pdfcraft_cos::Stream, before: Option<&Object>) -> Option<&'static str> { // File plumbing is the stream's role when signed; a later /Type label cannot change it. let was_plumbing = matches!(before, Some(Object::Stream(previous)) if matches!(previous.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))); - if self.old_content.contains(&r) { + if self.old_content.contains(&r.num) { Some(kind::PAGE_CONTENT) } else if before.is_none() || was_plumbing { None - } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r) == self.old_metadata { + } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r.num) == self.old_metadata { // Only the signed catalog's XMP stream is metadata. Some(kind::METADATA) - } else if self.old_xfa_data.contains(&r) { + } else if self.old_xfa_data.contains(&r.num) { // Signed XFA data and state packets are changed by a form fill. Some(kind::FORM_FILL) } else { @@ -238,7 +285,7 @@ impl ChangedObjectClassifier<'_> { /// The streams of `doc`'s XFA form that hold its data and state: the `datasets` and `form` /// packets of the AcroForm's `/XFA` array (name, stream, name, stream, …). A form given as one /// XFA stream holds its template too, and is not listed. -fn xfa_data_packets(doc: &Document) -> HashSet { +fn xfa_data_packets(doc: &Document) -> HashSet { let Some(form) = acroform(doc) else { return HashSet::new() }; let Some(xfa) = form.get(b"XFA").map(|x| doc.resolve(x)) else { return HashSet::new() }; let Object::Array(xfa) = &*xfa else { return HashSet::new() }; @@ -247,6 +294,7 @@ fn xfa_data_packets(doc: &Document) -> HashSet { .iter() .filter(|[name, _]| doc.resolve(name).as_string().is_some_and(|s| matches!(s.to_text().as_str(), "datasets" | "form"))) .filter_map(|[_, packet]| packet.as_ref()) + .map(|r| r.num) .collect() } @@ -442,23 +490,24 @@ fn is_signature_dict(d: &Dict) -> bool { /// properties), where any change changes what the page shows. Resources are the page's own or /// inherited. Bounded by depth and by the number of objects visited; past the bound the rest is /// judged as before. -fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap { +fn signed_page_resources(old: &Document, pages: &[ObjRef]) -> HashMap { const CATEGORIES: [&[u8]; 7] = [b"Font", b"XObject", b"ExtGState", b"ColorSpace", b"Pattern", b"Shading", b"Properties"]; const MAX_VISITED: usize = 50_000; const MAX_DEPTH: u8 = 32; const MAX_INHERITANCE: usize = 64; /// Everything reachable from `o` is content the page draws (a leaf role, overriding a - /// container one reached by another path). A signed page or the catalog reached this way - /// (a `/P` or `/Parent` back-reference) keeps the role of its own: the walk stops there. - fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { + /// container one reached by another path, under any generation alias). A signed page or the + /// catalog reached this way (a `/P` or `/Parent` back-reference) keeps the role of its own: + /// the walk stops there. + fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { if roles.len() >= MAX_VISITED || depth > MAX_DEPTH { return; } let resolved; let o = match o { Object::Ref(r) => { - if stop.contains(r) || roles.insert(*r, false).is_some() { + if stop.contains(&r.num) || roles.insert(r.num, false).is_some() { return; } resolved = old.get(*r); @@ -490,14 +539,14 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap = pages.iter().map(|r| r.num).collect(); + stop.extend(old.root().map(|r| r.num)); for page in pages { let Some(res) = resources(old, *page) else { continue }; - let container = |o: &Object, roles: &mut HashMap| -> Arc { + let container = |o: &Object, roles: &mut HashMap| -> Arc { match o { Object::Ref(r) => { - roles.entry(*r).or_insert(true); + roles.entry(r.num).or_insert(true); old.get(*r) } other => Arc::new(other.clone()), @@ -516,21 +565,6 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap) -> &'static str { - match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => { - let was = refs(old, b.get(b"Annots")); - let added: Vec = refs(doc, d.get(b"Annots")).into_iter().filter(|r| !was.contains(r)).collect(); - added_annots(doc, &added) - } - None => kind::PAGE_CONTENT, - } -} - /// Whether the catalog `d` keeps every developer extension the signed catalog `b` declared, each /// with the same value: it may only declare more. fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { @@ -557,6 +591,14 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) { return kind::PAGE_CONTENT; } + // A signed form field or widget keeps that role: its value, appearance or signature may + // change, but turning it into a comment, another kind of annotation or another type of + // field is no form fill. + if before.is_some_and(|b| { + (b.name(b"Subtype") == Some(b"Widget") || b.contains(b"FT")) && (d.name(b"Subtype") != b.name(b"Subtype") || d.name(b"FT") != b.name(b"FT")) + }) { + return kind::OTHER; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs new file mode 100644 index 0000000000..ed338c24de --- /dev/null +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -0,0 +1,207 @@ +//! Changes made after signing are judged against what a reader resolves and by the role each +//! object held when signed: a newer generation, a repaired generation alias, or a widget +//! replaced, removed or moved never passes as a permitted change. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object}; + +use super::Modification; +use super::changes::classify_changes; + +/// One cross-reference row: an object written with header generation `.0` under row generation `.1`. +enum Row<'a> { + Obj(u16, u16, &'a str), +} + +/// A classic-xref file built one revision at a time; each later revision points back with /Prev. +#[derive(Clone)] +struct File { + bytes: Vec, + size: u32, + xref: Option, +} + +impl File { + fn new() -> Self { + File { bytes: b"%PDF-1.7\n".to_vec(), size: 1, xref: None } + } + + fn revision(mut self, rows: &[(u32, Row<'_>)]) -> Self { + let mut table = Vec::new(); + for (num, row) in rows { + self.size = self.size.max(num + 1); + let Row::Obj(header, generation, body) = row; + table.push(format!("{num} 1\n{:010} {generation:05} n \n", self.bytes.len())); + self.bytes.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + let at = self.bytes.len(); + self.bytes.extend_from_slice(b"xref\n"); + if self.xref.is_none() { + self.bytes.extend_from_slice(b"0 1\n0000000000 65535 f \n"); + } + self.bytes.extend_from_slice(table.concat().as_bytes()); + let prev = self.xref.map(|p| format!(" /Prev {p}")).unwrap_or_default(); + self.bytes.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R{prev} >>\nstartxref\n{at}\n%%EOF\n", self.size).as_bytes()); + self.xref = Some(at); + self + } + + fn open(&self) -> Document { + Document::open(Arc::new(self.bytes.clone())).unwrap() + } +} + +const PAGES: &str = "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"; +const TEXT_FIELD: &str = "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 20 120 40] /P 3 0 R >>"; +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 20 40 40] /Contents (note) >>"; + +fn page(extra: &str) -> String { + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] {extra} >>") +} + +fn r(num: u32, generation: u16) -> ObjRef { + ObjRef { num, generation } +} + +fn assert_refused(doc: &Document, old: Document, p: u8) { + let verdict = classify_changes(doc, Some(old), Some(p)); + assert!(matches!(verdict, Modification::Disallowed(_)), "P={p}: main returns {verdict:?}"); +} + +/// Two pages and a text field on page 3, listed in /Fields and in the page's inline /Annots. +fn form(page3: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(page3))), + (4, Row::Obj(0, 0, &page("/Annots []"))), + (5, Row::Obj(0, 0, TEXT_FIELD)), + (6, Row::Obj(0, 0, "[5 0 R]")), + ]) +} + +#[test] +fn content_stream_intermediate_generation_cover() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(""))), + (4, Row::Obj(0, 0, &page("/Contents 5 0 R"))), + (5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let now = signed + .clone() + .revision(&[(5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nQ q\nendstream"))]) + .revision(&[(5, Row::Obj(1, 1, "<< /Length 3 >>\nstream\nq Q\nendstream"))]); + let (doc, old) = (now.open(), signed.open()); + assert_eq!(doc.revisions().len(), 3); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.get(r(5, 1)), old.get(r(5, 0)), "the newest generation holds the signed bytes"); + assert_refused(&doc, old, 2); +} + +#[test] +fn widget_removed_from_inline_annots() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn widget_moved_between_pages() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + doc.update_dict(r(4, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(r(5, 0))]))).unwrap(); + doc.update_dict(r(5, 0), |w| w.set(b"P".to_vec(), Object::Ref(r(4, 0)))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn annots_moved_to_new_object_dropping_widget() { + let signed = form("/Annots 6 0 R"); + let mut doc = signed.open(); + let empty = doc.add(Object::Array(Vec::new())); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Ref(empty))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_page_relabelled_as_field() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 5 R 4 0 R] /Count 2 >>")), + (3, Row::Obj(5, 0, &page(""))), + (4, Row::Obj(0, 0, &page(""))), + ]); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| { + p.remove(b"Type"); + p.set(b"FT".to_vec(), Object::name("Tx")); + p.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +#[test] +fn resource_container_and_drawn_alias() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Contents 6 0 R /Resources << /Font 4 0 R /ExtGState << /GS1 4 5 R >> >>"))), + (4, Row::Obj(5, 0, "<< /F1 5 0 R /CA 1 >>")), + (5, Row::Obj(0, 0, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), + (6, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let mut doc = signed.open(); + doc.update_dict(r(4, 0), |d| d.set(b"FT".to_vec(), Object::name("Tx"))).unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +/// A field signed as `4 5 R` / `4 0 R` (header 5, row 0) that a later revision rewrites. +fn aliased_field(fields: &str, annots: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, &format!("<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> >>"))), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page(&format!("/Annots [{annots}]")))), + (4, Row::Obj(5, 0, TEXT_FIELD)), + ]) +} + +fn set_annots(doc: &mut Document, annot: ObjRef) { + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annot)]))).unwrap(); +} + +#[test] +fn repaired_alias_set_changes_widget_to_comment() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 0, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_slot_becomes_disjoint_generation() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 7, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn hidden_shared_alias_form_fill() { + let signed = aliased_field("4 5 R", "4 0 R"); + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let now = signed.clone().revision(&[(4, Row::Obj(5, 1, &filled))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 1)); + // At P=3 a relisted annotation is permitted whatever it is: only the generations refuse it. + for p in [2, 3] { + assert_refused(&doc, signed.open(), p); + } +} diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index d526b01e7c..791b2b503b 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -7,4 +7,5 @@ mod cms; mod crypto; mod encrypted_signing; mod pdf; +mod revision_classification; mod store_keys; diff --git a/crates/sign/tests/it/revision_classification.rs b/crates/sign/tests/it/revision_classification.rs new file mode 100644 index 0000000000..9d36c3f57e --- /dev/null +++ b/crates/sign/tests/it/revision_classification.rs @@ -0,0 +1,158 @@ +//! A certification signature stays valid only while what a reader shows is what was signed: +//! revisions appended by hand that reach back to an older generation, or that replace a signed +//! form field in place, are refused. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// One row of a hand-written revision: `num header obj body` under row generation `row`, or a +/// free row with its next generation. +enum Row<'a> { + Obj { num: u32, header: u16, row: u16, body: &'a str }, + Free { num: u32, next: u16 }, +} + +/// `base` with one more revision of `rows`, written as a classic table pointing back with /Prev. +fn append(base: &[u8], rows: &[Row<'_>]) -> Vec { + let doc = open(base); + let size = doc.trailer().get(b"Size").and_then(Object::as_int).unwrap(); + let root = doc.root().unwrap(); + let tail = String::from_utf8_lossy(&base[base.len().saturating_sub(64)..]).into_owned(); + let prev: usize = tail.rsplit("startxref").next().unwrap().split_whitespace().next().unwrap().parse().unwrap(); + let mut out = base.to_vec(); + let mut table = String::new(); + for row in rows { + match row { + Row::Obj { num, header, row, body } => { + table.push_str(&format!("{num} 1\n{:010} {row:05} n \n", out.len())); + out.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + Row::Free { num, next } => table.push_str(&format!("{num} 1\n0000000000 {next:05} f \n")), + } + } + let xref = out.len(); + out.extend_from_slice( + format!("xref\n{table}trailer\n<< /Size {size} /Root {} {} R /Prev {prev} >>\nstartxref\n{xref}\n%%EOF\n", root.num, root.generation) + .as_bytes(), + ); + out +} + +fn opts(certify: u8) -> SignOptions { + SignOptions { + page: 0, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(certify), + ..SignOptions::default() + } +} + +/// The certification signature of `bytes`, which must be refused. +fn assert_refused(bytes: &[u8]) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed).unwrap(); + assert!(matches!(s.modification, Modification::Disallowed(_)) && s.status != Status::Valid, "main returns {:?}, {:?}", s.status, s.modification); +} + +fn certify(unsigned: &[u8], p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + pdfcraft_sign::sign(&open(unsigned), &id, &opts(p)).unwrap() +} + +const PAGE2: &str = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>"; + +/// Revision 2 rotates page 2 (`4 0 obj`); revision 3 writes the signed page back as `4 1 obj`. +/// The page tree still says `4 0 R`. +fn intermediate_generation_cover(p: u8) { + let unsigned = write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>", + PAGE2, + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page one) Tj ET\nendstream", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page two) Tj ET\nendstream", + ]); + let signed = certify(&unsigned, p); + let rotated = PAGE2.replace(" >> >> >>", " >> >> /Rotate 180 >>"); + let two = append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: &rotated }]); + let three = append(&two, &[Row::Obj { num: 4, header: 1, row: 1, body: PAGE2 }]); + let doc = open(&three); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.revisions().len(), open(&signed).revisions().len() + 2); + assert_eq!(doc.get(pdfcraft_cos::ObjRef { num: 4, generation: 1 }), open(&signed).get(pdfcraft_cos::ObjRef { num: 4, generation: 0 })); + assert_refused(&three); +} + +#[test] +fn intermediate_generation_cover_p1() { + intermediate_generation_cover(1); +} + +#[test] +fn intermediate_generation_cover_p2() { + intermediate_generation_cover(2); +} + +#[test] +fn intermediate_generation_cover_p3() { + intermediate_generation_cover(3); +} + +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 200 40 220] /Contents (note) >>"; + +/// A page with text field `4 0 R`, listed in /Fields and in the page's inline /Annots, certified +/// allowing comments. +fn certified_form() -> Vec { + certify( + &write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [4 0 R] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + ]), + 3, + ) +} + +#[test] +fn widget_replaced_in_place_by_comment() { + let signed = certified_form(); + assert_refused(&append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} + +#[test] +fn widget_freed_then_reused_as_comment() { + let signed = certified_form(); + let freed = append(&signed, &[Row::Free { num: 4, next: 1 }]); + assert_refused(&append(&freed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} From de344a4bb1dd02bad3a9cffd13cf69a5ac73292b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 22:44:26 +0000 Subject: [PATCH 63/85] sign: compare annotation lists by what resolves, within a bound An annotation list is compared by the entries that resolve in each revision, and an unchanged list is no change. Expanding annotation lists is bounded per classification. A damaged slot rewritten with fewer accepted generations is the same slot when nothing signed refers to a generation it gave up. Co-Authored-By: Claude Opus 5.5 --- crates/sign/src/pdf/changes.rs | 142 +++++++++++++++--- .../src/pdf/revision_classification_tests.rs | 77 ++++++++++ 2 files changed, 196 insertions(+), 23 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 81467542e6..ef4b8868c6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -1,5 +1,11 @@ use super::types::{Modification, kind}; use super::{Arc, Dict, Document, HashMap, HashSet, ObjRef, Object}; +use std::cell::{Cell, OnceCell}; + +/// The most annotation-list entries one classification expands. Pages sharing one long /Annots +/// array, or many pages rewritten to list one, would otherwise cost pages × entries; past this +/// the lists are not compared, and the change is refused. +const MAX_ANNOT_ENTRIES: usize = 1 << 22; /// What later revisions changed, classified as Acrobat reports it, under DocMDP `p` (or none: /// an approval signature permits form fill, comments and further signatures). @@ -47,13 +53,31 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< .collect(); // The annotations the signed pages listed. One of them listed again elsewhere was moved or // copied there, not added. - let placed_annots: HashSet = - signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); + let annot_budget = Cell::new(MAX_ANNOT_ENTRIES); + let (mut placed_annots, mut expanded, mut annots_unbounded) = (HashSet::new(), HashSet::new(), false); + for pg in &signed_page_refs { + let page = old.get(*pg); + let Some(list) = page.as_dict().and_then(|d| d.get(b"Annots")) else { continue }; + // Pages sharing one array object list the same annotations. + if list.as_ref().is_some_and(|r| !expanded.insert(r.num)) { + continue; + } + let Some(found) = annot_refs(&old, Some(list), &annot_budget) else { + annots_unbounded = true; + break; + }; + placed_annots.extend(found.into_iter().map(|r| r.num)); + } // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); + if annots_unbounded { + disallowed.push(kind::OTHER); + } + // Every reference the signed revision makes, read only when a slot's generations narrowed. + let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. let same_place = |num: u32| -> bool { @@ -94,6 +118,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< signed_pages: &signed_pages, signed_annots: &signed_annots, placed_annots: &placed_annots, + annot_budget: &annot_budget, signed_resources: &signed_resources, old_info, dss: &dss, @@ -106,9 +131,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // it accepts others now, those references may reach nothing here while another reader // follows them back through /Prev to an intermediate revision, or to another alias, and // shows what that holds: the object compared below is not provably what is displayed. - // No writer changes a signed object's generation. + // No writer changes a signed object's generation. A damaged slot rewritten cleanly + // accepts fewer: that is the same slot when nothing signed refers to it by a generation + // it gave up. let signed_generations = old.accepted_generations(num); - if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let generations = doc.accepted_generations(num); + let narrowed_unreferenced = || { + !generations.is_empty() + && generations.iter().all(|g| signed_generations.contains(g)) + && signed_references.get_or_init(|| references(&old)).as_ref().is_some_and(|found: &HashSet| { + signed_generations.iter().filter(|g| !generations.contains(g)).all(|g| !found.contains(&ObjRef { num, generation: *g })) + }) + }; + if !signed_generations.is_empty() && signed_generations != generations && !narrowed_unreferenced() { let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); @@ -174,6 +209,7 @@ struct ChangedObjectClassifier<'a> { signed_pages: &'a HashSet, signed_annots: &'a HashSet, placed_annots: &'a HashSet, + annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, dss: &'a HashSet, @@ -188,8 +224,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { - Object::Array(now) if self.is_signed_annotation_array(r, before) => Some(self.annotation_array_change(now, before)), - Object::Dict(dict) => Some(self.dictionary_change(r, dict, before)), + Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), + Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), _ if before.is_none() => None, _ => Some(kind::OTHER), @@ -213,13 +249,21 @@ impl ChangedObjectClassifier<'_> { self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } - fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + fn annotation_array_change(&self, now: &Object, before: Option<&Object>) -> Option<&'static str> { + match (annot_refs(self.old, before, self.annot_budget), annot_refs(self.doc, Some(now), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + } } - /// A signed page's annotations going from `was` to `now`, compared by object number. - fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + /// A signed page's annotations going from `was` to `now`, compared by object number. Only an + /// entry that resolves is on the page: one rewritten to a generation its slot does not accept + /// draws nothing, whatever its number. `None` when the page shows the same annotations in + /// the same order. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> Option<&'static str> { + let live = + |doc: &Document, list: &[ObjRef]| -> Vec { list.iter().filter(|r| !matches!(*doc.get(**r), Object::Null)).copied().collect() }; + let (was, now) = (live(self.old, was), live(self.doc, now)); let had: HashSet = was.iter().map(|r| r.num).collect(); let kept: HashSet = now.iter().map(|r| r.num).collect(); let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); @@ -227,27 +271,31 @@ impl ChangedObjectClassifier<'_> { let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); // A signed annotation listed on another page was moved or copied there. let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + // The annotations kept, in their new order: another order draws another one on top. + let reordered = !now.iter().map(|r| r.num).filter(|num| had.contains(num)).eq(was.iter().map(|r| r.num)); if removed.iter().chain(&moved).any(|num| was_widget(*num)) { // A form field or signature widget taken off its page, or onto another: no permission covers it. - kind::OTHER - } else if !removed.is_empty() || !moved.is_empty() { - // Deleting or moving comments is a comment change (permitted at P 3 only). - kind::COMMENTS + Some(kind::OTHER) + } else if !removed.is_empty() || !moved.is_empty() || reordered { + // Deleting, moving or restacking comments is a comment change (permitted at P 3 only). + Some(kind::COMMENTS) + } else if added.is_empty() { + None } else { - added_annots(self.doc, &added) + Some(added_annots(self.doc, &added)) } } - fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { + fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> Option<&'static str> { let previous = before.and_then(Object::as_dict); if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. - kind::METADATA + Some(kind::METADATA) } else { - change_kind(self.doc, self.old, dict, previous) + Some(change_kind(self.doc, self.old, dict, previous)) } } @@ -255,11 +303,14 @@ impl ChangedObjectClassifier<'_> { /// rather than by any `/Type` its new version carries. Only its annotations may change (see /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped /// `/Type`) is a page content change. - fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> Option<&'static str> { match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), - None => kind::PAGE_CONTENT, + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => Some(kind::PAGE_CONTENT), + Some(b) => match (annot_refs(self.old, b.get(b"Annots"), self.annot_budget), annot_refs(self.doc, d.get(b"Annots"), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + }, + None => Some(kind::PAGE_CONTENT), } } @@ -439,6 +490,51 @@ fn added_annots(doc: &Document, added: &[ObjRef]) -> &'static str { } } +/// The references in the annotation list `o` (inline or indirect), its length spent from +/// `budget`. `None` past the budget. +fn annot_refs(doc: &Document, o: Option<&Object>, budget: &Cell) -> Option> { + let held; + let items = match o { + Some(Object::Ref(_)) => { + held = o.map(|o| doc.resolve(o)); + held.as_deref().and_then(Object::as_array) + } + other => other.and_then(Object::as_array), + }; + let Some(items) = items else { return Some(Vec::new()) }; + budget.set(budget.get().checked_sub(items.len())?); + Some(items.iter().filter_map(Object::as_ref).collect()) +} + +/// Every indirect reference in `doc`'s objects and trailer. `None` (nothing can be shown +/// unreferenced) when an object can't be read, or past a bound on references or nesting. +fn references(doc: &Document) -> Option> { + const MAX_REFERENCES: usize = 1 << 22; + const MAX_DEPTH: u8 = 64; + fn walk(o: &Object, depth: u8, found: &mut HashSet) -> bool { + if depth > MAX_DEPTH { + return false; + } + match o { + Object::Ref(r) => { + found.insert(*r); + found.len() <= MAX_REFERENCES + } + Object::Dict(d) => d.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Stream(s) => s.dict.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Array(a) => a.iter().all(|v| walk(v, depth + 1, found)), + _ => true, + } + } + let mut found = HashSet::new(); + for num in doc.object_numbers() { + if !walk(&*doc.try_get(num).ok()?, 0, &mut found) { + return None; + } + } + doc.trailer().iter().all(|(_, v)| walk(v, 0, &mut found)).then_some(found) +} + fn refs(doc: &Document, o: Option<&Object>) -> Vec { o.map(|o| doc.resolve(o)).and_then(|a| a.as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() } diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs index ed338c24de..b5e72fcae0 100644 --- a/crates/sign/src/pdf/revision_classification_tests.rs +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -205,3 +205,80 @@ fn hidden_shared_alias_form_fill() { assert_refused(&doc, signed.open(), p); } } + +fn verdict(doc: &Document, old: Document, p: Option) -> Modification { + classify_changes(doc, Some(old), p) +} + +/// A widget whose /Annots entry is rewritten to a generation its slot never had no longer +/// resolves: it is taken off the page, inline or through an indirect array. +#[test] +fn widget_dropped_through_a_dead_generation() { + for (page3, num) in [("/Annots 6 0 R", 6), ("/Annots [5 0 R]", 3)] { + let signed = form(page3); + let body = if num == 6 { "[5 3 R]".to_owned() } else { page("/Annots [5 3 R]") }; + let now = signed.clone().revision(&[(num, Row::Obj(0, 0, &body))]); + for p in [Some(2), Some(3), None] { + let v = verdict(&now.open(), signed.open(), p); + assert!(matches!(v, Modification::Disallowed(_)), "{page3} P={p:?}: {v:?}"); + } + } +} + +/// The same annotations, listed in a new array object, change nothing on the page; listed in +/// another order they change which is drawn on top. +#[test] +fn an_unchanged_annotation_list_is_no_change_and_a_reordered_one_is_a_comment_change() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Annots [4 0 R 5 0 R]"))), + (4, Row::Obj(0, 0, COMMENT)), + (5, Row::Obj(0, 0, COMMENT)), + ]); + let moved = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots 6 0 R"))), (6, Row::Obj(0, 0, "[4 0 R 5 0 R]"))]); + assert_eq!(verdict(&moved.open(), signed.open(), Some(2)), Modification::Allowed(Vec::new())); + let reordered = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots [5 0 R 4 0 R]")))]); + assert_eq!(verdict(&reordered.open(), signed.open(), Some(2)), Modification::Disallowed(vec![super::kind::COMMENTS])); +} + +/// Many signed pages sharing one long /Annots array, all rewritten to list it through a new +/// object, are judged in bounded time. +#[test] +fn shared_annotation_arrays_are_judged_in_bounded_time() { + const PAGES: u32 = 30_000; + let first_page = 10; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let annots: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let mut rows = vec![ + (1, "<< /Type /Catalog /Pages 2 0 R >>".to_owned()), + (2, format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>")), + (3, format!("[{annots}]")), + ]; + rows.extend((0..PAGES).map(|i| (first_page + i, page("/Annots 3 0 R")))); + let rows: Vec<(u32, Row<'_>)> = rows.iter().map(|(n, b)| (*n, Row::Obj(0, 0, b))).collect(); + let signed = File::new().revision(&rows); + let repointed: Vec = (0..PAGES).map(|_| page("/Annots 4 0 R")).collect(); + let mut later: Vec<(u32, Row<'_>)> = repointed.iter().enumerate().map(|(i, b)| (first_page + i as u32, Row::Obj(0, 0, b))).collect(); + let copy = format!("[{annots}]"); + later.push((4, Row::Obj(0, 0, ©))); + let now = signed.clone().revision(&later); + let (doc, old) = (now.open(), signed.open()); + let started = std::time::Instant::now(); + let _ = verdict(&doc, old, Some(3)); + assert!(started.elapsed() < std::time::Duration::from_secs(20), "took {:?}", started.elapsed()); +} + +/// A damaged signed slot (header 5, row 0) rewritten cleanly as `4 0 obj` keeps the generation +/// every signed reference uses: a form fill. Had anything signed referred to it as `4 5 R`, that +/// reference would no longer resolve, and the change is refused. +#[test] +fn a_repaired_slot_rewritten_cleanly_is_judged_by_its_references() { + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert_eq!(verdict(&now.open(), signed.open(), Some(2)), Modification::Allowed(vec![super::kind::FORM_FILL])); + let signed = aliased_field("4 5 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert!(matches!(verdict(&now.open(), signed.open(), Some(2)), Modification::Disallowed(_))); +} From e1a3c033aaf749a758010bd1c9342c426eee2b39 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 6 Nov 2026 00:19:58 +0000 Subject: [PATCH 64/85] sign: under no-changes certification, only invisible document timestamps may follow DocMDP P=1 allowed any change classified as a signature. A signature added after signing now breaks the certification unless it is a document timestamp drawing nothing (no positive-area /Rect, or Hidden/NoView). The added objects are judged directly, so a widget with no /T or one only in a page's /Annots is covered. The document security store and a genuine timestamp still pass. Co-Authored-By: Claude Opus 4.8 --- crates/sign/src/pdf/changes.rs | 51 ++++++++++++++++++++++++ crates/sign/tests/it/pdf.rs | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..3f77fd0554 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -136,9 +136,60 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< disallowed.push(what); } } + // DocMDP P=1 ("no changes allowed", ISO 32000-2 §12.8.2.2) permits only validation data to + // follow: the document security store, and document timestamps added for PAdES long-term + // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an + // approval signature, an unsigned signature widget, or a widget that paints over the signed + // page — is a change the certification forbids, however it is labelled. + if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + allowed.retain(|k| *k != kind::SIGNATURE); + if !disallowed.contains(&kind::SIGNATURE) { + disallowed.push(kind::SIGNATURE); + } + } if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } } +/// Whether every signature object `doc` gained or changed since the signed revision `old` is an +/// invisible document timestamp. A signature annotation (a widget, or a dictionary carrying a +/// signature value or role) that the signed revision did not hold byte for byte must be a +/// `/SubFilter /ETSI.RFC3161` value in a widget that shows nothing — no positive-area `/Rect`, +/// or the Hidden or NoView flag set. An approval signature, an empty signature widget, or a +/// visible one fails, whatever `/Type` or `/SubFilter` it claims. Judged on the objects +/// themselves, not on the `/AcroForm /Fields` listing, so a widget with no `/T` or one reached +/// only through a page's `/Annots` is covered too. +fn only_invisible_timestamps_added(doc: &Document, old: &Document) -> bool { + /// A widget shows nothing: no `/Rect` enclosing a positive area, or Hidden (2) / NoView (32). + fn invisible(doc: &Document, w: &Dict) -> bool { + let flags = w.get(b"F").and_then(|o| doc.resolve(o).as_int()).unwrap_or(0); + let area = + super::listing::nums(doc, w, b"Rect").filter(|r| r.len() == 4).is_some_and(|r| (r[0] - r[2]).abs() > 0.5 && (r[1] - r[3]).abs() > 0.5); + !area || flags & (2 | 32) != 0 + } + for num in doc.object_numbers() { + let new = doc.get(ObjRef { num, generation: doc.generation(num) }); + let Some(d) = new.as_dict() else { continue }; + // A signature object: a signature value, or a field or widget that carries one. + let is_value = matches!(d.name(b"Type"), Some(b"Sig" | b"DocTimeStamp")); + let is_field = d.name(b"FT") == Some(b"Sig") || d.name(b"Subtype") == Some(b"Widget"); + if !is_value && !is_field { + continue; + } + // Unchanged since signing: the certification already covered it. + if old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)).as_deref() == Some(&*new) { + continue; + } + let value = if is_value { Some(d.clone()) } else { d.get(b"V").map(|v| doc.resolve(v)).and_then(|v| v.as_dict().cloned()) }; + // A new or changed signature object passes only as an invisible document timestamp. A + // widget with no value (an empty signature field placed on the page) never does. + let timestamp = value.as_ref().is_some_and(super::validate::is_doc_timestamp); + if !timestamp || (is_field && !invisible(doc, d)) { + return false; + } + } + true +} + struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 869a4bebd5..57efadbaf3 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -351,6 +351,77 @@ fn later_changes_are_classified_under_the_signature_permissions() { assert_eq!((s.certify, s.status), (Some(1), Status::Invalid), "{:?}", s.modification); } +/// Certify `fixture` with "no changes allowed" (DocMDP P=1). +fn certified_p1() -> (DigitalId, TrustStore, Vec) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + (id, trust, signed) +} + +/// Add a signature widget to page 3's /Annots (not to /Fields), with value `v` and rectangle +/// `rect`. +fn add_sig_widget(doc: &mut Document, v: Option<&[(&[u8], Object)]>, rect: [i64; 4]) { + let mut w = Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"Rect".to_vec(), Object::Array(rect.map(Object::Int).to_vec())); + w.set(b"P".to_vec(), Object::Ref(ObjRef { num: 3, generation: 0 })); + if let Some(entries) = v { + let mut value = Dict::new(); + for (k, o) in entries { + value.set(k.to_vec(), o.clone()); + } + let vr = doc.add(Object::Dict(value)); + w.set(b"V".to_vec(), Object::Ref(vr)); + } + let wr = doc.add(Object::Dict(w)); + doc.update_dict(ObjRef { num: 3, generation: 0 }, |pg| { + let mut a = pg.get(b"Annots").and_then(Object::as_array).cloned().unwrap_or_default(); + a.push(Object::Ref(wr)); + pg.set(b"Annots".to_vec(), Object::Array(a)); + }) + .unwrap(); +} + +/// Under P=1, a signature added after signing breaks the certification unless it is an invisible +/// document timestamp. A visible overlay, an unsigned widget, an approval signature and a visible +/// widget wearing a timestamp /SubFilter each break it; a genuine document timestamp does not. +#[test] +fn only_an_invisible_document_timestamp_survives_a_no_changes_certification() { + let (_, trust, signed) = certified_p1(); + let check = |bytes: &[u8]| signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify == Some(1)).unwrap(); + assert_eq!(check(&signed).status, Status::Valid, "{:?}", check(&signed).details); + let approval: &[(&[u8], Object)] = &[(b"Type", Object::name("Sig")), (b"SubFilter", Object::name("adbe.pkcs7.detached"))]; + let timestamp: &[(&[u8], Object)] = &[(b"Type", Object::name("DocTimeStamp")), (b"SubFilter", Object::name("ETSI.RFC3161"))]; + // Each of these breaks the certification: it is a change P=1 does not allow. + for (what, v, rect) in [ + ("visible overlay widget", None, [0, 0, 300, 300]), + ("invisible unsigned widget", None, [0, 0, 0, 0]), + ("visible approval signature", Some(approval), [0, 0, 300, 300]), + ("invisible approval signature", Some(approval), [0, 0, 0, 0]), + ("visible widget wearing a timestamp subfilter", Some(timestamp), [0, 0, 300, 300]), + ] { + let edited = edit_after(&signed, |doc| add_sig_widget(doc, v, rect)); + let s = check(&edited); + assert!(matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::SIGNATURE)), "{what}: {:?}", s.modification); + assert_ne!(s.status, Status::Valid, "{what}"); + } +} + +/// A genuine document timestamp, which draws nothing, is validation data P=1 permits: the +/// certification stays valid. +#[test] +fn a_document_timestamp_keeps_a_no_changes_certification_valid() { + let (_, trust, signed) = certified_p1(); + let tsa = TestTsa { id: tsa_id(), time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 } }; + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let s = signatures(&open(&stamped), &stamped, &trust).into_iter().find(|s| s.certify == Some(1)).unwrap(); + assert_eq!(s.status, Status::Valid, "{:?}", s.details); + assert!(matches!(&s.modification, Modification::Allowed(k) if k.contains(&kind::SIGNATURE)), "{:?}", s.modification); +} + #[test] fn invisible_signatures_and_refusals() { let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); From ddeca878dff1ea6cc2fa55adb2db84513eecda1f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 21:40:30 -0500 Subject: [PATCH 65/85] sign: judge the signed page tree, catalog and form by their signed roles Rebased onto 102cae3b/9afa1266 for landing (#6358). The signed page tree's nodes, a /Kids array of its own, the catalog by object number and an indirect AcroForm keep the roles they had when signed; a field dropped from the form is no permitted change at any level; a signed widget's /Rect or /F is page content. Co-Authored-By: Martin Ball --- crates/sign/src/pdf/changes.rs | 117 ++++++++++--- crates/sign/tests/it/main.rs | 1 + crates/sign/tests/it/signed_structure.rs | 199 +++++++++++++++++++++++ 3 files changed, 295 insertions(+), 22 deletions(-) create mode 100644 crates/sign/tests/it/signed_structure.rs diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 03253b79fb..a44c08bb7c 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -68,14 +68,26 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< }; placed_annots.extend(found.into_iter().map(|r| r.num)); } + // The signed page tree's nodes: a change to one changes the pages, whatever it is labelled + // now. Every node, and every /Kids array object of its own, is read once; past the bound the + // tree is not known in full, and no change is judged against it. + let page_tree = page_tree_nodes(&old); + let page_tree_unbounded = page_tree.is_none(); + let (page_nodes, kids_arrays) = page_tree.unwrap_or_default(); // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + // The signed catalog's /AcroForm, when it is an object of its own. + let old_form = old.root().and_then(|c| old.get(c).as_dict().and_then(|c| c.get(b"AcroForm").and_then(Object::as_ref))).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); if annots_unbounded { disallowed.push(kind::OTHER); } + if page_tree_unbounded { + // Past the bound the tree is not known in full: no change is judged against it. + disallowed.push(kind::PAGES); + } // Every reference the signed revision makes, read only when a slot's generations narrowed. let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); @@ -117,6 +129,9 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + page_nodes: &page_nodes, + kids_arrays: &kids_arrays, + old_form, placed_annots: &placed_annots, annot_budget: &annot_budget, signed_resources: &signed_resources, @@ -259,10 +274,13 @@ struct ChangedObjectClassifier<'a> { old_xfa_data: &'a HashSet, signed_pages: &'a HashSet, signed_annots: &'a HashSet, + page_nodes: &'a HashSet, + kids_arrays: &'a HashSet, placed_annots: &'a HashSet, annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, + old_form: Option, dss: &'a HashSet, } @@ -275,6 +293,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { + // A signed page-tree node's /Kids array of its own lists the pages. + Object::Array(_) if before.is_some() && self.kids_arrays.contains(&r.num) => Some(kind::PAGES), Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), @@ -342,6 +362,15 @@ impl ChangedObjectClassifier<'_> { if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) + } else if previous.is_some() && self.page_nodes.contains(&r.num) { + // A signed page-tree node is a page-tree change, whatever its new version is labelled. + Some(kind::PAGES) + } else if let Some(previous) = previous.filter(|_| self.old.root().is_some_and(|root| root.num == r.num)) { + // The signed catalog, whatever /Type its new version carries. + Some(catalog_kind(self.doc, self.old, dict, previous)) + } else if let Some(previous) = previous.filter(|_| self.old_form == Some(r.num)) { + // The signed form dictionary, held in an object of its own, whatever keys it carries now. + Some(acroform_kind(self.doc, self.old, dict, Some(previous))) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. Some(kind::METADATA) @@ -601,6 +630,10 @@ fn acroform(doc: &Document) -> Option { fn acroform_kind(doc: &Document, old: &Document, now: &Dict, before: Option<&Dict>) -> &'static str { let fallback = acroform(old).unwrap_or_default(); let before = before.unwrap_or(&fallback); + // A field taken out of the form: no DocMDP permission covers dropping a signed field. + if lost_entries(doc, old, before.get(b"Fields"), now.get(b"Fields")) { + return kind::OTHER; + } let strip = |d: &Dict| -> Vec<(Vec, Object)> { let mut v: Vec<(Vec, Object)> = d.iter().filter(|(k, _)| !matches!(k.as_slice(), b"Fields" | b"SigFlags")).map(|(k, o)| (k.to_vec(), o.clone())).collect(); @@ -724,6 +757,62 @@ fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { } } +/// A change to the signed catalog `b`, now `d`. +fn catalog_kind(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> &'static str { + let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; + if without(d, keys) != without(b, keys) { + kind::STRUCTURE + } else if !extensions_kept(doc, old, d, b) { + // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A writer + // adding a /DSS also declares the ESIC extension it belongs to, so a declaration may be + // added, but none that was signed may change or go. + kind::STRUCTURE + } else if d.get(b"AcroForm") != b.get(b"AcroForm") { + // Compare the forms themselves, inline or not. + let now = acroform(doc).unwrap_or_default(); + acroform_kind(doc, old, &now, None) + } else if b.get(b"Perms") != d.get(b"Perms") { + kind::SIGNATURE + } else { + kind::METADATA + } +} + +/// The nodes of `doc`'s page tree and the /Kids arrays that are objects of their own, by number: +/// every dictionary with /Kids from the catalog's /Pages down. Each node, and each /Kids array +/// object, is read once. `None` past a million nodes. +fn page_tree_nodes(doc: &Document) -> Option<(HashSet, HashSet)> { + const MAX_NODES: usize = 1_000_000; + let (mut nodes, mut arrays) = (HashSet::new(), HashSet::new()); + let top = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|c| c.get(b"Pages").and_then(Object::as_ref))); + let mut stack: Vec = top.into_iter().collect(); + while let Some(r) = stack.pop() { + if nodes.len() >= MAX_NODES { + return None; + } + let node = doc.get(r); + // A leaf (a page) has no /Kids. + let Some(kids) = node.as_dict().and_then(|d| d.get(b"Kids")) else { continue }; + if !nodes.insert(r.num) || kids.as_ref().is_some_and(|a| !arrays.insert(a.num)) { + continue; + } + if let Some(kids) = doc.resolve(kids).as_array() { + stack.extend(kids.iter().filter_map(Object::as_ref)); + } + } + Some((nodes, arrays)) +} + +/// Whether a list of references (`/Fields`) no longer holds an entry it held when signed. An +/// entry is still held only by a reference that resolves now and that the signed revision +/// resolved too, so to the same slot (a repaired alias counts): one rewritten to a generation +/// the slot does not accept names nothing, and one to a newer generation names another object. +fn lost_entries(doc: &Document, old: &Document, was: Option<&Object>, now: Option<&Object>) -> bool { + let live = |d: &Document, r: &ObjRef| !matches!(*d.get(*r), Object::Null); + let now: HashSet = refs(doc, now).into_iter().filter(|r| live(doc, r) && live(old, r)).map(|r| r.num).collect(); + refs(old, was).iter().filter(|r| live(old, r)).any(|r| !now.contains(&r.num)) +} + fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) -> &'static str { let ty = d.name(b"Type"); // A later revision cannot make an existing object a signature by relabelling it: a signature @@ -746,31 +835,15 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) }) { return kind::OTHER; } + // Where a signed widget sits and whether it shows are what the page shows: filling the form + // changes its value and appearance, never its rectangle or its flags. + if before.is_some_and(|b| b.name(b"Subtype") == Some(b"Widget") && (d.get(b"Rect") != b.get(b"Rect") || d.get(b"F") != b.get(b"F"))) { + return kind::PAGE_CONTENT; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, - Some(b"Catalog") => { - let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; - return match before { - Some(b) if without(d, keys) != without(b, keys) => kind::STRUCTURE, - // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A - // writer adding a /DSS also declares the ESIC extension it belongs to, so a - // declaration may be added, but none that was signed may change or go. - Some(b) if !extensions_kept(doc, old, d, b) => kind::STRUCTURE, - Some(b) => { - if d.get(b"AcroForm") != b.get(b"AcroForm") { - // Compare the forms themselves, inline or not. - let now = acroform(doc).unwrap_or_default(); - acroform_kind(doc, old, &now, None) - } else if b.get(b"Perms") != d.get(b"Perms") { - kind::SIGNATURE - } else { - kind::METADATA - } - } - None => kind::FORM_FILL, - }; - } + Some(b"Catalog") => return before.map_or(kind::FORM_FILL, |b| catalog_kind(doc, old, d, b)), Some(b"Page") => { return match before { Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index 791b2b503b..bcd6334719 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -8,4 +8,5 @@ mod crypto; mod encrypted_signing; mod pdf; mod revision_classification; +mod signed_structure; mod store_keys; diff --git a/crates/sign/tests/it/signed_structure.rs b/crates/sign/tests/it/signed_structure.rs new file mode 100644 index 0000000000..471a43c8fe --- /dev/null +++ b/crates/sign/tests/it/signed_structure.rs @@ -0,0 +1,199 @@ +//! A certified document's page tree, catalog and form keep the roles they had when signed: a +//! later revision that relabels them, drops a field from the form, or hides or moves a signed +//! field's widget is no form fill. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object, PdfString, SaveOptions, write_incremental}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, kind, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn r(num: u32) -> ObjRef { + ObjRef { num, generation: 0 } +} + +/// Two pages, listed by the /Kids array object 6; page 3 holds text field 5, listed in /Fields. +fn form() -> Vec { + write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>", + "<< /Type /Pages /Kids 6 0 R /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [5 0 R] >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + "[3 0 R 4 0 R]", + ]) +} + +fn certified(p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let opts = SignOptions { + page: 1, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(p), + ..SignOptions::default() + }; + pdfcraft_sign::sign(&open(&form()), &id, &opts).unwrap() +} + +fn edit_after(signed: &[u8], f: impl FnOnce(&mut Document)) -> Vec { + let mut doc = open(signed); + f(&mut doc); + write_incremental(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap() +} + +/// The certification signature of `bytes`: its status and modification verdict. +fn verdict(bytes: &[u8]) -> (Status, Modification) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify.is_some()).unwrap(); + (s.status, s.modification) +} + +/// `bytes` is refused, for (among others) `expected`. +fn assert_refused(what: &str, bytes: &[u8], expected: &str) { + let (status, modification) = verdict(bytes); + assert!( + matches!(&modification, Modification::Disallowed(kinds) if kinds.contains(&expected)) && status != Status::Valid, + "{what}: {status:?} {modification:?}, expected {expected}" + ); +} + +#[test] +fn a_signed_page_tree_node_is_judged_as_the_page_tree() { + let signed = certified(2); + // Relabelled as a field and dropping a page. + assert_refused( + "node relabelled", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.remove(b"Type"); + n.set(b"T".to_vec(), PdfString::text("x")); + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(r(4))])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Turning every page through an inherited attribute. + assert_refused( + "inherited rotation", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.set(b"DR".to_vec(), Object::Dict(Default::default())); + n.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Reordering the pages through the node's /Kids array object. + assert_refused( + "pages reordered", + &edit_after(&signed, |doc| doc.set(r(6), Object::Array(vec![Object::Ref(r(4)), Object::Ref(r(3))]))), + kind::PAGES, + ); +} + +#[test] +fn a_signed_catalog_is_judged_as_the_catalog_whatever_its_type() { + let signed = certified(2); + assert_refused( + "catalog relabelled", + &edit_after(&signed, |doc| { + let content = doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(Default::default(), b"0 0 m 300 300 l S".to_vec()))); + let pages = doc.add(Object::Dict(Default::default())); + let mut page = pdfcraft_cos::Dict::new(); + page.set(b"Parent".to_vec(), Object::Ref(pages)); + page.set(b"MediaBox".to_vec(), Object::Array([0, 0, 300, 300].map(Object::Int).to_vec())); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + doc.update_dict(pages, |n| { + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(page)])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + c.remove(b"Type"); + c.set(b"FT".to_vec(), Object::name("Tx")); + c.set(b"Pages".to_vec(), Object::Ref(pages)); + }) + .unwrap(); + }), + kind::STRUCTURE, + ); +} + +#[test] +fn a_field_dropped_from_the_form_is_no_change_any_level_permits() { + for p in [1, 2] { + let signed = certified(p); + let set_fields = |fields: Vec| { + edit_after(&signed, move |doc| { + let root = doc.root().unwrap(); + let mut form = + doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm")).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap(); + let kept: Vec = fields + .into_iter() + .chain( + form.get(b"Fields") + .and_then(Object::as_array) + .into_iter() + .flatten() + .filter(|f| f.as_ref().is_some_and(|f| f.num != 5)) + .cloned(), + ) + .collect(); + form.set(b"Fields".to_vec(), Object::Array(kept)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(form))).unwrap(); + }) + }; + assert_refused(&format!("P={p} field dropped"), &set_fields(Vec::new()), kind::OTHER); + let dead = ObjRef { num: 5, generation: 3 }; + assert_refused(&format!("P={p} field listed by a dead generation"), &set_fields(vec![Object::Ref(dead)]), kind::OTHER); + // The field's slot taken by a newer generation: another object, listed in its place. + let newer = ObjRef { num: 5, generation: 1 }; + let replaced = edit_after(&set_fields(vec![Object::Ref(newer)]), |doc| { + let mut field = doc.get(r(5)).as_dict().cloned().unwrap(); + field.set(b"V".to_vec(), PdfString::text("Eve")); + doc.set(newer, Object::Dict(field)); + }); + assert_refused(&format!("P={p} field replaced by a newer generation"), &replaced, kind::OTHER); + } +} + +#[test] +fn a_signed_fields_widget_may_be_filled_but_not_hidden_or_moved() { + let signed = certified(2); + let edit = |f: &dyn Fn(&mut pdfcraft_cos::Dict)| edit_after(&signed, |doc| doc.update_dict(r(5), |w| f(w)).unwrap()); + let filled = edit(&|w| w.set(b"V".to_vec(), PdfString::text("Bob"))); + assert_eq!(verdict(&filled), (Status::Valid, Modification::Allowed(vec![kind::FORM_FILL]))); + assert_refused("hidden", &edit(&|w| w.set(b"F".to_vec(), Object::Int(2))), kind::PAGE_CONTENT); + assert_refused("moved", &edit(&|w| w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4]))), kind::PAGE_CONTENT); +} From 560c68db899ae332b9fda18677583ae9a45e3043 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:41:29 -0500 Subject: [PATCH 66/85] fix(sign): an existing object is judged by the role it had when signed Rebased #5187 onto 9afa1266. Landing the signed-role rule only: an existing object that gains form or annotation keys is judged by what it was when signed, and a signed annotation turned into another kind is no comment. The two P1 regressions the original PR carried are not landed: the signature decision still reads the new /FT (a signed widget relabelled as a text field is not signing), and the dropped-/Fields check runs before the form-key comparison (already carried by #6358's lost_entries). Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 23 ++++-- crates/sign/tests/it/pdf.rs | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index a44c08bb7c..8cfd823fb0 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -858,19 +858,32 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) Some(b"Pages") => return if before.is_some() { kind::PAGES } else { kind::STRUCTURE }, _ => {} } - match d.name(b"Subtype") { + // The role an existing object plays (annotation, form field, the form) is the one it had when + // signed: keys its new version carries are written by whoever changed it. An indirect + // /OCProperties rewritten with an added /FT still decides which layers show (#1787 judges a + // signed page's resources this way; this applies it to every other object). + let role = before.unwrap_or(d); + // Nor does an existing annotation or widget turned into another kind of annotation edit the + // one it was: an empty signature widget rewritten as a /Square that covers signed text is no + // signature change, and no comment either. + if before.is_some_and(|b| b.name(b"Subtype") != d.name(b"Subtype")) { + return kind::OTHER; + } + // The new version's /FT still decides whether this is a signature: a signed widget relabelled + // as a text field is not a signature change, whatever it was when signed. + match role.name(b"Subtype") { Some(b"Widget") => return if d.name(b"FT") == Some(b"Sig") && signature_ok { kind::SIGNATURE } else { kind::FORM_FILL }, Some(b"Link") => return kind::LINKS, - Some(_) if ty == Some(b"Annot") || d.contains(b"Rect") => return kind::COMMENTS, + Some(_) if role.name(b"Type") == Some(b"Annot") || role.contains(b"Rect") => return kind::COMMENTS, _ => {} } - if d.name(b"FT") == Some(b"Sig") && signature_ok { + if role.name(b"FT") == Some(b"Sig") && signature_ok { return kind::SIGNATURE; } - if d.contains(b"Fields") { + if role.contains(b"Fields") { return acroform_kind(doc, old, d, before); } - if d.contains(b"FT") || (d.contains(b"T") && d.contains(b"Kids")) || d.contains(b"DR") { + if role.contains(b"FT") || (role.contains(b"T") && role.contains(b"Kids")) || role.contains(b"DR") { return kind::FORM_FILL; } if before.is_none() { diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 57efadbaf3..8f327115d2 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -1620,6 +1620,145 @@ fn a_dictionary_labelled_metadata_is_not_a_metadata_change() { }); } +/// A one-page document whose text sits in an optional-content group (object 8) that the +/// catalog's indirect `/OCProperties` (object 7) lists as shown. +fn fixture_with_layer() -> Vec { + let content = b"/OC /oc1 BDC BT /F1 14 Tf 20 250 Td (Clause 4: the buyer pays) Tj ET EMC"; + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> /OCProperties 7 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Properties << /oc1 8 0 R >> >> /Annots [6 0 R] >>".to_vec(), + [format!("<< /Length {} >>\nstream\n", content.len()).as_bytes(), content, b"\nendstream"].concat(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>".to_vec(), + b"<< /OCGs [8 0 R] /D << /ON [8 0 R] >> >>".to_vec(), + b"<< /Type /OCG /Name (Clause 4) >>".to_vec(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// Hide the clause: rewrite `/OCProperties` (object 7) so its group starts off, adding `extra`. +fn hide_layer(doc: &mut Document, extra: &[(&str, Object)]) { + let mut d = dict_of(vec![("OCGs", Object::Array(vec![oref(8)])), ("D", Object::Dict(dict_of(vec![("OFF", Object::Array(vec![oref(8)]))])))]); + for (k, v) in extra { + d.set(k.as_bytes().to_vec(), v.clone()); + } + doc.set(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, Object::Dict(d)); +} + +#[test] +fn an_existing_object_given_form_keys_after_signing_is_still_judged() { + // An existing object is judged by the role it had when signed, not only when a signed page + // draws it (#1787): the catalog's indirect /OCProperties decides which layers show, and a + // rewrite that hides the clause stays a change whatever form-field keys it adds. + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &[])); + let zeros = Object::Array(vec![Object::Int(0); 4]); + for extra in [ + vec![("FT", Object::name("Tx"))], + vec![("DR", Object::Dict(pdfcraft_cos::Dict::new()))], + vec![("T", Object::String(PdfString::text("x"))), ("Kids", Object::Array(Vec::new()))], + vec![("Subtype", Object::name("Widget")), ("Rect", zeros.clone())], + vec![("Subtype", Object::name("Square")), ("Rect", zeros)], + ] { + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &extra)); + } +} + +#[test] +fn an_unsigned_signature_widget_turned_into_a_comment_is_no_edit_of_it() { + // The empty signature field "Approval" (object 6) rewritten as a white /Square annotation that + // covers the signed text. Judged by what it was, it is a signature widget; but an annotation + // of another kind is no edit that widget permits, at any level. + assert_refused_at_every_level(&fixture(), "other changes", |doc| { + let white = Object::Array(vec![Object::Int(1); 3]); + doc.set( + pdfcraft_cos::ObjRef { num: 6, generation: 0 }, + Object::Dict(dict_of(vec![ + ("Type", Object::name("Annot")), + ("Subtype", Object::name("Square")), + ("Rect", Object::Array(vec![Object::Int(0), Object::Int(0), Object::Int(300), Object::Int(300)])), + ("C", white.clone()), + ("IC", white), + ("P", oref(3)), + ])), + ); + }); +} + +/// `fixture()` with the form an indirect dictionary (object 7) listing the unsigned field. +fn fixture_with_indirect_acroform() -> Vec { + let objs: Vec<&[u8]> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm 7 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>", + b"<< /Length 44 >>\nstream\nBT /F1 14 Tf 20 250 Td (Contract text) Tj ET\nendstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>", + b"<< /Fields [6 0 R] >>", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +#[test] +fn the_forms_signed_fields_are_not_dropped_after_signing() { + // Signed, then timestamped. A later revision drops the form's /Fields, or empties it: the + // fields the timestamp covered, the signature among them, are gone from the form. That is no + // signature change, so the timestamp reports it as not permitted. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let tsa = TestTsa { + id: pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(), + time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 }, + }; + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture_with_indirect_acroform()), &id, &opts()).unwrap(); + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let mut failures = Vec::new(); + for (how, fields) in [("dropped", None), ("emptied", Some(Object::Array(Vec::new())))] { + let edited = edit_after(&stamped, |doc| { + doc.update_dict(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, |af| match &fields { + Some(f) => af.set(b"Fields".to_vec(), f.clone()), + None => { + af.remove(b"Fields"); + } + }) + .unwrap(); + }); + let stamp = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.doc_timestamp).expect("the document timestamp"); + if !matches!(stamp.modification, Modification::Disallowed(_)) { + failures.push(format!("/Fields {how}: {:?}, {:?}", stamp.status, stamp.modification)); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + /// A one-page document like `fixture()` (objects 1–6) plus a stream, object 7, with the given /// dictionary entries and body. `catalog_extra` goes into the catalog; the page draws object 7 /// as a Form XObject when `drawn`. From fbe079c953d3f2644ac59d40894a07fb061f21b0 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:43:14 -0500 Subject: [PATCH 67/85] fix(sign): compare the catalog's /Perms by content under P=1 Rebased #5173 onto 9afa1266. Its later-signature predicate is superseded by #6364's object-based check, so only the /Perms rule lands: under DocMDP P=1 a /Perms that differs from the signed catalog's, by content, is a structure change however it was rewritten. Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 73 +++++++++++++++++++++++++++++++++- crates/sign/tests/it/pdf.rs | 29 ++++++++++++++ parity/acrobat-features.toml | 4 +- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 8cfd823fb0..712c9fcfff 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -217,13 +217,30 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an // approval signature, an unsigned signature widget, or a widget that paints over the signed // page — is a change the certification forbids, however it is labelled. - if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + if p == Some(1) { + only_validation_data(&mut allowed, &mut disallowed, doc, &old); + } + if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } +} + +/// DocMDP P=1 ("no changes allowed") permits only validation data to follow the signed revision: +/// the security store, and document timestamps that draw nothing. A later approval signature, a +/// visible widget or a changed catalog `/Perms` breaks the certification. +fn only_validation_data(allowed: &mut Vec<&'static str>, disallowed: &mut Vec<&'static str>, doc: &Document, old: &Document) { + if allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, old) { allowed.retain(|k| *k != kind::SIGNATURE); if !disallowed.contains(&kind::SIGNATURE) { disallowed.push(kind::SIGNATURE); } } - if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } + // The catalog's /Perms names the certification signature itself. Elsewhere a change to it is + // judged as signing; under P=1 dropping or redirecting it is a change like any other. + // Compared by content: an indirect /Perms rewritten in place keeps its reference (and, shared + // with the trailer's /Info, would otherwise pass as a metadata change). + let perms = |d: &Document| d.root().and_then(|r| d.get(r).as_dict().and_then(|c| c.get(b"Perms").map(|p| (*d.resolve(p)).clone()))); + if perms(doc) != perms(old) && !disallowed.contains(&kind::STRUCTURE) { + disallowed.push(kind::STRUCTURE); + } } /// Whether every signature object `doc` gained or changed since the signed revision `old` is an @@ -892,3 +909,55 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) } kind::OTHER } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + /// A PDF of `objs` (numbered from 1) whose trailer also holds `trailer_extra`. + fn pdf(objs: &[&str], trailer_extra: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer_extra} >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out + } + + /// A P=1 signed revision whose catalog `/Perms` and trailer `/Info` are one indirect + /// dictionary (object 3). A later revision points its `/DocMDP` at another signature + /// dictionary already in the file, in place: the reference stays, no object is added, and + /// as the signed `/Info` the object reads as metadata, but the certification's own record + /// has changed. + #[test] + fn a_perms_dictionary_shared_with_info_rewritten_in_place_is_a_structure_change() { + let signed = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R /Perms 3 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /DocMDP 4 0 R >>", + "<< /Type /Sig >>", + "<< /Type /Sig >>", + ], + "/Info 3 0 R", + ); + let old = Document::open(Arc::new(signed.clone())).unwrap(); + let mut doc = Document::open(Arc::new(signed)).unwrap(); + let other = ObjRef { num: 5, generation: 0 }; + doc.update_dict(ObjRef { num: 3, generation: 0 }, |p| p.set(b"DocMDP".to_vec(), Object::Ref(other))).unwrap(); + let edited = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions { object_streams: false, ..Default::default() }).unwrap(); + let doc = Document::open(Arc::new(edited)).unwrap(); + match classify_changes(&doc, Some(old), Some(1)) { + Modification::Disallowed(kinds) => assert!(kinds.contains(&kind::STRUCTURE), "{kinds:?}"), + other => panic!("the certification's /Perms changed in place under P=1: {other:?}"), + } + } +} diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 8f327115d2..8c9a827e6d 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -3164,3 +3164,32 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val all.iter().map(|s| (&s.status, &s.modification)).collect::>() ); } + +#[test] +fn the_catalogs_perms_do_not_change_under_no_changes() { + // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later + // revision that drops it, or points it elsewhere, breaks the certification. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let certified = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + let elsewhere = dict_of(vec![("DocMDP", oref(5))]); + for (how, perms) in [("dropped", None), ("pointed elsewhere", Some(Object::Dict(elsewhere)))] { + let edited = edit_after(&certified, |doc| { + let root = doc.root().unwrap(); + doc.update_dict(root, |c| match &perms { + Some(p) => c.set(b"Perms".to_vec(), p.clone()), + None => { + c.remove(b"Perms"); + } + }) + .unwrap(); + }); + let s = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.field == "Signature1").unwrap(); + assert!( + s.status == Status::Invalid && matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::STRUCTURE)), + "/Perms {how} under P=1: {:?}, {:?}", + s.status, + s.modification + ); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ec144d3a07..10b040e09f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6535,8 +6535,8 @@ milestone = "M9" status = "shipped" commands = ["sign.panel"] tools = ["sign_list"] -evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill"] -notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision" +evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill", "crates/sign/tests/it/pdf.rs::the_catalogs_perms_do_not_change_under_no_changes"] +notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision. Under P=1 a change to the catalog's /Perms breaks the certification." [[feature]] id = "sign.view-signed-version" From d7ce875dd692109789bcdad1b683ffc5a2d07dc7 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 10:15:07 +0200 Subject: [PATCH 68/85] redact: independent redaction proof machinery (snapshot, sweep, budget) (#5557) * redact: add the independent redaction proof machinery * redact: harden the proof's paint check and count removed codes The manifest counts character codes, not raw bytes: a composite font's codes are several bytes, so the byte length overstated what was cut. Annotation appearances are checked whatever their storage (a direct dictionary has no reference of its own to dedupe on), and a stream shared by several annotations is checked at each annotation's own placement. The graphics-state stack and the clip holes of one walk are bounded; deeper or more is unverifiable, never unbounded memory. Even-odd clips cut holes only where the rule can be told cheaply - all subpaths axis-aligned, disjoint, few - and anything else keeps its paint reported, failing closed. The /W widths of a descendant font cost bounded work, like a CMap. * redact: settle the survivors a page sweep counted before its work ran out A page that the search budget stops on left the counts made on earlier pages unsettled: the survivors were seen but never reported, and the regions that owned them were never marked. The sweep now stops and still settles, so a located survivor is reported where it sits (the page reads unswept either way, and the proof still fails). * redact: prove the redacted output of every supported encryption The authenticated-reopen regression now covers all four crypt algorithms the writer supports, not only AES-128: the caller's credentials prove the saved output, and empty or wrong credentials still fail closed. * redact: answer the review round on the proof A glyph lying under several regions is now accounted for by every one of them, so overlapping marks each carry their own share in a shared proof instead of the first one absorbing the rest and the others reading as if no text was there. Each finding is recorded once per region as it is found, so a page that paints over a region a thousand times neither grows the report nor the memory with repeats. The byte search walks the haystack once (the pattern's prefix function is built once), so a needle of one letter over a haystack full of it can no longer spend comparisons the search budget never charged for. A numeric XML character reference may carry any number of leading zeros; the padded form now decodes like the short one, so a survivor spelled that way in the XMP is still found. The /W widths of a descendant font are paid for from the shared work budget: a form invoked again rebuilds its fonts, and a per-font counter alone let the same array be paid for many times over. * redact: sweep the proof's survivors out of every place a reader could look Plant the removed text where the review listed it and expect the sweep to report it: a compressed object stream, a page thumbnail, a form field value, an XFA packet, and an object nothing refers to. A file whose page content is bigger than the proof may decode for one stream is now shown to come out unverified on its surfaces and its region, never clean. * redact: spare the names a redacted word legitimately sits in, and charge failed decodes A redacted word that is also a structural name (a /Subtype, a /Filter, a /BaseFont) is structure, not a leftover copy: the name surface is now swept against the matches the input itself had, and only a surplus of matches is a survivor. The input's names are counted when the snapshot is taken, for the region's needles and for strings added later alike. A decode that fails after doing part of its work paid nothing: the proof's cumulative decode allowance now charges every attempt, failures included, so the same bomb cannot be decoded (and fail) again and again for free. * redact: count the input's names once each, and say so when the count runs out The name allowance counted one entry per occurrence, so a document whose pages repeat the same structural name paid for the same scan again and again; names are now kept once, with their count. A name search that runs out of work read as no matches, which would have swept names with none allowed: that is reported on the text surface now, and the proof fails rather than pass on an uncounted allowance. * Bound redaction proof searches across many needles Assisted-by: OpenAI Codex * Redaction proof: drop find_all, which nothing calls Co-Authored-By: Claude Opus 5.5 * Cargo.lock: put aho-corasick in its sorted place Co-Authored-By: Claude Opus 5.5 * Redaction proof: review notes A word split across a TJ array is looked for whole. Overlapping marks count a removed copy once. A spent decode budget is a problem, not a raw-only stream. Short needles stay out of the attachment text search, and an indirect /Decode counts. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + crates/redact/Cargo.toml | 2 + crates/redact/src/image.rs | 2 +- crates/redact/src/interp.rs | 100 +- crates/redact/src/lib.rs | 11 +- crates/redact/src/tests_verify.rs | 1380 ++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 711 +++++++++++ crates/redact/src/verify/geometry.rs | 500 ++++++++ crates/redact/src/verify/limits.rs | 42 + crates/redact/src/verify/mod.rs | 389 ++++++ crates/redact/src/verify/search_tests.rs | 34 + crates/redact/src/verify/snapshot.rs | 532 +++++++++ crates/redact/src/verify/sweep.rs | 759 ++++++++++++ crates/redact/src/verify/util.rs | 301 +++++ 14 files changed, 4759 insertions(+), 15 deletions(-) create mode 100644 crates/redact/src/tests_verify.rs create mode 100644 crates/redact/src/verify/extract.rs create mode 100644 crates/redact/src/verify/geometry.rs create mode 100644 crates/redact/src/verify/limits.rs create mode 100644 crates/redact/src/verify/mod.rs create mode 100644 crates/redact/src/verify/search_tests.rs create mode 100644 crates/redact/src/verify/snapshot.rs create mode 100644 crates/redact/src/verify/sweep.rs create mode 100644 crates/redact/src/verify/util.rs diff --git a/Cargo.lock b/Cargo.lock index 4816a44ec0..e1f271ed12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4610,6 +4619,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4620,6 +4630,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index fe21d13b32..c02335e3c4 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +aho-corasick = "1" pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } @@ -20,6 +21,7 @@ pdfcraft-xfa = { workspace = true } # CCITT Group 4 and JBIG2 encoding for cleared one-bit images. pdfcraft-filters = { workspace = true } thiserror = { workspace = true } +sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 4b3c44b0fd..a99e179895 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -12,7 +12,7 @@ use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed), or to be stored /// again without shared `/JBIG2Globals` (see [`without_globals`]). -const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; +pub(crate) const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; fn components(doc: &Document, s: &Stream) -> Option { let cs = doc.resolve(s.dict.get(b"ColorSpace")?); diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 460085a320..7a2cd26eca 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -53,6 +53,23 @@ pub(crate) enum Mode { Verify, } +/// A run of glyphs cut out under a region, kept for the redaction proof +/// ([`Snapshot::capture`](crate::verify::Snapshot::capture)). +pub(crate) struct Removed { + /// Index of the region (in `Scope::rects`) the run lay under. + pub(crate) rect: usize, + /// The character codes as they stood in the string. + pub(crate) raw: Vec, + /// What the font says they showed (codes without a known meaning are left out). + pub(crate) text: String, + /// How many character codes the run holds (a composite font's codes are several bytes). + pub(crate) codes: usize, + /// Which text operator showed it, counted from the start of the page. + pub(crate) op: usize, + /// Where in that operator it starts: the operand's place in a `TJ` array, and the byte in it. + pub(crate) at: (usize, usize), +} + #[derive(Clone)] struct Gs { ctm: Matrix, @@ -124,6 +141,12 @@ pub(crate) struct Scope<'a> { xobject_oc_cache: HashMap, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Collects the glyph runs removed under a region in apply mode (`None` = not collected). + /// The redaction proof's snapshot is what sets it: it must know what applying is about to + /// remove, by the interpreter's own judgement, without trusting anything else to it. + pub removed: Option>, + /// How many text operators have been shown, for [`Removed::op`]. + shown: usize, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -149,6 +172,8 @@ impl<'a> Scope<'a> { ocmd_cache: HashMap::new(), xobject_oc_cache: HashMap::new(), layer_blocks: 0, + removed: None, + shown: 0, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -169,21 +194,33 @@ impl<'a> Scope<'a> { self.rects.iter().any(|r| contains(*r, b, 0.01)) } - /// Does a glyph box count as under a region? Any real overlap does (more than a fifth of + /// Which regions a glyph box counts as under? Any real overlap does (more than a fifth of /// the glyph or a point, whichever is less, in both directions), so neighbours that only /// touch an edge survive while partly covered glyphs go. Degenerate boxes (zero-size text) - /// count when their origin is inside. - fn glyph_hit(&self, b: [f64; 4], origin: (f64, f64)) -> bool { + /// count when their origin is inside. Empty = under no region (a glyph kept). A glyph may + /// lie under several regions at once (overlapping marks); each is reported. + fn glyph_rects(&self, b: [f64; 4], origin: (f64, f64)) -> Vec { let (gw, gh) = (b[2] - b[0], b[3] - b[1]); if gw < 0.01 || gh < 0.01 { - return self.rects.iter().any(|r| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]); + return self + .rects + .iter() + .enumerate() + .filter(|(_, r)| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]) + .map(|(i, _)| i) + .collect(); } let (mx, my) = ((gw * 0.2).min(1.0), (gh * 0.2).min(1.0)); - self.rects.iter().any(|r| { - let ox = b[2].min(r[2]) - b[0].max(r[0]); - let oy = b[3].min(r[3]) - b[1].max(r[1]); - ox > mx && oy > my - }) + self.rects + .iter() + .enumerate() + .filter(|(_, r)| { + let ox = b[2].min(r[2]) - b[0].max(r[0]); + let oy = b[3].min(r[3]) - b[1].max(r[1]); + ox > mx && oy > my + }) + .map(|(i, _)| i) + .collect() } } @@ -223,6 +260,8 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The regions it lies under (empty for hidden text). + rects: Vec, } /// Process a scope's content streams (state flows from one stream to the next). @@ -644,11 +683,14 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +/// In apply mode the runs removed under each region are also handed to `scope.removed`. +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + scope.shown += 1; + let op = scope.shown; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -661,7 +703,7 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let bytes = &s.bytes; @@ -674,18 +716,50 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let degenerate = b[2] - b[0] < 0.01 && b[3] - b[1] < 0.01; matches!(gs.render_mode, 3 | 7) || !(overlaps(page, b, 0.0) || degenerate) } - None => scope.glyph_hit(b, origin), + None => { + rects = scope.glyph_rects(b, origin); + !rects.is_empty() + } }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, rects }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } + if scope.mode == Mode::Apply + && let Some(sink) = scope.removed.as_mut() + { + // Consecutive removed glyphs under one region are one run, recorded for + // every region the glyphs lie under: overlapping marks each account for + // their own share of what was cut. + let mut under: Vec = glyphs.iter().flat_map(|g| g.rects.iter().copied()).collect(); + under.sort_unstable(); + under.dedup(); + for r in under { + let mut k = 0; + while let Some(g) = glyphs.get(k) { + if !g.rects.contains(&r) { + k += 1; + continue; + } + let mut end = k; + while glyphs.get(end + 1).is_some_and(|n| n.rects.contains(&r)) { + end += 1; + } + let span = g.bytes.start..glyphs.get(end).map_or(g.bytes.end, |l| l.bytes.end); + let at = (place, span.start); + let raw = bytes.get(span).unwrap_or_default().to_vec(); + sink.push(Removed { rect: r, text: f.decode(&raw), raw, codes: end - k + 1, op, at }); + k = end + 1; + } + } + } let mut run: Vec = Vec::new(); for g in &glyphs { if g.hit { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index b6a8885c03..6bb8e5cc06 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,10 +27,14 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_verify; pub(crate) mod type3; pub(crate) mod type3_tables; +pub mod verify; use interp::{Mode, Scope, process}; +pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -59,6 +63,11 @@ pub enum RedactError { "redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)" )] OverlayLimit, + /// A stream the redaction created to draw the marks' boxes and labels can't be read back, so + /// the proof can't tell it apart from content that was there before and nothing can be + /// exempted from the sweep. + #[error("a stream the redaction created can't be read back, so the result can't be proven")] + OverlayUnreadable, /// The structure tree couldn't be cleaned completely (#3302): an element written in place that /// would need changing, an element whose inherited page can't be found, or a tree too large to /// walk. Tags that kept the removed text or annotation would undo the redaction. @@ -164,7 +173,7 @@ pub struct Report { pub overlays: Vec, } -fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { +pub(crate) fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { let v: Vec = doc.resolve(o?).as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == 4 && v.iter().all(|x| x.is_finite())).then(|| [v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]) } diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs new file mode 100644 index 0000000000..ee443f3209 --- /dev/null +++ b/crates/redact/src/tests_verify.rs @@ -0,0 +1,1380 @@ +//! The independent redaction proof: what it catches, what it refuses to call clean. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Algorithm, Dict, Document, NewEncryption, Object, SaveOptions, Stream, write_full, write_incremental}; +use sha2::Digest; + +use super::*; +use crate::verify::tests_support::{self, count_text_operators, survivor_encodings}; +use crate::verify::{Status, Surface, SurfaceReport, Verdict}; + +const SECRET: &str = "SECRETVALUE"; + +pub(super) fn stream(dict: &str, data: &[u8]) -> Vec { + let mut v = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + v.extend_from_slice(data); + v.extend_from_slice(b"\nendstream"); + v +} + +pub(super) fn pdf_bytes(objs: Vec>, trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +pub(super) fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// Every glyph is 500 units wide (5 pt at 10 pt). +pub(super) const FONT: &str = + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor << /Ascent 800 /Descent -200 >> >>"; + +/// A composite font without a /ToUnicode map: its text can't be decoded. +const OPAQUE_FONT: &str = "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 >>] >>"; + +/// Options of a one-page file: catalog, page and trailer entries, page resources, and more +/// objects (numbered from 7). +#[derive(Default)] +struct Spec<'a> { + catalog: &'a str, + page: &'a str, + trailer: &'a str, + resources: &'a str, + extra: Vec>, +} + +/// "Public" at the top, the secret lower down at x 10..65, y 100..110. +const CONTENT: &[u8] = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + +pub(super) const REGION: [f64; 4] = [8.0, 95.0, 70.0, 112.0]; + +fn build_with(content: &[u8], spec: Spec<'_>) -> Vec { + let mut objs: Vec> = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {} >>", spec.catalog).into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R {} /Resources << /Font << /F1 5 0 R >> {} >> >>", + spec.page, spec.resources + ) + .into_bytes(), + stream("", content), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + objs.extend(spec.extra); + pdf_bytes(objs, spec.trailer) +} + +pub(super) fn mark(doc: &mut Document, rects: &[[f64; 4]]) { + let shape = Shape::Redact { quads: rects.iter().map(|r| rect_quad(*r)).collect(), overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Tester".into() }, &Meta::default()).unwrap(); +} + +/// The standard page with `spec`, the secret's region marked. +fn marked(spec: Spec<'_>) -> Document { + marked_content(CONTENT, spec) +} + +fn marked_content(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[REGION]); + doc +} + +/// Apply, and tell the snapshot which streams the apply created (its overlay boxes and labels). +fn applied(doc: &mut Document, snap: &mut Snapshot) -> Result { + let result = apply(doc, None); + if let Ok(report) = &result { + snap.set_overlay_streams(doc, &report.overlays)?; + } + result +} + +/// Capture, apply, prove. +pub(super) fn run(doc: &mut Document) -> (Result, Proof) { + let mut snap = Snapshot::capture(doc, None).unwrap(); + let result = applied(doc, &mut snap); + let proof = snap.prove(doc, None, &ProofOptions::default()); + (result, proof) +} + +/// Prove a document the redaction was never applied to (as a full save would write it). +pub(super) fn prove_unapplied(doc: &mut Document) -> Proof { + let snap = Snapshot::capture(doc, None).unwrap(); + doc.require_full_save(); + snap.prove(doc, None, &ProofOptions::default()) +} + +pub(super) fn surface(p: &Proof, s: Surface) -> &SurfaceReport { + p.surfaces.iter().find(|r| r.surface == s).unwrap() +} + +fn residue(r: &Result) -> bool { + matches!(r, Err(RedactError::Residue(n)) if *n > 0) +} + +#[test] +fn the_proof_reads_a_pages_streams_as_the_one_stream_they_are() { + // The TJ array of the secret ends the first piece and its operator starts the second + // (#153): the proof reads the pieces as the one stream they are, so the removed text is + // neither reported as a survivor nor does the page count as unreadable. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td [(SECRETVALUE)]"), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"TJ ET BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let (r, proof) = run(&mut doc); + let r = r.unwrap(); + assert_eq!(r.glyphs, 11, "every glyph of the split string is removed: {r:?}"); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.survivors(), 0); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); +} + +#[test] +fn a_clean_redaction_passes_and_the_neighbour_is_untouched() { + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries.len(), 1); + let e = &proof.entries[0]; + assert_eq!((e.status, e.page, e.region), (Status::Verified, 0, REGION)); + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len())); + let content = page_content(&doc); + assert!(content.contains("(Public) Tj") && !content.contains(SECRET), "{content}"); + for s in &proof.surfaces { + assert!(matches!(s.verdict, Verdict::Clean | Verdict::Absent), "{s:?}"); + } + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Clean); +} + +#[test] +fn the_manifest_has_digests_and_counts_but_no_plaintext() { + let mut doc = marked(Spec::default()); + // The digest of the original content: a shared proof must not carry it. + let original: String = sha2::Sha256::digest(page_content(&doc).as_bytes()).iter().map(|b| format!("{b:02x}")).collect(); + let (r, proof) = run(&mut doc); + r.unwrap(); + let e = &proof.entries[0]; + let after = e.page_sha256_after.as_deref().unwrap(); + assert!(after.len() == 64 && after.bytes().all(|c| c.is_ascii_hexdigit())); + assert_ne!(after, original); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(&original), "{shown}"); + } + // Two glyph-carrying operators before; the removed one left a number-only TJ, which carries + // no glyphs and is not counted. + assert_eq!((e.text_ops_before, e.text_ops_after), (Some(2), Some(1))); + assert!(e.removed_text.is_none()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } + + // Plaintext only on request. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + assert!(!format!("{snap:?}").contains(SECRET)); + applied(&mut doc, &mut snap).unwrap(); + let with = snap.prove(&doc, None, &ProofOptions { include_plaintext: true }); + assert_eq!(with.entries[0].removed_text.as_deref(), Some(&[SECRET.to_string()][..])); +} + +#[test] +fn survivors_do_not_echo_the_text_either() { + let mut doc = marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } +} + +#[test] +fn numeric_only_tj_is_not_glyph_carrying() { + assert_eq!(count_text_operators(b"BT (Tj) Tj [ -250 ] TJ [(x) -3 (y)] TJ () Tj [] TJ ET"), 2); + assert_eq!(count_text_operators(b"BT (a) ' 1 2 (b) \" [ 10 -10 ] TJ ET"), 2); +} + +#[test] +fn a_survivor_in_the_info_dictionary() { + let mut doc = + marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (Report SECRETVALUE) /Author (Someone) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_xmp() { + let xmp = stream("/Type /Metadata /Subtype /XML", b"Quarterly & SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_long_numeric_reference_in_xmp_is_a_survivor() { + // A character reference may carry any number of leading zeros: the padded form reads the + // same, and is swept the same. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_an_annotation_string() { + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +// ── Every place a reader could recover the removed text from ──────────────────────────────── +// Each test plants the removed text in one location and expects the sweep to report it there. + +#[test] +fn a_survivor_in_a_compressed_object_stream_is_found() { + // A full save packs non-stream objects into compressed object streams: a copy of the + // removed text left in one (an annotation away from the region) is neither in the stored + // bytes nor on any page, only inside the /ObjStm. The sweep reads the saved file like any + // reader, objects in object streams included, and reports the copy. + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.windows(b"/ObjStm".len()).any(|w| w == b"/ObjStm"), "the output packs objects into object streams"); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the plaintext is compressed away"); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Clean, "the copy sits in the object stream, not the stored bytes"); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_page_thumbnail_is_found() { + // The page's /Thumb holds a picture of the page as it was, secret included (uncompressed + // pixels, so the stored bytes are the pixels). A thumbnail is not page content, so only + // the object sweep can find it. + let thumb = stream("/Type /XObject /Subtype /Image /Width 7 /Height 3 /ColorSpace /DeviceGray /BitsPerComponent 8", b"junk SECRETVALUE junk"); + let mut doc = marked(Spec { page: "/Thumb 7 0 R", extra: vec![thumb], ..Spec::default() }); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_form_field_value_is_found() { + // The redacted text also sits in the form: a text field whose /V holds it. The field + // dictionary is an object like any other, and its strings are swept wherever the field + // hangs (AcroForm and the page's annotations both point at it). + let field = b"<< /Type /Annot /Subtype /Widget /Rect [200 20 280 40] /FT /Tx /T (form) /V (note: SECRETVALUE) >>".to_vec(); + let spec = Spec { catalog: "/AcroForm << /Fields [7 0 R] >>", page: "/Annots [7 0 R]", extra: vec![field], ..Spec::default() }; + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + // The field's copy is the only place left (the save renumbers objects, so the place is + // named, not numbered from the input file; both views of the string count as survivors). + assert!(s.survivors >= 1, "{}", proof.to_text()); + assert_eq!(s.locations.len(), 1, "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_xfa_packet_is_found() { + // An XFA form mirrors its field values in the /XFA packets, XML of their own. Redaction + // strips an XFA form outright (crates/xfa does that, not this machinery); the proof must + // not take the stripper on trust: whatever it left (a shape it refused, a leftover + // packet) is swept like any stream, and what the packet holds is reported. + let xfa = stream("", b"
SECRETVALUE
"); + let spec = Spec { catalog: "/AcroForm << /XFA 7 0 R /Fields [] >>", extra: vec![xfa], ..Spec::default() }; + let doc = marked(spec); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_unused_object_is_found() { + // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that + // only walks the document tree misses it; the sweep walks every object the + // cross-reference table knows, orphans included, and names the object it sits in. + let orphan = format!("<< /Type /Annot /Subtype /Text /Contents (note: {SECRET}) >>").into_bytes(); + let doc = marked(Spec { extra: vec![orphan], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +/// The page draws "(Image)" under the marked region, and keeps an image XObject whose +/// `/Subtype /Image` is the same word. +fn image_word_doc() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let content = b"BT /F1 10 Tf 10 100 Td (Image) Tj ET"; + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[0.0, 90.0, 40.0, 115.0]]); + doc +} + +#[test] +fn a_word_that_remains_in_a_structural_name_does_not_fail_the_proof() { + // "Image" is redacted from the page, and is also the /Subtype of the image the page keeps. + // The name is structure redaction never edits, not a leftover copy: the proof must pass, + // and the name surface must read clean, not unswept. + let mut doc = image_word_doc(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Clean); + + // But a name the input did not have is a copy like any other: a surplus of name matches + // is a survivor, wherever it sits. + let snap = Snapshot::capture(&image_word_doc(), None).unwrap(); + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", catalog: "/Extra << /Stolen /Image >>", extra: vec![image], ..Spec::default() }; + let mut shown = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec)); + mark(&mut shown, &[[0.0, 90.0, 40.0, 115.0]]); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +fn attachment_spec(payload: &[u8]) -> Spec<'static> { + Spec { + catalog: "/Names << /EmbeddedFiles << /Names [(a.bin) 7 0 R] >> >>", + extra: vec![b"<< /Type /Filespec /F (a.bin) /EF << /F 8 0 R >> >>".to_vec(), stream("/Type /EmbeddedFile", payload)], + ..Spec::default() + } +} + +#[test] +fn a_survivor_in_an_attachment() { + let mut doc = marked(attachment_spec(b"the SECRETVALUE is attached")); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); + + // UTF-16LE with a byte order mark. + let mut le = vec![0xFF, 0xFE]; + le.extend(SECRET.encode_utf16().flat_map(u16::to_le_bytes)); + let mut doc = marked(attachment_spec(&le)); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_a_nested_attachment() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", Spec::default()); + let middle = build_with(b"", attachment_spec(&inner)); + let mut doc = marked(attachment_spec(&middle)); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::EmbeddedFiles); + assert_eq!(s.verdict, Verdict::Survivor); + assert!(s.locations.iter().any(|l| l.matches(" > ").count() >= 1), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_clean_nested_attachment_is_swept_and_clean() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (harmless) Tj ET", Spec::default()); + let mut doc = marked(attachment_spec(&inner)); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Clean); +} + +#[test] +fn unsweepable_attachments_fail_closed() { + for payload in [&b"PK\x03\x04 zipped"[..], b"%PDF-1.7 not really a pdf", b"\x1f\x8b gzip"] { + let mut doc = marked(attachment_spec(payload)); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Unswept, "{}", String::from_utf8_lossy(payload)); + assert!(!proof.passed(), "a proof that could not sweep must not pass"); + } +} + +#[test] +fn an_undecodable_stream_fails_closed() { + let junk = stream("/Filter /NoSuchDecode", b"\x00\x01\x02 junk"); + let mut doc = marked(Spec { catalog: "/Extra 7 0 R", extra: vec![junk], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn an_incremental_remnant_fails_the_proof() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + // A "redaction" saved as an incremental update: the page shows something else now, but the + // first revision still holds the original content. + let mut other = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let new = other.add(Object::Stream(Stream::flate(Dict::new(), b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"))); + let page = pdfcraft_model::pages(&other).swap_remove(0); + other.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + let saved = write_incremental(&other, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Survivor); + // The page text itself is clean: only the file structure gives it away. + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); + + // The real redaction, saved as apply() requires, is a single clean revision. + applied(&mut doc, &mut snap).unwrap(); + let full = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&full, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn the_secret_in_each_encoding_is_found() { + // Spelled as PDF syntax in a catalog entry. + let utf16: String = SECRET.encode_utf16().flat_map(u16::to_be_bytes).map(|b| format!("{b:02X}")).collect(); + let utf16le: String = SECRET.encode_utf16().flat_map(u16::to_le_bytes).map(|b| format!("{b:02X}")).collect(); + let hex8: String = SECRET.bytes().map(|b| format!("{b:02x}")).collect(); + let forms = [ + format!("({SECRET})"), + format!(""), + format!(""), + format!("", hex8.to_uppercase()), + format!("<{hex8}>"), + format!("<{utf16}>"), + "(\\123ECRETVALUE)".to_string(), + ]; + for f in forms { + let mut doc = marked(Spec { catalog: &format!("/Extra {f}"), ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed(), "{f}: {}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{f}: {}", proof.to_text()); + } + + // Spelled as bytes in the file, including the backslash-escaped literal form. + let paren = "SECR(ET)X\\Y"; + let enc = survivor_encodings(paren); + assert!(enc.len() >= 10); + for spelling in enc { + let mut hay = b"junk ".to_vec(); + hay.extend(&spelling); + hay.extend(b" junk"); + assert!(tests_support::found(paren, &hay), "{}", String::from_utf8_lossy(&spelling)); + } + assert!(!tests_support::found(paren, b"nothing to see")); +} + +#[test] +fn a_rotated_page_is_proven_too() { + let mut doc = marked(Spec { page: "/Rotate 90", ..Spec::default() }); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::Verified); +} + +#[test] +fn a_region_without_text_is_recorded_as_such() { + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + assert!(proof.passed()); +} + +fn opaque_spec() -> Spec<'static> { + Spec { resources: "/Font << /F1 5 0 R /F2 7 0 R >>", extra: vec![OPAQUE_FONT.as_bytes().to_vec()], ..Spec::default() } +} + +#[test] +fn text_that_cannot_be_decoded_makes_a_region_unverifiable() { + let mut doc = open(build_with(b"BT /F2 10 Tf 10 200 Td <00410042> Tj ET", opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(proof.entries[0].status, Status::Unverifiable); + assert!(!proof.passed(), "no PASS while a region is unverifiable"); +} + +#[test] +fn removed_text_of_an_undecodable_font_is_swept_by_its_codes() { + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].status, Status::Verified, "{}", proof.to_text()); + + // Left in place, the codes are still on the page and the proof says so. + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn codes_removed_counts_codes_not_bytes() { + // A composite font's codes are two bytes each: the manifest counts the codes, not the bytes. + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].codes_removed, 4, "{}", proof.to_text()); + + // A simple font's codes are one byte each: the two counts agree. + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!((proof.entries[0].runs_removed, proof.entries[0].codes_removed), (1, SECRET.len())); +} + +#[test] +fn overlapping_marks_each_account_for_their_own_share() { + // Two regions over the same text: each gets the runs, codes and status it is owed — the + // second one is not waved through as "no text in the region". + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries.len(), 2); + for e in &proof.entries { + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len()), "{}", proof.to_text()); + assert_eq!(e.status, Status::Verified, "{}", proof.to_text()); + } + + // Left in place, both regions name the survivor as theirs. + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert!(proof.entries.iter().all(|e| matches!(e.status, Status::Failed | Status::Unverifiable)), "{}", proof.to_text()); +} + +#[test] +fn nothing_applied_means_survivors_where_the_text_was() { + let mut doc = marked(Spec::default()); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn a_document_that_would_save_incrementally_is_not_proven() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + // A reopened copy no longer carries the full-save requirement. + let reopened = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let proof = snap.prove(&reopened, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); +} + +fn image_page() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0xFFu8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + marked_region(b"q 100 0 0 100 100 100 cm /Im1 Do Q 0 0 1 rg 120 120 60 60 re f", spec) +} + +fn marked_region(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[100.0, 100.0, 150.0, 150.0]]); + doc +} + +#[test] +fn images_and_paths_under_a_region_are_checked_independently() { + let mut doc = image_page(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + + // The same page unredacted: the image and the path are both still there. + let proof = prove_unapplied(&mut image_page()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed); + assert!(e.detail.contains("image") && e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn a_shading_under_a_region_must_be_clipped() { + let sh = + b"<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> >>" + .to_vec(); + let spec = || Spec { resources: "/Shading << /Sh1 7 0 R >>", extra: vec![sh.clone()], ..Spec::default() }; + let mut doc = marked_region(b"/Sh1 sh", spec()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + + let proof = prove_unapplied(&mut marked_region(b"/Sh1 sh", spec())); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn an_even_odd_clip_around_the_region_hides_what_it_cuts_out() { + // How kept content is told to hide: an even-odd clip of the page and the region. A path + // painted inside the region under that clip is clipped away; the same path under a clip + // the rule cannot read (a triangle, overlapping inners), or with no clip, is reported. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |tail: &str| -> Proof { + let content = format!("BT /F1 10 Tf 10 200 Td (Public) Tj ET {tail}"); + let mut shown = marked_content(content.as_bytes(), Spec::default()); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let proof = prove("q 0 0 300 300 re 8 95 62 17 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert!(proof.passed(), "{}", proof.to_text()); + + // A triangle is not a rectangle: no hole is cut, and the paint is reported. + let proof = prove("q 0 0 300 300 re 8 95 m 70 95 l 8 112 l h W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + // Overlapping inners change the parity: no hole is cut either. + let proof = prove("q 0 0 300 300 re 8 95 62 17 re 5 90 70 25 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + + // Unclipped, the same path is a plain finding. + let proof = prove("0 0 0 rg 10 100 20 10 re f"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); +} + +#[test] +fn a_graphics_state_stack_too_deep_to_follow_is_unverifiable() { + // Saved states carry their clip holes with them: a `q` stack deep enough to grow without + // bound is refused, not followed. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut deep = Vec::new(); + for _ in 0..crate::verify::limits::MAX_GS_STACK + 8 { + deep.extend_from_slice(b"q "); + } + deep.extend_from_slice(b"0 0 0 rg 10 100 20 10 re f"); + let mut shown = marked_content(&deep, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("too deeply"), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_page_full_of_paint_reports_its_finding_once() { + // Fifty paths over a region are one finding: the report names the problem once, not once + // per operator, and the report does not grow with the operator count. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut content = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET ".to_vec(); + for k in 0..50 { + content.extend_from_slice(&format!("0 0 1 rg {} 100 4 4 re f ", 10 + k).into_bytes()); + } + let mut shown = marked_content(&content, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert_eq!(proof.entries[0].detail.matches("a path still paints").count(), 1, "{}", proof.to_text()); +} + +#[test] +fn text_kept_elsewhere_does_not_fail_the_proof() { + // The same word is redacted in one place and legitimately remains in another. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let mut doc = marked_content(content, Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("remain in retained content")); + + // A copy that should have gone is still caught, by count. + let proof = prove_unapplied(&mut marked_content(content, Spec::default())); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn the_proof_text_keeps_its_honesty_wording() { + // The closing caveat of a proof is a promise to whoever reads a shared proof: pin it so the + // wording cannot silently drop out of the text for either verdict. + let honesty = "Evidence, not a guarantee"; + let passed = Proof { entries: vec![], surfaces: vec![], failures: vec![] }; + assert!(passed.passed()); + assert!(passed.to_text().contains(honesty), "{}", passed.to_text()); + + let failed = Proof { entries: vec![], surfaces: vec![], failures: vec!["the marked text is still readable".into()] }; + assert!(!failed.passed()); + assert!(failed.to_text().contains(honesty), "{}", failed.to_text()); +} + +// ── Independence, scope and bounds of the sweep ───────────────────────────────────────────── + +/// Hang `obj` off the catalog (as `/Extra`) so the saved file keeps it. +pub(super) fn attach(doc: &mut Document, obj: Object) { + let r = doc.add(obj); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(r))).unwrap(); +} + +/// Add a content stream (with `dict`) after the page's own. +fn append_content(doc: &mut Document, dict: Dict, data: &[u8]) { + let new = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + let page = pdfcraft_model::pages(doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); +} + +#[test] +fn a_glyph_the_interpreter_kept_is_found_by_placement_alone() { + // The snapshot is taken where the region holds no text, so it has nothing to search for: + // only the independent placement of glyphs can notice the text in the output. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut shown = marked(Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert!(!proof.passed()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("glyph(s) still show"), "{}", e.detail); + assert_eq!(proof.survivors(), 0, "no needle was involved"); + // The neighbour at y 200 is outside the region and is not reported. + assert!(!e.detail.contains("12 glyph"), "{}", e.detail); +} + +#[test] +fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { + // The secret drawn through a rotated/translated text matrix inside a scaled form: the + // placement has to follow `Tm` and the form's `/Matrix`, not the raw operands. + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix [1 0 0 1 0 0] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 1 0 0 1 10 100 Tm (SECRETVALUE) Tj ET", + ); + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut doc = marked_content(b"/Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("glyph(s) still show")); +} + +#[test] +fn an_ambiguous_needle_is_still_swept_outside_page_content() { + // The word also stays on the page legitimately (so page text, the raw file and decoded + // streams can only count it), but a copy in the Info dictionary is not page content. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let spec = Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }; + let mut doc = marked_content(content, spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); + + // Likewise an XMP packet and an annotation string. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked_content(content, Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::Xmp).verdict, Verdict::Survivor); + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (SECRETVALUE) >>".to_vec(); + let mut doc = marked_content(content, Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::ObjectStrings).verdict, Verdict::Survivor); +} + +#[test] +fn a_pcmark_key_in_the_input_does_not_exempt_a_stream() { + // A content stream that claims to be a redaction box paints inside the region, and was not + // created by this run: it counts as paint like any other. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut d = Dict::new(); + d.set(b"PCMark".to_vec(), Object::Name(b"Redaction".to_vec())); + append_content(&mut doc, d, b"0 0 0 rg 8 95 62 17 re f"); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn only_the_streams_this_run_created_are_exempt() { + // The same stream, declared as created by this run, is the box that was drawn on purpose. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let new = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"0 0 0 rg 8 95 62 17 re f".to_vec()))); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); + doc.require_full_save(); + snap.set_overlay_streams(&doc, &[new]).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn overlay_text_does_not_collide_with_what_was_removed() { + // "Account" is redacted and the label says "Account [redacted]": the label is not a survivor. + let content = b"BT /F1 10 Tf 10 100 Td (Account 1234) Tj ET"; + let mut doc = open(build_with(content, Spec::default())); + let shape = Shape::Redact { quads: vec![rect_quad([8.0, 95.0, 100.0, 112.0])], overlay: "Account [redacted]".into(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "T".into() }, &Meta::default()).unwrap(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); +} + +#[test] +fn removed_annotation_and_field_text_is_searched_for_too() { + let spec = || Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Subject (call back ANNOTWORDS) >>".to_vec()], ..Spec::default() }; + // Not asked for: the proof does not know about it. + let mut doc = marked(spec()); + let (_, proof) = run(&mut doc); + assert!(proof.passed(), "{}", proof.to_text()); + // Asked for: it is found where it was left. + let mut doc = marked(spec()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&[" call back ANNOTWORDS ".to_string(), "x".to_string(), String::new()]); + let result = applied(&mut doc, &mut snap); + assert!(result.is_ok() || residue(&result)); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + // And an absent string is clean. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&["never present anywhere".to_string()]); + applied(&mut doc, &mut snap).unwrap(); + assert!(snap.prove(&doc, None, &ProofOptions::default()).passed()); +} + +#[test] +fn a_damaged_stream_tail_is_unswept_not_truncated() { + let text: Vec = (0..4000u32).map(|i| b'a' + (i * 7 % 23) as u8).collect(); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + // Nothing was removed (the snapshot's region is blank), so the only issue is the stream. + let snap = Snapshot::capture(&marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()), None).unwrap(); + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + attach(&mut doc, Object::Stream(s)); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_damaged_page_content_tail_makes_the_page_unreadable() { + let text: Vec = b"BT /F1 10 Tf 10 100 Td (filler) Tj ET\n".repeat(200); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + let mut doc = marked(Spec::default()); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(s)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{}", proof.to_text()); +} + +#[test] +fn a_malformed_form_matrix_or_bbox_is_unswept() { + for dict in ["/Matrix [1 0 0]", "/Matrix [1 0 0 1 0 (x)]", "/BBox [0 0 300]"] { + let form = stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {dict}"), b"BT /F1 10 Tf 10 100 Td (x) Tj ET"); + let spec = Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }; + let mut doc = marked_content(b"/Fm1 Do", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{dict}: {}", proof.to_text()); + assert!(!proof.passed()); + assert!(proof.entries[0].detail.contains("malformed"), "{dict}: {}", proof.to_text()); + } +} + +#[test] +fn images_hidden_in_patterns_soft_masks_and_type3_glyphs_are_not_skipped() { + let image = "BI /W 1 /H 1 /CS /G /BPC 8 ID \u{ff} EI"; + let tile = + |data: &str| stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", data.as_bytes()); + let group = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency >> /Resources << >>", image.as_bytes()); + let proc = stream("", format!("1000 0 d0 {image}").as_bytes()); + let cases: Vec<(&str, Spec)> = vec![ + ("pattern", Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile(image)], ..Spec::default() }), + ( + "soft mask", + Spec { resources: "/ExtGState << /G1 << /SMask << /Type /Mask /S /Luminosity /G 7 0 R >> >> >>", extra: vec![group], ..Spec::default() }, + ), + ( + "type3", + Spec { + resources: "/Font << /F3 << /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 7 0 R >> /Encoding << /Type /Encoding /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] >> >>", + extra: vec![proc], + ..Spec::default() + }, + ), + ]; + for (what, spec) in cases { + // The region holds no text and no visible image: only the hidden image is in question. + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Unverifiable, "{what}: {}", proof.to_text()); + assert!(e.detail.contains("pattern, soft mask or Type3"), "{what}: {}", e.detail); + assert!(!proof.passed()); + } + // The same resources without an image are fine. + let spec = Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile("0 0 5 5 re f")], ..Spec::default() }; + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion, "{}", proof.to_text()); +} + +#[test] +fn the_sweep_is_bounded() { + // A CMap whose ranges rewrite the same 64Ki codes again and again costs work, whatever the map size. + let mut cmap = b"1 begincodespacerange <0000> endcodespacerange\n".to_vec(); + for _ in 0..200 { + cmap.extend_from_slice(b"100 beginbfrange\n"); + for _ in 0..100 { + cmap.extend_from_slice(b"<0000> <0041>\n"); + } + cmap.extend_from_slice(b"endbfrange\n"); + } + let start = std::time::Instant::now(); + assert!(!tests_support::cmap_is_complete(&cmap)); + assert!(start.elapsed().as_secs() < 20, "bounded work"); + // A small CMap is read in full. + assert!(tests_support::cmap_is_complete(b"1 beginbfchar <01> <0041> endbfchar")); + // Nested objects, cumulative decoding and searching all stop and say so. + assert!(tests_support::walks_in_full(60)); + assert!(!tests_support::walks_in_full(200)); + assert_eq!(tests_support::decode_budget_runs_out(), (true, true)); + assert_eq!(tests_support::search_budget_runs_out(), (true, true)); + // A decode that fails past the one-stream cap pays for the attempt: the same bomb cannot + // be decoded (and fail) again and again for free. + assert_eq!(tests_support::failed_decode_charges_the_budget(), (true, true, true)); +} + +#[test] +fn the_search_finds_what_is_there_whatever_the_prefix() { + assert!(tests_support::found("aaaab", b"xxaaaaaaab")); + assert!(tests_support::found("needle", b"needle")); + assert!(!tests_support::found("needle", b"needl")); + assert!(!tests_support::found("aaab", b"aaaaaaaaaa")); + assert!(tests_support::found("abc", b"zabc")); + // The worst shape for a skip-based search (a needle of one letter and a haystack full of + // it) is walked once per byte, and still finds what is there. + let needle = format!("{}b", "a".repeat(64)); + let mut hay = vec![b'a'; 20_000]; + assert!(!tests_support::found(&needle, &hay)); + hay.extend_from_slice(needle.as_bytes()); + assert!(tests_support::found(&needle, &hay)); +} + +#[test] +fn the_sweep_charge_stays_fair_for_a_bulk_of_needles() { + // 80 removed strings swept over one 5 MiB haystack (a stream decoded in full): the search + // really is one pass per needle, but the budget must not charge the haystack once per + // needle — that charge (haystack × needle × spelling: 80 × 20 × 5 MiB, about twice the 4 GiB + // work budget) would refuse an ordinary bulk redaction out of hand. The haystack is charged + // once per scan and a needle's own charge is capped, so the sweep stays under the budget. + let hay = vec![b'x'; 5 << 20]; + let word = "SECRETVALUE-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + assert_eq!(tests_support::bulk_counts(&hay, word, 80, 0), Some(0)); + // Genuinely unbounded work still fails closed: the region scanned is charged in full, so a + // haystack that no longer fits in the remaining budget is an unswept surface, never a pass. + assert_eq!(tests_support::bulk_counts(&hay, word, 80, crate::verify::tests_support::work_budget() - (1 << 20)), None); +} + +#[test] +fn survivors_seen_before_the_work_limit_are_still_reported() { + // The page sweep stops when the work budget runs out, but the survivors it has already + // counted are settled: a survivor on an earlier page is reported where it sits, not hidden + // behind the limit (the page still reads unswept, so the proof still fails). + let (reported, unswept) = tests_support::page_sweep_survivors_survive_the_limit(); + assert!(reported, "a survivor counted before the limit is reported"); + assert!(unswept, "the page the limit stopped on is unswept"); +} + +#[test] +fn a_file_past_the_proof_budget_is_unverified_never_clean() { + // A page whose content is bigger than the proof may decode for one stream: the budget + // stops the sweep, and the result must say so. The surfaces read unswept (a page content + // stream is never excused as raw-only), the region reads unverifiable, the proof fails. + // Work that stopped is "not verified", never a pass. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let content = Stream::flate(Dict::new(), &big); + drop(big); + let mut doc = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default())); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(content)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed(), "{}", proof.to_text()); + assert!(!proof.failures.is_empty(), "{}", proof.to_text()); + for s in [Surface::ExtractedText, Surface::DecodedStreams] { + let r = surface(&proof, s); + assert_eq!(r.verdict, Verdict::Unswept, "{s:?}: {}", proof.to_text()); + assert_eq!((r.survivors, r.raw_only), (0, 0), "{s:?}: {}", proof.to_text()); + } + assert!(matches!(proof.entries[0].status, Status::Unverifiable | Status::Failed), "the region is not verified: {}", proof.to_text()); +} + +#[test] +fn font_widths_parsing_shares_the_work_budget() { + // A form invoked again rebuilds its fonts, so the `/W` array is paid for from the shared + // work budget, not from a per-font counter that every rebuild resets. + let (full, refused) = tests_support::font_widths_charge_shares_the_budget(); + assert!(full, "a small /W array is read in full"); + assert!(refused, "a spent work budget refuses the /W parsing"); +} + +#[test] +fn an_unreadable_overlay_stream_is_its_own_error() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + let broken = doc.add(Object::Stream(Stream::from_raw(dict, b"not deflate data".to_vec()))); + let err = snap.set_overlay_streams(&doc, &[broken]).unwrap_err(); + assert_eq!(err, RedactError::OverlayUnreadable); + assert!(!err.to_string().contains("page 0"), "{err}"); +} + +// ── The caller's credentials ───────────────────────────────────────────────────────────────── + +#[test] +fn a_nonempty_user_password_document_is_proven_with_those_credentials() { + // The redacted document is protected with a user password and saved: the proof's subject is + // now an encrypted file, and only the caller's own credentials can read it back. Every + // supported encryption algorithm. + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "pw-secret", + owner_password: "owner-secret", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + + // The credentials the caller holds prove the saved output. + let proof = snap.prove(&doc, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // An already saved file too. + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // Empty and wrong credentials fail closed: the output can't be read back, which is an + // unswept proof, never a pass. + for pw in [None, Some("wrong")] { + let proof = snap.prove_saved(&saved, pw, &ProofOptions::default()); + assert!(!proof.passed(), "{alg:?} {pw:?}: {}", proof.to_text()); + for s in &proof.surfaces { + assert_eq!(s.verdict, Verdict::Unswept, "{alg:?} {pw:?}: {}", proof.to_text()); + } + } + } +} + +// ── Compressed copies inside binary streams ────────────────────────────────────────────────── + +/// A stream object whose stored bytes are the Flate encoding of `data` (as `Stream::flate` +/// writes them), with the extra dictionary entries `dict`. +fn flate_object(dict: &str, data: &[u8]) -> Vec { + stream(&format!("{dict} /Filter /FlateDecode"), &Stream::flate(Dict::new(), data).raw) +} + +/// A one-page file whose resources point at a fake font: /F2 is a /TrueType font whose +/// /FontDescriptor carries `font_file` as its /FontFile2 (objects 7, 8 and 9). The font carries +/// `/Widths` (object 6) so the interpreter resolves it; nothing on the page draws with it. +fn font_file_spec(font_file: Vec) -> Spec<'static> { + Spec { + resources: "/Font << /F1 5 0 R /F2 7 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /TrueType /BaseFont /X /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor 8 0 R >>".to_vec(), + b"<< /Type /FontDescriptor /FontName /X /Flags 4 /FontFile2 9 0 R >>".to_vec(), + font_file, + ], + ..Spec::default() + } +} + +#[test] +fn a_compressed_copy_in_a_fake_font_file_is_found() { + // The visible copy is redacted normally, but a Flate-compressed copy of the secret hides in + // a fake /FontFile2. /Length1 makes the sweep class the stream as a binary font program, so + // its stored bytes hold no plaintext: only the decoded data gives the copy away. + let spec = font_file_spec(flate_object("/Length1 900", SECRET.as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor, "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert!(s.locations.iter().any(|l| l.contains("(decoded)")), "{:?}", s.locations); + assert!(!proof.passed()); +} + +#[test] +fn a_legitimate_font_file_is_swept_and_clean() { + // A font program that holds none of what was removed (as real ones do not) is searched and + // lets the redaction pass: no false survivor. + let spec = font_file_spec(flate_object("/Length1 900", b"\x00\x01\x00\x00\x00\x10glyph tables")); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Clean, "{}", proof.to_text()); +} + +#[test] +fn a_font_program_that_will_not_decode_is_searched_as_stored_bytes() { + // An unrelated, damaged font program the page reaches is not a reason to refuse: nothing + // redacted touched it, so its stored bytes are searched and it is counted as raw only. + let raw = Stream::flate(Dict::new(), b"a font program without the secret").raw.as_ref().to_vec(); + let cut = raw.len() - 8; + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..cut])); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Clean, 1), "{}", proof.to_text()); +} + +/// The standard page plus an image (object 7) drawn away from the secret. +fn image_doc(image: Vec) -> Document { + let mut content = CONTENT.to_vec(); + content.extend_from_slice(b" q 20 0 0 20 200 200 cm /Im1 Do Q"); + marked_content(&content, Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }) +} + +const IMG: &str = "/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"; + +fn assert_redacted_cleanly(mut doc: Document) -> Proof { + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); + proof +} + +#[test] +fn an_unrelated_truncated_flate_image_does_not_fail_the_proof() { + let raw = Stream::flate(Dict::new(), &[0x80u8; 400]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_image_over_the_stream_cap_does_not_fail_the_proof() { + // Highly compressible: stored in a few dozen KiB, decoded past MAX_STREAM. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let raw = Stream::flate(Dict::new(), &big).raw.as_ref().to_vec(); + drop(big); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_truncated_font_program_does_not_fail_the_redaction() { + let raw = Stream::flate(Dict::new(), b"glyph tables").raw.as_ref().to_vec(); + let doc = marked(font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..raw.len() - 6]))); + assert_redacted_cleanly(doc); +} + +#[test] +fn a_secret_in_the_stored_bytes_of_a_font_program_is_still_found() { + // The font program cannot be decoded (it claims Flate but holds plain bytes), so only its + // stored bytes are searched: the copy of the secret in them is a survivor. + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.to_text().contains(SECRET)); + assert!(!proof.passed()); +} + +/// The standard page plus an /Indexed colour space whose lookup table is object 7. +fn indexed_doc(lookup: Vec) -> Document { + marked_content(CONTENT, Spec { resources: "/ColorSpace << /CS1 [/Indexed /DeviceRGB 255 7 0 R] >>", extra: vec![lookup], ..Spec::default() }) +} + +#[test] +fn an_indexed_lookup_stream_is_searched_as_stored_bytes_only() { + // Unrelated and damaged: nothing redacted touched it, so it must not fail the proof. + let raw = Stream::flate(Dict::new(), &[0x40u8; 300]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(indexed_doc(stream("/Filter /FlateDecode", &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); + // A secret stored raw inside one that will not decode is still found. + let mut doc = indexed_doc(stream("/Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +/// Capture a sound document, then damage the stream `num` (a Flate stream cut short, with the +/// extra dictionary entries `dict`) and prove it. +fn prove_damaged(mut doc: Document, num: u32, dict: &str, data: &[u8]) -> Proof { + let snap = Snapshot::capture(&doc, None).unwrap(); + let raw = Stream::flate(Dict::new(), data).raw.as_ref().to_vec(); + let cut = raw[..raw.len() - 8].to_vec(); + let broken = open(pdf_bytes(vec![b"<< /Type /Catalog >>".to_vec(), stream(&format!("{dict} /Filter /FlateDecode"), &cut)], "")); + let obj = broken.get(ObjRef::new(2, 0)).as_ref().clone(); + doc.set(ObjRef::new(num, 0), obj); + doc.require_full_save(); + snap.prove(&doc, None, &ProofOptions::default()) +} + +#[test] +fn damaged_content_appearance_and_metadata_still_fail_closed() { + // A truncated second content stream. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let proof = prove_damaged(doc, 7, "", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + assert!(!surface(&proof, Surface::DecodedStreams).problems.is_empty(), "{}", proof.to_text()); + + // A truncated appearance stream. + let annot = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 8 0 R >> >>".to_vec(); + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot, ap], ..Spec::default() }); + doc.require_full_save(); + let proof = prove_damaged(doc, 8, "/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + + // A truncated XMP packet. + let xmp = stream("/Type /Metadata /Subtype /XML", b"some metadata of the file"); + let doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let proof = prove_damaged(doc, 7, "/Type /Metadata /Subtype /XML", b"some metadata of the file"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::Xmp).problems.is_empty(), "{}", proof.to_text()); +} + +#[test] +fn an_annotation_appearance_is_checked_however_it_is_stored() { + // The region holds no text, so only the paint check can speak. An annotation written in + // place (a direct dictionary, no reference of its own) is checked like an indirect one, and + // a stream shared by two annotations is checked at each annotation's own placement, so the + // copy that lands over the region is seen even though the first placement does not reach it. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |spec: Spec<'_>| -> Proof { + let mut shown = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + + let proof = prove(Spec { + page: "/Annots [<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>]", + extra: vec![ap], + ..Spec::default() + }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + let far = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 7 0 R >> >>".to_vec(); + let near = b"<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>".to_vec(); + let proof = prove(Spec { page: "/Annots [8 0 R 9 0 R]", extra: vec![ap, far, near], ..Spec::default() }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); +} + +#[test] +fn a_null_entry_in_the_contents_array_is_an_empty_piece() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R null] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + assert_redacted_cleanly(doc); +} + +/// The page's decoded content streams, joined (what the redaction left on the page). +fn page_content(doc: &Document) -> String { + let p = &pdfcraft_model::pages(doc)[0]; + let (_, data) = page_streams(doc, &p.dict, 0).unwrap(); + data.iter().map(|d| String::from_utf8_lossy(d).into_owned()).collect::>().join("\n") +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs new file mode 100644 index 0000000000..f6db96229b --- /dev/null +++ b/crates/redact/src/verify/extract.rs @@ -0,0 +1,711 @@ +//! A minimal text extractor, independent of the interpreter's font metrics and hit test. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::util::{Ctx, latin1, resolve_strict, squash}; +use crate::annots_of; +use crate::verify::limits::{MAX_CMAP, MAX_CMAP_ENTRIES, MAX_CMAP_WORK, MAX_DEPTH}; + +/// Is this one of the glyph-carrying text-showing operators (`Tj`, `'`, `"`, and `TJ` with at +/// least one non-empty string)? A `TJ` of numbers only (the gap redaction leaves for a removed +/// string) moves the cursor but shows nothing, so it does not count. +pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { + let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); + match op.op.as_slice() { + b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), + b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + _ => false, + } +} + +/// A page's text as two whitespace-free views: decoded text, and the character codes as Latin-1 +/// (what fonts without a usable mapping show). +#[derive(Default)] +pub(super) struct Extracted { + pub(super) text: String, + pub(super) raw: String, + pub(super) glyph_ops: usize, + /// Glyph-carrying operators whose text could not be decoded. + pub(super) undecodable_ops: usize, + /// Per region asked about: glyphs whose centre this extractor places inside it. + pub(super) shown_in: Vec, + /// Glyph-carrying operators drawn where this extractor can't place them (tiling patterns, + /// or an annotation appearance without a usable `/Rect`). + pub(super) unplaced: usize, +} + +/// How one font turns codes into text. +#[derive(Clone, Default)] +pub(super) struct Decoder { + pub(super) map: HashMap, + /// Bytes per code. + pub(super) width: usize, + /// No way to read these codes as text (composite font without a mapping). + pub(super) opaque: bool, +} + +impl Decoder { + fn latin1() -> Self { + Decoder { map: HashMap::new(), width: 1, opaque: false } + } + + /// `Err` when the font's own data can't be read in full (a damaged or oversized CMap). + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let composite = font.name(b"Subtype") == Some(b"Type0"); + let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; + let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); + if let Some(Object::Stream(s)) = tu.as_deref() { + let data = ctx.decode(s)?; + if data.len() > MAX_CMAP { + return Err(()); + } + let (map, width) = parse_cmap(&data).ok_or(())?; + if !map.is_empty() { + d.map = map; + if let Some(w) = width { + d.width = w; + } + return Ok(d); + } + } + if composite { + d.opaque = true; + return Ok(d); + } + // Simple font: Latin-1, with the /Differences of the encoding. + if let Some(enc) = font.get(b"Encoding").map(|e| doc.resolve(e)) + && let Some(diff) = enc.as_dict().and_then(|e| e.get(b"Differences")).map(|x| doc.resolve(x)) + && let Some(items) = diff.as_array() + { + let mut code = 0u32; + for it in items { + match &*doc.resolve(it) { + Object::Int(i) => code = u32::try_from(*i).unwrap_or(0), + Object::Name(n) => { + if let Some(c) = std::str::from_utf8(n).ok().and_then(pdfcraft_fonts::pdf::glyph_unicode) { + d.map.insert(code, c.to_string()); + } + code = code.saturating_add(1); + } + _ => {} + } + } + } + Ok(d) + } + + /// Each code of a shown string, with the text it gives (`None`: no way to read it as text). + fn split(&self, s: &[u8]) -> Vec<(u32, Option)> { + let w = self.width.max(1); + s.chunks(w) + .map(|c| { + let code = code_of(c); + let text = if self.opaque { + None + } else { + match self.map.get(&code) { + Some(t) => Some(t.clone()), + None if self.map.is_empty() || w == 1 && code < 256 => Some(char::from(u8::try_from(code).unwrap_or(b'?')).to_string()), + None => None, + } + }; + (code, text) + }) + .collect() + } + + /// The text of a shown string. + fn decode(&self, s: &[u8]) -> String { + self.split(s).into_iter().filter_map(|(_, t)| t).collect() + } +} + +/// A font as the geometric check sees it: its decoder and its own simple width model. +#[derive(Clone)] +pub(super) struct Font { + pub(super) dec: Decoder, + /// Advance per code, in text space units per 1 em of font size (0.5 = half an em). + pub(super) widths: HashMap, + /// Advance of a code `widths` doesn't list. + pub(super) default_width: f64, + pub(super) vertical: bool, +} + +/// The width a font without usable `/Widths` is assumed to have (half an em). +pub(super) const FLAT_WIDTH: f64 = 0.5; + +impl Font { + fn flat() -> Self { + Font { dec: Decoder::latin1(), widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false } + } + + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let dec = Decoder::new(doc, font, ctx)?; + let mut f = Font { dec, widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false }; + let num = |o: &Object| doc.resolve(o).as_f64(); + if font.name(b"Subtype") == Some(b"Type0") { + f.vertical = font.get(b"Encoding").map(|e| doc.resolve(e)).and_then(|e| e.as_name().map(|n| n.ends_with(b"-V"))).unwrap_or(false); + let desc = font.get(b"DescendantFonts").map(|d| doc.resolve(d)).and_then(|d| d.as_array().and_then(|a| a.first().cloned())); + let desc = desc.and_then(|d| doc.resolve(&d).as_dict().cloned()); + if let Some(d) = desc { + f.default_width = d.get(b"DW").and_then(num).map_or(1.0, |w| w / 1000.0); + // The array is charged to the shared work budget as it is taken (a form + // invoked again rebuilds its fonts, so a per-font counter alone would let the + // same work be paid for many times), and every entry is paid for there too, + // whether or not it grows the map (a range may rewrite the same codes again + // and again, and the array is bounded only by the file). + let items: Vec = match d.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + Some(a) => { + if !ctx.charge(a.len()) { + return Err(()); + } + a + } + None => Vec::new(), + }; + let mut work = 0usize; + let mut step = |n: usize| -> Result<(), ()> { + work = work.saturating_add(n); + if work <= MAX_CMAP_WORK && ctx.charge(n) { Ok(()) } else { Err(()) } + }; + let mut i = 0; + while let Some(first) = items.get(i).and_then(num) { + let first = first.max(0.0) as u32; + match items.get(i + 1).map(|o| doc.resolve(o)) { + Some(a) if a.as_array().is_some() => { + step(a.as_array().map_or(0, Vec::len))?; + for (k, w) in a.as_array().into_iter().flatten().enumerate() { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w / 1000.0); + } + } + i += 2; + } + Some(last) => { + let (Some(last), Some(w)) = (last.as_f64(), items.get(i + 2).and_then(num)) else { break }; + let last = (last.max(0.0) as u32).min(first.saturating_add(0xFFFF)); + // Each code the range covers is paid for, grown or not (a span is at + // most 0x10000 codes, so the count always fits). + step(last.saturating_sub(first) as usize + 1)?; + for c in first..=last { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + f.widths.insert(c, w / 1000.0); + } + i += 3; + } + None => break, + } + } + } + } else { + // Glyph space units per em: 1000, or what a Type3 font's matrix says. + let scale = if font.name(b"Subtype") == Some(b"Type3") { + font.get(b"FontMatrix").map(|m| doc.resolve(m)).and_then(|m| m.as_array().and_then(|a| a.first().and_then(num))).unwrap_or(0.001) + } else { + 0.001 + }; + let first = font.get(b"FirstChar").and_then(num).map_or(0, |v| v.max(0.0) as u32); + let missing = font + .get(b"FontDescriptor") + .map(|d| doc.resolve(d)) + .and_then(|d| d.as_dict().and_then(|d| d.get(b"MissingWidth").and_then(num))) + .map(|w| w * scale); + if let Some(ws) = font.get(b"Widths").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + if ws.len() > MAX_CMAP_ENTRIES { + return Err(()); + } + for (k, w) in ws.iter().enumerate() { + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w * scale); + } + } + f.default_width = missing.unwrap_or(0.0); + } else if let Some(std) = pdfcraft_fonts::Std14::from_base_font(&String::from_utf8_lossy(font.name(b"BaseFont").unwrap_or(b""))) { + // No /Widths: a standard font has exact published widths, by WinAnsi code (Symbol, + // ZapfDingbats and other encodings are refused before the proof is reached). + for code in 0..=255u8 { + f.widths.insert(u32::from(code), std.width(code) / 1000.0); + } + f.default_width = if std.is_fixed_pitch() { 0.6 } else { 0.0 }; + } + } + Ok(f) + } + + fn width(&self, code: u32) -> f64 { + self.widths.get(&code).copied().unwrap_or(self.default_width) + } +} + +pub(super) fn hex_token(tok: &[u8]) -> Option> { + let digits: Vec = tok.iter().filter_map(|c| (*c as char).to_digit(16).map(|d| d as u8)).collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + Some(digits.chunks(2).map(|p| p.iter().fold(0u8, |a, d| (a << 4) | d)).collect()) +} + +pub(super) fn utf16_text(b: &[u8]) -> String { + if let [c] = b { + return char::from(*c).to_string(); + } + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) +} + +pub(super) fn code_of(b: &[u8]) -> u32 { + b.iter().fold(0u32, |a, x| a.wrapping_shl(8) | u32::from(*x)) +} + +pub(super) enum Tok { + Hex(Vec), + Word(String), + Open, + Close, +} + +pub(super) fn cmap_tokens(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = data.get(i) { + match c { + b'<' if data.get(i + 1) != Some(&b'<') => { + let end = data.get(i..).and_then(|d| d.iter().position(|x| *x == b'>')).map_or(data.len(), |e| i + e); + if let Some(h) = data.get(i + 1..end).and_then(hex_token) { + out.push(Tok::Hex(h)); + } + i = end + 1; + } + b'[' => { + out.push(Tok::Open); + i += 1; + } + b']' => { + out.push(Tok::Close); + i += 1; + } + b'%' => { + while data.get(i).is_some_and(|x| !matches!(x, b'\n' | b'\r')) { + i += 1; + } + } + c if c.is_ascii_alphabetic() => { + let start = i; + while data.get(i).is_some_and(|x| x.is_ascii_alphanumeric()) { + i += 1; + } + out.push(Tok::Word(String::from_utf8_lossy(data.get(start..i).unwrap_or_default()).into_owned())); + } + _ => i += 1, + } + } + out +} + +/// `bfchar` / `bfrange` entries of a ToUnicode CMap, and the code length of its codespace. +/// `None` when the CMap is bigger than [`MAX_CMAP_ENTRIES`] entries or costs more than +/// [`MAX_CMAP_WORK`] steps: a map that is cut short would read as "no such code". +pub(super) fn parse_cmap(data: &[u8]) -> Option<(HashMap, Option)> { + let toks = cmap_tokens(data); + let mut map: HashMap = HashMap::new(); + let mut width = None; + // Every step is paid for, whether or not it grows the map (a range may rewrite the same + // codes again and again). + let mut work = 0usize; + let mut step = |n: usize| -> Option<()> { + work = work.saturating_add(n); + (work <= MAX_CMAP_WORK).then_some(()) + }; + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + let Tok::Word(w) = t else { continue }; + match w.as_str() { + "begincodespacerange" => { + if let Some(Tok::Hex(h)) = toks.get(i) { + width = Some(h.len()); + } + } + "beginbfchar" => { + while let (Some(Tok::Hex(src)), Some(Tok::Hex(dst))) = (toks.get(i), toks.get(i + 1)) { + step(1)?; + map.insert(code_of(src), utf16_text(dst)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 2; + } + } + "beginbfrange" => { + while let (Some(Tok::Hex(lo)), Some(Tok::Hex(hi))) = (toks.get(i), toks.get(i + 1)) { + let (lo, hi) = (code_of(lo), code_of(hi)); + let span = hi.saturating_sub(lo).min(0xFFFF); + match toks.get(i + 2) { + Some(Tok::Hex(dst)) => { + let base = utf16_text(dst); + step(usize::try_from(span).ok()?.saturating_add(1))?; + for k in 0..=span { + let mut chars: Vec = base.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = char::from_u32(u32::from(*last).saturating_add(k)).unwrap_or('\u{fffd}'); + } + map.insert(lo.saturating_add(k), chars.into_iter().collect()); + } + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 3; + } + Some(Tok::Open) => { + i += 3; + let mut k = 0u32; + while let Some(Tok::Hex(d)) = toks.get(i) { + step(1)?; + map.insert(lo.saturating_add(k), utf16_text(d)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + k = k.saturating_add(1); + i += 1; + } + i += 1; // the closing bracket + } + _ => break, + } + } + } + _ => {} + } + } + Some((map, width)) +} + +pub(super) fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { + res.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default() +} + +pub(super) fn push_squashed(into: &mut String, s: &str) { + into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); +} + +/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error +/// (guessing the identity would put its content in the wrong place). +pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + match dict.get(b"Matrix") { + None => Ok(Matrix::IDENTITY), + Some(m) => { + let m = doc.resolve(m); + let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + } + } +} + +/// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. +pub(super) fn bbox_of(doc: &Document, dict: &Dict) -> Result, ()> { + let Some(b) = dict.get(b"BBox") else { return Ok(None) }; + let b = doc.resolve(b); + let v: Vec = + b.as_array().ok_or(())?.iter().map(|x| doc.resolve(x).as_f64().filter(|v| v.is_finite()).ok_or(())).collect::>()?; + match v[..] { + [a, b, c, d] => Ok(Some([a.min(c), b.min(d), a.max(c), b.max(d)])), + _ => Err(()), + } +} + +/// The content streams of a page as `(raw object, decoded)`, a missing or unreadable entry an +/// error. `skip_overlay` leaves out the streams this run drew the boxes and labels with. +pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: bool) -> Result, ()> { + let list: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*resolve_strict(doc, c)? { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut data: Vec = Vec::new(); + for o in &list { + match &*resolve_strict(doc, o)? { + Object::Stream(s) => { + let d = ctx.decode(s)?; + if skip_overlay && ctx.is_overlay(&d) { + continue; + } + data.extend(d); + data.push(b'\n'); + } + // A null entry is an empty piece, as for the redaction itself (ISO 32000-2 7.3.9). + Object::Null => {} + _ => return Err(()), + } + } + Ok(data) +} + +/// The text of a page: its content, the forms and patterns it reaches (visible or not), and the +/// appearance streams of its annotations; and which of `regions` still hold glyphs. +pub(super) fn extract_page(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> Result { + let data = page_content(doc, page, ctx, true)?; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut ex = Extractor { doc, ctx, regions, out: Extracted { shown_in: vec![0; regions.len()], ..Extracted::default() }, stack: Vec::new() }; + ex.scope(&data, &res, Matrix::IDENTITY, 0, true)?; + for a in annots_of(doc, page) { + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = + d.get(b"Rect").and_then(|r| doc.resolve(r).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect::>())); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec<(Option, Object)> = match &*s { + Object::Stream(_) => vec![(state.as_ref(), (*s).clone())], + Object::Dict(sub) => sub.iter().map(|(_, v)| (v.as_ref(), (*doc.resolve(v)).clone())).collect(), + _ => Vec::new(), + }; + for (r, o) in streams { + if let Object::Stream(st) = o { + // Where the appearance lands on the page (ISO 32000-2 §12.5.5), if it can be told. + let ctm = rect.as_deref().and_then(|r| appearance_matrix(doc, &st, r)); + // A font of the appearance is looked up in the appearance's own resources + // first (what the redaction itself does), then in the page's. + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + let parent = own.as_ref().unwrap_or(&res); + ex.stream(&st, r, parent, ctm.unwrap_or_default(), 1, ctm.is_some())?; + } + } + } + } + Ok(ex.out) +} + +/// The matrix that maps an appearance stream's form space onto an annotation's `/Rect` +/// (ISO 32000-2 §12.5.5): the appearance's `/Matrix`, then the scale that fits the mapped +/// `/BBox` into the rectangle. `None` when that can't be told (no usable `/Rect`, `/BBox` or +/// `/Matrix`, or a mapped box that degenerates). +pub(crate) fn appearance_matrix(doc: &Document, ap: &Stream, rect: &[f64]) -> Option { + let [rx0, ry0, rx1, ry1] = rect[..] else { return None }; + let bbox = bbox_of(doc, &ap.dict).ok()??; + let m = matrix_of(doc, &ap.dict).ok()?; + let t = m.bbox(bbox); + let (tw, th) = (t[2] - t[0], t[3] - t[1]); + if tw <= 0.0 || th <= 0.0 { + return None; + } + let (rx0, rx1, ry0, ry1) = (rx0.min(rx1), rx0.max(rx1), ry0.min(ry1), ry0.max(ry1)); + let (sx, sy) = ((rx1 - rx0) / tw, (ry1 - ry0) / th); + let a = Matrix([sx, 0.0, 0.0, sy, rx0 - t[0] * sx, ry0 - t[1] * sy]); + Some(m.then(&a)) +} + +/// Text-state parameters that `q`/`Q` save and restore. +#[derive(Clone)] +pub(super) struct TextGs { + pub(super) ctm: Matrix, + pub(super) tc: f64, + pub(super) tw: f64, + pub(super) th: f64, + pub(super) tl: f64, + pub(super) rise: f64, + pub(super) size: f64, + pub(super) font: Vec, +} + +pub(super) struct Extractor<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) out: Extracted, + pub(super) stack: Vec, +} + +impl Extractor<'_> { + /// A form or pattern stream. `ctm` maps its space onto the page's; `placed` is false where + /// that mapping is not known. + fn stream(&mut self, s: &Stream, id: Option, parent: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if let Some(r) = id { + if self.stack.contains(&r) { + return Ok(()); + } + self.stack.push(r); + } + let result = (|| { + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) + })(); + if id.is_some() { + self.stack.pop(); + } + result + } + + /// Show the strings of one text-showing operator: extract their text, and place each glyph + /// to count the ones inside a region. `tm` moves along with the pen. + fn show_text(&mut self, font: &Font, op: &pdfcraft_content::Op, gs: &TextGs, tm: &mut Matrix, placed: bool) { + let o = op.op.as_slice(); + // Strings and TJ adjustments, in order. + let items: Vec<&Object> = match o { + b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => op.operands.last().into_iter().collect(), + }; + let mut any_text = false; + for it in items { + if let Some(n) = it.as_f64() { + let d = -n / 1000.0 * gs.size; + let m = if font.vertical { Matrix::translate(0.0, d) } else { Matrix::translate(d * gs.th, 0.0) }; + *tm = m.then(tm); + continue; + } + let Some(s) = it.as_string() else { continue }; + let t = font.dec.decode(&s.bytes); + any_text |= !squash(&t).is_empty(); + push_squashed(&mut self.out.text, &t); + push_squashed(&mut self.out.raw, &latin1(&s.bytes)); + for (code, text) in font.dec.split(&s.bytes) { + let w = font.width(code); + let space = text.as_deref().is_some_and(|t| t == " "); + // The glyph's centre, in text space (the box runs from 0.2 em below the + // baseline to 0.8 em above), and where the pen moves to. + let (centre, advance) = if font.vertical { + let adv = -gs.size + gs.tc + if space { gs.tw } else { 0.0 }; + ((0.0, gs.rise - 0.5 * gs.size), Matrix::translate(0.0, adv)) + } else { + let adv = (w * gs.size + gs.tc + if space { gs.tw } else { 0.0 }) * gs.th; + ((0.5 * w * gs.size * gs.th, gs.rise + 0.3 * gs.size), Matrix::translate(adv, 0.0)) + }; + let visible = text.as_deref().is_none_or(|t| !squash(t).is_empty()); + if placed && visible && !self.regions.is_empty() { + let (x, y) = tm.then(&gs.ctm).apply(centre.0, centre.1); + for (r, n) in self.regions.iter().zip(self.out.shown_in.iter_mut()) { + if x >= r[0] && x <= r[2] && y >= r[1] && y <= r[3] { + *n += 1; + } + } + } + *tm = advance.then(tm); + } + } + if carries_glyphs(op) { + self.out.glyph_ops += 1; + if !any_text { + self.out.undecodable_ops += 1; + } + if !placed { + self.out.unplaced += 1; + } + } + } + + fn scope(&mut self, data: &[u8], res: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if depth > MAX_DEPTH { + return Err(()); + } + let doc = self.doc; + let fonts = res_dict(doc, res, b"Font"); + let xobjects = res_dict(doc, res, b"XObject"); + let mut cache: HashMap, Font> = HashMap::new(); + let mut gs = TextGs { ctm, tc: 0.0, tw: 0.0, th: 1.0, tl: 0.0, rise: 0.0, size: 0.0, font: Vec::new() }; + let mut saved: Vec = Vec::new(); + let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); + for op in parse(data).ops { + let o = op.op.as_slice(); + match o { + b"q" => { + if saved.len() >= 4096 { + return Err(()); + } + saved.push(gs.clone()); + } + b"Q" => { + if let Some(g) = saved.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"BT" => { + tm = Matrix::IDENTITY; + tlm = Matrix::IDENTITY; + } + b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + b"Tf" => { + gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num(1).unwrap_or(0.0); + } + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if o == b"TD" { + gs.tl = -y; + } + tlm = Matrix::translate(x, y).then(&tlm); + tm = tlm; + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + tm = Matrix(m); + tlm = tm; + } + } + b"T*" => { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + b"Tj" | b"TJ" | b"'" | b"\"" => { + if matches!(o, b"'" | b"\"") { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + if o == b"\"" + && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + { + (gs.tw, gs.tc) = (aw, ac); + } + if !cache.contains_key(&gs.font) { + let f = match fonts.get(&gs.font).map(|f| doc.resolve(f)) { + Some(f) => f.as_dict().map_or_else(|| Ok(Font::flat()), |d| Font::new(doc, d, self.ctx))?, + None => Font::flat(), + }; + cache.insert(gs.font.clone(), f); + } + let Some(font) = cache.get(&gs.font) else { continue }; + self.show_text(font, &op, &gs, &mut tm, placed); + } + b"Do" => { + let Some(name) = op.name(0) else { continue }; + let Some(entry) = xobjects.get(name) else { continue }; + if let Object::Stream(s) = &*doc.resolve(entry) + && s.dict.name(b"Subtype") == Some(b"Form") + { + let m = matrix_of(doc, &s.dict)?; + bbox_of(doc, &s.dict)?; + self.stream(s, entry.as_ref(), res, m.then(&gs.ctm), depth + 1, placed)?; + } + } + _ => {} + } + } + // Tiling patterns draw content of their own, repeated where the pattern is used: its + // glyphs can't be placed. + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + if let Object::Stream(s) = &*doc.resolve(p) { + self.stream(s, p.as_ref(), res, Matrix::IDENTITY, depth + 1, false)?; + } + } + Ok(()) + } +} diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs new file mode 100644 index 0000000000..fb69f91351 --- /dev/null +++ b/crates/redact/src/verify/geometry.rs @@ -0,0 +1,500 @@ +//! The non-text content under the regions: images, paths and shadings, and the placement of the +//! glyphs still shown, checked independently of the interpreter. + +use std::collections::HashSet; + +use pdfcraft_content::{Matrix, contains, overlaps, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::extract::{appearance_matrix, bbox_of, matrix_of, page_content, res_dict}; +use super::util::{Ctx, image_codec}; +use crate::verify::limits::{MAX_CLIP_SUBPATHS, MAX_DEPTH, MAX_GS_STACK, MAX_HOLES, MAX_PIXELS}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum Finding { + /// This many glyphs are placed inside the region. + Text(usize), + Image, + Path, + Shading, + Missing, + Unverifiable(&'static str), +} + +#[derive(Clone)] +pub(super) struct Gs { + pub(super) ctm: Matrix, + /// Device-space boxes cut out of the clip (even-odd clips with a hole). + pub(super) holes: Vec<[f64; 4]>, +} + +pub(super) struct Geo<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) found: Vec<(usize, Finding)>, + /// What `found` already holds: each finding is recorded once per region, so a page that + /// paints thousands of times cannot grow the report (or the memory) with repeats. + reported: HashSet<(usize, Finding)>, + /// Pattern, soft-mask and Type3 resources already searched for images. + pub(super) searched: HashSet, +} + +/// What still paints inside `regions` on a page of the output (the overlay boxes excepted): +/// images with pixels there, paths and shadings not clipped away from there. +pub(super) fn check_geometry(doc: &Document, page: &Dict, regions: &[[f64; 4]], ctx: &Ctx) -> Vec<(usize, Finding)> { + let unverifiable = |why: &'static str| (0..regions.len()).map(|i| (i, Finding::Unverifiable(why))).collect(); + // The redaction boxes are painted on purpose. + let Ok(data) = page_content(doc, page, ctx, true) else { return unverifiable("the page content can't be read in full") }; + let mut geo = Geo { doc, ctx, regions, found: Vec::new(), reported: HashSet::new(), searched: HashSet::new() }; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + geo.run(&data, &res, Matrix::IDENTITY, Vec::new(), 0); + geo.appearances(page, &res); + geo.found +} + +impl Geo<'_> { + /// Record a finding for region `i`, once: a page that paints over a region a thousand + /// times is one finding, not a thousand. + fn report(&mut self, i: usize, f: Finding) { + if self.reported.insert((i, f.clone())) { + self.found.push((i, f)); + } + } + + /// The appearances of the annotations still on the page, checked like the page itself: their + /// content is placed with the appearance matrix (ISO 32000-2 §12.5.5) and swept for images, + /// paths and shadings under the regions. An appearance that has content but can't be placed + /// makes every region unverifiable: the redaction refuses such documents, so one in the + /// output means what it paints can't be vouched for. + fn appearances(&mut self, page: &Dict, res: &Dict) { + let doc = self.doc; + for a in crate::annots_of(doc, page) { + // A direct dictionary has no reference of its own; it is checked like an indirect + // one (what its appearance paints does not depend on how it is stored). + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + // A redaction mark's appearance outlines the marked area on purpose (and applying + // removes the mark whole): it is exempt like the boxes drawn over the regions. + if d.name(b"Subtype") == Some(b"Redact") { + continue; + } + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = crate::rect_of(doc, d.get(b"Rect")); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec = match &*s { + Object::Stream(st) => vec![st.clone()], + Object::Dict(sub) => sub + .iter() + .filter_map(|(_, v)| match &*doc.resolve(v) { + Object::Stream(st) => Some(st.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + for st in streams { + // A stream shared by several annotations is checked at each annotation's + // own placement: where it lands comes from each `/Rect`, so the first + // placement tells nothing about the others. Repeated decodes spend the + // same bounded budget as everything else. + let Ok(data) = self.ctx.decode(&st) else { + self.unverifiable("an annotation appearance can't be read in full"); + continue; + }; + if data.is_empty() { + continue; + } + let Some(ctm) = rect.as_ref().and_then(|r| appearance_matrix(doc, &st, r)) else { + self.unverifiable("an annotation appearance can't be placed, so what it paints can't be checked"); + continue; + }; + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(res), ctm, Vec::new(), 1); + } + } + } + } + + fn touching(&self, b: [f64; 4]) -> Vec { + (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| overlaps(*r, b, 0.0))).collect() + } + + /// Does something painting over `within` reach region `i` through the clip? Only the part of + /// the region the painting could touch has to be cut out. + fn unclipped(&self, i: usize, within: [f64; 4], gs: &Gs) -> bool { + let Some(r) = self.regions.get(i) else { return false }; + let reach = [r[0].max(within[0]), r[1].max(within[1]), r[2].min(within[2]), r[3].min(within[3])]; + !gs.holes.iter().any(|h| contains(*h, reach, 0.01)) + } + + fn run(&mut self, data: &[u8], res: &Dict, ctm: Matrix, holes: Vec<[f64; 4]>, depth: usize) { + if depth > MAX_DEPTH { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable("forms are nested too deep to check")); + } + return; + } + let xobjects = res_dict(self.doc, res, b"XObject"); + self.hidden_images(res, depth); + let mut gs = Gs { ctm, holes }; + let mut stack: Vec = Vec::new(); + // The current path in device space: its subpaths, and the clip operator it carries. + let mut sub: Vec> = Vec::new(); + let mut clip: Option = None; + for op in parse(data).ops { + let o = op.op.as_slice(); + if matches!(o, b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" | b"W" | b"W*") { + extend_path(&op, &gs.ctm, &mut sub, &mut clip); + continue; + } + if matches!(o, b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n") { + self.paint_path(o, &mut gs, &sub, clip); + sub.clear(); + clip = None; + continue; + } + sub.clear(); + clip = None; + match o { + b"q" => { + // The saved state carries the holes with it, so an unbounded stack of them + // is unbounded memory: past the cap the walk says so instead of growing. + if stack.len() >= MAX_GS_STACK { + return self.unverifiable("the content stacks graphics states too deeply to check"); + } + stack.push(gs.clone()); + } + b"Q" => { + if let Some(g) = stack.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"sh" => { + for i in 0..self.regions.len() { + if self.unclipped(i, [f64::MIN, f64::MIN, f64::MAX, f64::MAX], &gs) { + self.report(i, Finding::Shading); + } + } + } + b"BI" => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + self.report(i, Finding::Image); + } + } + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let obj = self.doc.resolve(entry); + let Object::Stream(s) = &*obj else { continue }; + match s.dict.name(b"Subtype") { + Some(b"Image") => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + let Some(region) = self.regions.get(i) else { continue }; + match image_cleared(self.doc, self.ctx, s, &gs.ctm, *region) { + Ok(true) => {} + Ok(false) => self.report(i, Finding::Image), + Err(()) => self.report(i, Finding::Unverifiable("an image under the region can't be inspected")), + } + } + } + Some(b"Form") => self.form(s, &gs, res, depth), + _ => {} + } + } + _ => {} + } + } + } + + /// A path-painting operator `o` ends the current path: report what it paints over a region, + /// and let an even-odd clip add its holes to `gs`. + fn paint_path(&mut self, o: &[u8], gs: &mut Gs, sub: &[Vec<(f64, f64)>], clip: Option) { + let pts: Vec<(f64, f64)> = sub.iter().flatten().copied().collect(); + let bbox = pts.iter().fold(None, |b: Option<[f64; 4]>, &(u, v)| { + Some(match b { + None => [u, v, u, v], + Some(b) => [b[0].min(u), b[1].min(v), b[2].max(u), b[3].max(v)], + }) + }); + if o != b"n" + && let Some(b) = bbox + { + for i in self.touching(b) { + if self.unclipped(i, b, gs) { + self.report(i, Finding::Path); + } + } + } + // An even-odd clip whose subpaths are all axis-aligned rectangles lets through what an + // odd number of the inner rectangles covers. Where a region lies under the outer box + // and an odd number of inner ones, it is cut out of the clip; under an even number it + // is painted and must be reported. Anything that is not a stock of disjoint rectangles + // (a triangle, a rotated box, overlapping inners, too many of them) cannot be told + // cheaply, so no hole is cut: what the clip lets through is then reported as paint + // (fail closed). This is how redaction itself clips regions out of what it leaves. + if clip != Some(true) { + return; + } + let Some(rects) = sub.iter().map(|s| axis_aligned_rect(s)).collect::>>() else { return }; + if rects.len() < 2 || rects.len() > MAX_CLIP_SUBPATHS { + return; + } + let (outer, inners) = (rects[0], &rects[1..]); + // Inner rectangles must be disjoint, or their overlaps would change the parity the + // reader draws with. + let disjoint = inners.iter().enumerate().all(|(x, a)| inners.iter().skip(x + 1).all(|b| !overlaps(*a, *b, 0.0))); + let regions: Vec<[f64; 4]> = self.regions.to_vec(); + let mut cut = Vec::new(); + for r in regions { + if contains(outer, r, 0.0) && disjoint && inners.iter().filter(|b| contains(**b, r, 0.0)).count() % 2 == 1 { + cut.push(r); + } + } + if gs.holes.len() + cut.len() > MAX_HOLES { + return self.unverifiable("the content cuts clips with too many pieces to check"); + } + gs.holes.extend(cut); + } + + fn unverifiable(&mut self, why: &'static str) { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable(why)); + } + } + + fn form(&mut self, s: &Stream, gs: &Gs, parent: &Dict, depth: usize) { + let (Ok(m), Ok(bbox)) = (matrix_of(self.doc, &s.dict), bbox_of(self.doc, &s.dict)) else { + self.unverifiable("a form under the region has a malformed /Matrix or /BBox"); + return; + }; + let fctm = m.then(&gs.ctm); + if let Some(bb) = bbox + && self.touching(fctm.bbox(bb)).is_empty() + { + return; + } + let Ok(data) = self.ctx.decode(s) else { + self.unverifiable("a form under the region can't be decoded in full"); + return; + }; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(parent), fctm, gs.holes.clone(), depth + 1); + } + + /// Images drawn from tiling patterns, soft-mask groups and Type3 glyph procedures can't be + /// placed or cleared by this check: if `res` reaches any, every region is unverifiable. + fn hidden_images(&mut self, res: &Dict, depth: usize) { + let doc = self.doc; + let mut streams: Vec<(Option, Stream)> = Vec::new(); + let take = |o: &Object, streams: &mut Vec<(Option, Stream)>| { + if let Object::Stream(s) = &*doc.resolve(o) { + streams.push((o.as_ref(), s.clone())); + } + }; + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + take(p, &mut streams); + } + for (_, g) in res_dict(doc, res, b"ExtGState").iter() { + if let Some(Object::Dict(sm)) = doc.resolve(g).as_dict().and_then(|g| g.get(b"SMask")).map(|m| (*doc.resolve(m)).clone()).as_ref() + && let Some(group) = sm.get(b"G") + { + take(group, &mut streams); + } + } + for (_, f) in res_dict(doc, res, b"Font").iter() { + let Some(font) = doc.resolve(f).as_dict().cloned() else { continue }; + if font.name(b"Subtype") != Some(b"Type3") { + continue; + } + if let Some(procs) = font.get(b"CharProcs").and_then(|c| doc.resolve(c).as_dict().cloned()) { + for (_, p) in procs.iter() { + take(p, &mut streams); + } + } + } + for (id, s) in streams { + if let Some(r) = id + && !self.searched.insert(r) + { + continue; + } + match self.has_image(&s, res, depth + 1) { + Ok(false) => {} + Ok(true) => self.unverifiable("an image inside a pattern, soft mask or Type3 glyph can't be placed or cleared"), + Err(()) => self.unverifiable("a pattern, soft mask or Type3 glyph can't be read in full"), + } + } + } + + /// Does the content of `s` (forms it draws included) draw an image? + fn has_image(&mut self, s: &Stream, parent: &Dict, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(()); + } + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + let res = own.as_ref().unwrap_or(parent); + let xobjects = res_dict(self.doc, res, b"XObject"); + for op in parse(&data).ops { + match op.op.as_slice() { + b"BI" => return Ok(true), + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; + match x.dict.name(b"Subtype") { + Some(b"Image") => return Ok(true), + Some(b"Form") => { + if let Some(r) = entry.as_ref() + && !self.searched.insert(r) + { + continue; + } + if self.has_image(x, res, depth + 1)? { + return Ok(true); + } + } + _ => {} + } + } + _ => {} + } + } + Ok(false) + } +} + +/// Extend the current path (`sub`: its subpaths in device space; `clip`: the clip operator it +/// carries) by one construction operator. +fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { + let o = op.op.as_slice(); + let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); + let at = |x: f64, y: f64| ctm.apply(x, y); + match o { + b"m" => { + if let [x, y] = nums[..] { + sub.push(vec![at(x, y)]); + } + } + b"l" | b"c" | b"v" | b"y" => { + if let Some(cur) = sub.last_mut() { + cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + } + } + b"re" => { + if let [x, y, w, h] = nums[..] { + sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); + } + } + b"W" => *clip = Some(false), + b"W*" => *clip = Some(true), + _ => {} + } +} + +/// The axis-aligned rectangle a four-corner subpath draws, in device space: `None` for anything +/// else (open paths, more or fewer corners, skewed or rotated boxes, non-finite coordinates). +fn axis_aligned_rect(pts: &[(f64, f64)]) -> Option<[f64; 4]> { + // `re` leaves its rectangle open: the four corners are the whole subpath. A rectangle drawn + // by hand closes by repeating the first point at the end. Both spell the same four corners. + let pts = match pts { + [a, _, _, _, e] if a == e => &pts[..4], + _ => pts, + }; + let [p0, p1, p2, p3] = pts else { return None }; + let finite = |p: &(f64, f64)| p.0.is_finite() && p.1.is_finite(); + if !finite(p0) || !finite(p1) || !finite(p2) || !finite(p3) { + return None; + } + // The edges alternate between horizontal and vertical, whichever corner they start from and + // whichever way the rectangle is wound. + let flat = |a: &(f64, f64), b: &(f64, f64)| a.1 == b.1 && a.0 != b.0; + let steep = |a: &(f64, f64), b: &(f64, f64)| a.0 == b.0 && a.1 != b.1; + let even = flat(p0, p1) && steep(p1, p2) && flat(p2, p3) && steep(p3, p0); + let odd = steep(p0, p1) && flat(p1, p2) && steep(p2, p3) && flat(p3, p0); + if !even && !odd { + return None; + } + let (x0, x1) = (p0.0.min(p2.0), p0.0.max(p2.0)); + let (y0, y1) = (p0.1.min(p2.1), p0.1.max(p2.1)); + (x0 < x1 && y0 < y1).then_some([x0, y0, x1, y1]) +} + +/// Are the samples of image `s` whose centres fall in `region` all clear (zero; for image masks, +/// the sample that does not paint)? `Err` when the image can't be inspected. +#[allow(clippy::result_unit_err)] +pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, region: [f64; 4]) -> Result { + if image_codec(&s.dict) { + // Still encoded pixels: nothing was cleared. + return Ok(false); + } + let (w, h) = (s.dict.int(b"Width").ok_or(())?, s.dict.int(b"Height").ok_or(())?); + if w <= 0 || h <= 0 || (w as u64).saturating_mul(h as u64) > MAX_PIXELS { + return Err(()); + } + let (w, h) = (usize::try_from(w).map_err(|_| ())?, usize::try_from(h).map_err(|_| ())?); + let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + let (ncomp, bpc) = if mask { + (1, 1) + } else { + let cs = doc.resolve(s.dict.get(b"ColorSpace").ok_or(())?); + let name = match &*cs { + Object::Name(n) => n.clone(), + Object::Array(a) => a.first().and_then(Object::as_name).ok_or(())?.to_vec(), + _ => return Err(()), + }; + let n = match name.as_slice() { + b"DeviceGray" | b"G" | b"CalGray" | b"Indexed" | b"I" | b"Separation" => 1, + b"DeviceRGB" | b"RGB" | b"CalRGB" | b"Lab" => 3, + b"DeviceCMYK" | b"CMYK" => 4, + // One component per colorant name. + b"DeviceN" => doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?).as_array().ok_or(())?.len(), + b"ICCBased" => match &*doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?) { + Object::Stream(icc) => usize::try_from(icc.dict.int(b"N").ok_or(())?).map_err(|_| ())?, + _ => return Err(()), + }, + _ => return Err(()), + }; + (n, usize::try_from(s.dict.int(b"BitsPerComponent").ok_or(())?).map_err(|_| ())?) + }; + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { + return Err(()); + } + let data = ctx.decode(s)?; + let bits = ncomp.checked_mul(bpc).ok_or(())?; + let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); + if data.len() < row.checked_mul(h).ok_or(())? { + return Err(()); + } + let inverted = s.dict.get(b"Decode").is_some_and(|d| { + let d = doc.resolve(d); + d.as_array().and_then(|a| a.first()).is_some_and(|first| doc.resolve(first).as_f64() == Some(1.0)) + }); + let want = u8::from(mask && !inverted); + for j in 0..h { + let v = 1.0 - (j as f64 + 0.5) / h as f64; + // Rows that don't reach the region hold nothing to check. + let ((ax, ay), (bx, by)) = (ctm.apply(0.0, v), ctm.apply(1.0, v)); + if !overlaps(region, [ax.min(bx), ay.min(by) - 0.01, ax.max(bx), ay.max(by) + 0.01], 0.0) { + continue; + } + for i in 0..w { + let (x, y) = ctm.apply((i as f64 + 0.5) / w as f64, v); + if x < region[0] || x > region[2] || y < region[1] || y > region[3] { + continue; + } + let start = j.checked_mul(row).and_then(|v| v.checked_mul(8)).and_then(|v| v.checked_add(i.checked_mul(bits)?)).ok_or(())?; + for bit in start..start.checked_add(bits).ok_or(())? { + let byte = data.get(bit / 8).ok_or(())?; + if (byte >> (7 - bit % 8)) & 1 != want { + return Ok(false); + } + } + } + } + Ok(true) +} diff --git a/crates/redact/src/verify/limits.rs b/crates/redact/src/verify/limits.rs new file mode 100644 index 0000000000..e2972bb5f6 --- /dev/null +++ b/crates/redact/src/verify/limits.rs @@ -0,0 +1,42 @@ +//! Bounds of the redaction proof (`verify`). Hitting one is a failure the proof reports, never +//! a quiet skip: a proof that cannot sweep everything it must is not a passing proof. + +/// Most decoded bytes of one stream the proof reads. +pub(crate) const MAX_STREAM: usize = 64 << 20; +/// Most decoded bytes the whole proof reads, over every stream and every pass. +pub(crate) const MAX_TOTAL_DECODED: u64 = 2 << 30; +/// Most units of linear search work (input, patterns and reported matches) in one proof. +pub(crate) const MAX_WORK: u64 = 4 << 30; +/// Nested attachment levels that are swept (deeper ones are reported as unswept). +pub(crate) const MAX_NESTING: usize = 4; +/// Nesting bound of object walks (deeper is reported, never skipped silently). +pub(crate) const MAX_WALK: usize = 64; +/// Deepest graphics-state stack the paint check follows (saved states carry their clip holes +/// with them, so an unbounded stack is unbounded memory). Deeper is unverifiable. +pub(crate) const MAX_GS_STACK: usize = 4096; +/// Most clip holes one walk may cut (each is kept until the state that made it is restored). +/// More is unverifiable. +pub(crate) const MAX_HOLES: usize = 4096; +/// Most subpaths of one even-odd clip the hole rule examines (it compares them pairwise). +/// More, and no hole is cut: what the clip lets through is reported as paint. +pub(crate) const MAX_CLIP_SUBPATHS: usize = 64; +/// Most decoded bytes of an embedded CMap. +pub(crate) const MAX_CMAP: usize = 4 << 20; +/// Most CMap range steps one font may cost (a range that never grows the map still costs). +pub(crate) const MAX_CMAP_WORK: usize = 4 << 20; +/// Most entries one ToUnicode CMap may define. +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 20; +/// Needles shorter than this (in characters) are too common to look for in raw bytes. +pub(crate) const MIN_NEEDLE: usize = 3; +/// Smallest needle looked for in the raw file and in decoded streams. Those hold object syntax, +/// numbers and binary data, where three characters match by chance (`/Length 612`, a +/// `/MediaBox`, pixels); shorter needles are still swept on page text, strings, names, Info, XMP +/// and attachment text. +pub(crate) const MIN_STRUCTURED_NEEDLE: usize = 6; +/// Deepest nesting of forms, patterns and appearance streams the proof's independent walks +/// follow: the bound the interpreter itself walks forms at, so the proof reaches everything a +/// redaction could have removed and the two cannot drift apart. +pub(crate) const MAX_DEPTH: usize = crate::interp::MAX_DEPTH; +/// Most pixels an image may have for the proof to inspect it (larger ones are removed by +/// [`apply`](crate::apply), and the proof's pixel look has the same ceiling: `crate::image`). +pub(crate) const MAX_PIXELS: u64 = crate::image::MAX_PIXELS; diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs new file mode 100644 index 0000000000..b374f93fdc --- /dev/null +++ b/crates/redact/src/verify/mod.rs @@ -0,0 +1,389 @@ +//! An independent proof that redaction removed what it claimed to remove. +//! +//! [`apply`](crate::apply) cuts glyphs out with its own interpreter, so asking that interpreter +//! whether anything is left would only repeat its own mistakes. The proof therefore works from +//! the other end, and does not trust the interpreter's font metrics or hit test: +//! +//! 1. **Before** applying, [`Snapshot::capture`] records, per redaction region, the glyph runs the +//! interpreter is about to remove (their character codes and the text the font gives them), a +//! digest of each page's content, and the text of the whole document as a separate extractor +//! sees it. The strings the sweep looks for therefore *start* from what the interpreter +//! removed; a glyph it wrongly kept is caught by the geometric check below, not by a needle. +//! 2. **After** applying, [`Snapshot::prove`] serializes the document and sweeps the *output* +//! for those strings on every surface a reader could recover them from: the raw file, every +//! string object, every decoded stream (orphans included), the text of every page (read by a +//! minimal extractor of its own, not the interpreter's font metrics or hit test), the Info +//! dictionary, the XMP packet, embedded files (PDF attachments are swept recursively) and the +//! revision structure. Each string is looked for in every encoding a PDF can hold it in. +//! 3. It places every glyph still shown on the redacted pages with its own text-state tracking +//! and width model (the font's `/Widths`, else a flat guess) and fails a region when the +//! centre of a surviving glyph lies inside it, whatever the interpreter decided. +//! 4. It also re-checks the non-text content under each region — on the page and in the +//! appearance streams of the annotations left on it: no image still carries pixels there, and +//! no path or shading paints there unclipped. +//! +//! The proof fails closed: a survivor, but also anything it could not sweep (page content, a form +//! or an appearance stream, XMP, or an attachment that won't decode in full or can't be opened; a +//! nesting or work limit hit; an appearance with content that can't be placed on its page), makes +//! it fail. The one exception is binary data that redaction +//! never edits: a font program, colour profile, `/Indexed` lookup table, function sample table or +//! Flate/LZW image that is damaged or too long to decode is searched as stored bytes and reported +//! as `raw_only`, not as unswept. A proof that cannot fail is worthless. +//! +//! Work is bounded: every stream is decoded strictly under `limits::MAX_STREAM`, all decodes together +//! under `limits::MAX_TOTAL_DECODED`, and all searching under `limits::MAX_WORK`. A document that needs more +//! is reported as unswept, not skipped (binary data as above excepted). +//! +//! The result is a [`Proof`]: a per-region manifest (page, region, glyph counts, SHA-256 of +//! the page content after) and a verdict per surface. It holds counts and digests, +//! never the redacted text, unless [`ProofOptions::include_plaintext`] asks for it. +//! +//! Limits, stated plainly: text that exists only as pixels of an image is not text; a string +//! that also appears in content that was *not* redacted is counted, over the whole document, on +//! the page text, the raw file and the decoded streams (those hold the kept copy too) and fails +//! on a surplus; names are swept against the matches the input itself had, since a redacted word +//! that is also a `/Subtype` or a `/BaseFont` is structure, not a leftover copy, and a surplus of +//! matches still fails; every other surface is swept with no allowance. A word of six characters or more +//! that also spells a structural name (`/Filter`, `/MediaBox`) can still be reported by the raw-file and +//! decoded-stream surfaces, which search stored bytes and cannot tell a name from a copy: that is +//! over-reporting, never a pass. The raw file and the decoded +//! streams are searched for strings of at least six characters only (shorter ones match object +//! syntax and numbers by chance), skipping the trailer's file identifier and the +//! pixels of an image codec (DCT, JPX, JBIG2, CCITT), which are searched as stored bytes only; shorter strings are still found on the page text, +//! in strings, names, Info, XMP and attachment text. A region without extractable text is reported as such, and as +//! unverifiable when the page draws text this extractor cannot decode. The geometric check +//! places glyphs with the font's `/Widths` only: for a font without them the widths are a guess, +//! so a neighbouring glyph may be reported when the guess puts its centre inside a region. +//! `runs_removed` and `codes_removed` are exact counts of what was cut: a proof that is shared +//! discloses how much text sat under each region (never what it said). + +use std::fmt::Write as _; + +use pdfcraft_cos::ObjRef; + +mod extract; +mod geometry; +pub(crate) mod limits; +#[cfg(test)] +mod search_tests; +mod snapshot; +mod sweep; +mod util; + +pub use snapshot::Snapshot; +/// What the proof established for one region. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + /// What was removed was swept for on every surface and is gone. + Verified, + /// The region held no extractable text: nothing was removed, and that is recorded as such. + VerifiedNoTextInRegion, + /// Neither "text was removed" nor "there was none" can be checked (the page draws text this + /// extractor can't decode, or an image under the region can't be inspected). The proof can't + /// pass while any region is unverifiable. + Unverifiable, + /// A survivor was found, or the mechanical evidence is missing. + Failed, +} + +/// The places a redacted string is looked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Surface { + RawBytes, + ObjectStrings, + DecodedStreams, + ExtractedText, + Info, + Xmp, + EmbeddedFiles, + Revisions, +} + +const SURFACES: [Surface; 8] = [ + Surface::RawBytes, + Surface::ObjectStrings, + Surface::DecodedStreams, + Surface::ExtractedText, + Surface::Info, + Surface::Xmp, + Surface::EmbeddedFiles, + Surface::Revisions, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// Swept, nothing found. + Clean, + /// Swept, a redacted string (or an earlier revision) is still there. + Survivor, + /// Could not be swept; the proof fails. + Unswept, + /// There is nothing of this kind in the document. + Absent, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SurfaceReport { + pub surface: Surface, + pub verdict: Verdict, + /// Streams, strings, pages or files looked at. + pub scanned: usize, + /// Streams scanned as stored bytes only: image-codec pixels (not text), and font, colour or + /// image streams that could not be decoded in full. + pub raw_only: usize, + pub survivors: usize, + /// Where each survivor sits ("object 12", "page 1", "attachment #1 > object 4"); never the text. + pub locations: Vec, + /// Why the surface could not be swept. + pub problems: Vec, +} + +/// The manifest entry of one redaction region. +#[derive(Clone, Debug, PartialEq)] +pub struct Excision { + /// 0-based page. + pub page: usize, + /// The region in user space. + pub region: [f64; 4], + /// The Redact annotation it came from. + pub mark: ObjRef, + pub status: Status, + pub detail: String, + /// Glyph runs (consecutive glyphs of one string) removed under the region, and their + /// character codes. + pub runs_removed: usize, + pub codes_removed: usize, + /// SHA-256 of the page's decoded content streams, joined, after redaction. There is no digest + /// of the content before: a proof is meant to be shared, and a hash of the original page + /// would let a reader confirm a guess at what it said. + pub page_sha256_after: Option, + /// Glyph-carrying text operators on the page (forms included) before and after. A `TJ` of + /// numbers only (the gap left for a removed string) carries no glyphs and does not count. + pub text_ops_before: Option, + pub text_ops_after: Option, + /// The removed text, only when [`ProofOptions::include_plaintext`] is set. + pub removed_text: Option>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Proof { + pub entries: Vec, + pub surfaces: Vec, + /// Located, plain reasons for every failure (never the redacted text). + pub failures: Vec, +} + +impl Proof { + /// Survivors found across all surfaces. + pub fn survivors(&self) -> usize { + self.surfaces.iter().map(|s| s.survivors).sum() + } + + /// Surfaces that could not be swept. + pub fn unswept(&self) -> usize { + self.surfaces.iter().filter(|s| s.verdict == Verdict::Unswept || !s.problems.is_empty()).count() + } + + pub fn unverifiable(&self) -> usize { + self.entries.iter().filter(|e| e.status == Status::Unverifiable).count() + } + + /// Pass only when nothing failed, nothing is unverifiable and no surface went unswept. + pub fn passed(&self) -> bool { + self.failures.is_empty() + && self.survivors() == 0 + && self.unswept() == 0 + && self.entries.iter().all(|e| matches!(e.status, Status::Verified | Status::VerifiedNoTextInRegion)) + } + + /// A human-readable report: the verdict, the manifest and the surfaces. It names places and + /// counts, and the removed text only if it was asked for. + pub fn to_text(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "Redaction proof: {}", if self.passed() { "PASS" } else { "FAIL" }); + for f in &self.failures { + let _ = writeln!(s, " failure: {f}"); + } + for (i, e) in self.entries.iter().enumerate() { + let _ = writeln!( + s, + " region {} page {} {:?} {:?}: {} run(s), {} code(s) removed; text ops {:?} -> {:?}; page sha256 after {}{}", + i + 1, + e.page + 1, + e.region, + e.status, + e.runs_removed, + e.codes_removed, + e.text_ops_before, + e.text_ops_after, + e.page_sha256_after.as_deref().unwrap_or("-"), + if e.detail.is_empty() { String::new() } else { format!(" ({})", e.detail) }, + ); + } + for r in &self.surfaces { + let _ = writeln!( + s, + " {:?}: {:?} ({} scanned, {} raw only, {} survivor(s)){}", + r.surface, + r.verdict, + r.scanned, + r.raw_only, + r.survivors, + if r.locations.is_empty() { String::new() } else { format!(" at {}", r.locations.join(", ")) }, + ); + for p in &r.problems { + let _ = writeln!(s, " unswept: {p}"); + } + } + let _ = writeln!(s, " Evidence, not a guarantee: text that exists only as pixels of an image is not text, and is not searched."); + s + } +} + +/// What a proof may contain. +#[derive(Clone, Debug, Default)] +pub struct ProofOptions { + /// Include the removed text in the manifest. Off by default: a proof is meant to be shared, + /// and must not carry what it proves was removed. + pub include_plaintext: bool, +} + +/// Handles for the tests to reach the byte sweep without a whole document. +#[cfg(test)] +pub(crate) mod tests_support { + use pdfcraft_content::parse; + use pdfcraft_cos::{Dict, Object, Stream}; + + use super::extract::{carries_glyphs, parse_cmap}; + use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; + use super::sweep::{Needle, add_needle, bytes_counts, bytes_hits, walk_strings}; + use super::util::Ctx; + pub(crate) use super::util::survivor_encodings; + + /// The proof's whole search-work budget (for tests that pre-spend part of it). + pub(crate) fn work_budget() -> u64 { + MAX_WORK + } + + /// The byte sweep of `hay` for a bulk of `count` distinct needles, on a context already + /// `spent` units into its work budget: `Some` (with the needles that occur) while the sweep + /// stays under the budget, `None` once it ran out. + pub(crate) fn bulk_counts(hay: &[u8], text: &str, count: usize, spent: u64) -> Option { + let mut needles: Vec = Vec::new(); + for i in 0..count { + add_needle(&mut needles, false, format!("{text}{i:04}"), Some(i)); + } + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + bytes_counts(&needles, hay, MIN_STRUCTURED_NEEDLE, &ctx).map(|hits| hits.len()) + } + + /// The glyph-carrying text-showing operators of a decoded content stream. + pub fn count_text_operators(decoded: &[u8]) -> usize { + parse(decoded).ops.iter().filter(|op| carries_glyphs(op)).count() + } + + /// Does the byte sweep for `text` find it in `hay`? + pub fn found(text: &str, hay: &[u8]) -> bool { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, text.to_string(), Some(0)); + bytes_hits(&needles, hay, 3, &Ctx::new(Default::default())).is_some_and(|h| !h.is_empty()) + } + + /// Does this CMap parse completely (not cut short by a bound)? + pub fn cmap_is_complete(data: &[u8]) -> bool { + parse_cmap(data).is_some() + } + + /// Is an object nested `depth` levels deep walked in full? + pub fn walks_in_full(depth: usize) -> bool { + let mut o = Object::Null; + for _ in 0..depth { + o = Object::Array(vec![o]); + } + walk_strings(&o, 0, &mut |_| {}) + } + + /// Does decoding fail once the cumulative decode budget is used up (and not before)? + pub fn decode_budget_runs_out() -> (bool, bool) { + let ctx = Ctx::new(Default::default()); + let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let before = ctx.decode(&s).is_ok(); + let spent = Ctx::with_decode_left(50); + (before, spent.decode(&s).is_err()) + } + + /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb + /// cannot be decoded (and fail) again and again for free? + /// `(refused, charged, then refused for want of allowance)`. + pub fn failed_decode_charges_the_budget() -> (bool, bool, bool) { + let big = vec![0u8; MAX_STREAM + (1 << 20)]; + let bomb = Stream::flate(Dict::new(), &big); + drop(big); + let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); + let refused = ctx.decode(&bomb).is_err(); + let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; + let gone = ctx.decode(&small).is_err(); + (refused, charged, gone) + } + + /// Does the byte search report `None` once the work budget is used up (and not before)? + pub fn search_budget_runs_out() -> (bool, bool) { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, "secret".to_string(), Some(0)); + let ctx = Ctx::new(Default::default()); + let before = bytes_hits(&needles, b"some bytes", 3, &ctx).is_some(); + ctx.work.set(MAX_WORK); + (before, bytes_hits(&needles, b"some bytes", 3, &ctx).is_none()) + } + + /// Is a survivor counted on an earlier page still reported when the work budget runs out on + /// a later one? `(survivor reported, page left unswept)`. + pub fn page_sweep_survivors_survive_the_limit() -> (bool, bool) { + use super::Surface; + use super::sweep::{Acc, sweep_pages}; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"), + b"<< /Type /Font /Subtype /TrueType /BaseFont /Arial >>".to_vec(), + crate::tests_verify::stream("", &format!("BT /F1 10 Tf 10 100 Td ({}) Tj ET", "x".repeat(700_000)).into_bytes()), + ]; + let doc = pdfcraft_cos::Document::open(std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, ""))).unwrap(); + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRETVALUE".to_string(), Some(0)); + let mut acc = Acc::new(Ctx::new(Default::default())); + // Page 1 (the survivor's) fits in what is left; page 2 does not. + acc.ctx.work.set(MAX_WORK - 100_000); + sweep_pages(&needles, &doc, 0, "", &mut acc); + let text = acc.reps.get(&Surface::ExtractedText).unwrap(); + (text.survivors > 0, !text.problems.is_empty()) + } + + /// Does a font's `/W` parsing answer to the shared work budget, and not only to its own + /// counter (a form invoked again rebuilds its fonts, and would pay for the same array + /// again)? `(read in full, refused on a spent budget)`. + pub fn font_widths_charge_shares_the_budget() -> (bool, bool) { + use super::extract::extract_page; + let font = b"<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 1000 /W [1 65535 250] >>] >>".to_vec(); + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F2 5 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F2 10 Tf 10 100 Td <00410042> Tj ET"), + font, + ]; + let bytes = std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, "")); + let probe = |spent: u64| -> bool { + let doc = pdfcraft_cos::Document::open(bytes.clone()).unwrap(); + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + let page = &pdfcraft_model::pages(&doc)[0]; + extract_page(&doc, &page.dict, &ctx, &[]).is_ok() + }; + (probe(0), !probe(MAX_WORK - 10_000)) + } +} diff --git a/crates/redact/src/verify/search_tests.rs b/crates/redact/src/verify/search_tests.rs new file mode 100644 index 0000000000..4b8d84c243 --- /dev/null +++ b/crates/redact/src/verify/search_tests.rs @@ -0,0 +1,34 @@ +use super::{ + sweep::{add_needle, bytes_counts, bytes_hits}, + util::Ctx, +}; +use std::collections::HashSet; + +#[test] +fn a_thousand_needles_share_one_multimegabyte_scan() { + let mut needles = Vec::new(); + for i in 0..1000 { + add_needle(&mut needles, false, format!("SECRET_{i:04}"), None); + } + let hay = vec![b'z'; 8 << 20]; + assert_eq!(bytes_hits(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); +} + +#[test] +fn overlapping_encodings_count_one_occurrence() { + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRET42".into(), None); + let text: Vec = "SECRET42".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let mut hay = vec![0xfe, 0xff]; + hay.extend(text); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(vec![(0, 1)])); +} + +#[test] +fn match_count_exhaustion_is_incomplete_not_a_truncated_pass() { + let mut needles = Vec::new(); + add_needle(&mut needles, true, "SECRET42".into(), None); + let hay = b"SECRET42 ".repeat((1 << 16) + 1); + assert!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())).is_none()); +} diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs new file mode 100644 index 0000000000..fe74571bb4 --- /dev/null +++ b/crates/redact/src/verify/snapshot.rs @@ -0,0 +1,532 @@ +//! The snapshot taken before applying (what is about to go) and the proof run on the output. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full}; + +use super::extract::extract_page; +use super::geometry::{Finding, check_geometry}; +use super::sweep::{Acc, Needle, add_needle, bytes_hits, sweep_doc, walk_strings}; +use super::util::{Ctx, count, latin1, sha256, squash, words}; +use super::{Excision, Proof, ProofOptions, SURFACES, Status, Surface}; +use crate::interp::{Mode, Removed, Scope, process}; +use crate::verify::limits::MIN_NEEDLE; +use crate::{Mark, RedactError, Report, page_streams}; + +pub(super) struct RegionSnap { + pub(super) page: usize, + pub(super) region: [f64; 4], + pub(super) mark: ObjRef, + pub(super) runs: Vec<(Vec, String)>, + /// The runs of one text operator joined, where it split them: `[(sec) (ret)] TJ` reads as + /// `secret`, which a copy elsewhere in the file holds whole. + pub(super) joined: Vec<(Vec, String)>, + /// The character codes the runs hold (a composite font's codes are several bytes). + pub(super) codes: usize, +} + +pub(super) struct PageSnap { + pub(super) sha: Option, + pub(super) text_ops: Option, + pub(super) undecodable_ops: usize, +} + +/// What redaction is about to remove, recorded before it does. +pub struct Snapshot { + regions: Vec, + pages: BTreeMap, + needles: Vec, + /// Things that made the "before" side unreadable (reported on the text surface). + problems: Vec, + /// The document's page text before redaction (decoded view, code view), for the allowance + /// of needles added later. + src: (String, String), + /// The names of the document (structure redaction never edits), for the name allowance of + /// needles added later. + names: BTreeMap, usize>, + /// Digests of the decoded content of the streams this run created to draw the redaction + /// boxes and labels (see [`Snapshot::set_overlay_streams`]). + overlay: HashSet, +} + +impl std::fmt::Debug for Snapshot { + // The snapshot holds what is about to be redacted: never print it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Snapshot").field("regions", &self.regions.len()).field("needles", &self.needles.len()).finish() + } +} + +impl Snapshot { + /// Record what redacting the marks on `pages` (0-based; `None` = all) will remove. Call it + /// before [`apply`](crate::apply); it changes nothing. + pub fn capture(doc: &Document, pages: Option<&[usize]>) -> Result { + let all = crate::marks(doc); + let chosen: Vec<&Mark> = all.iter().filter(|m| pages.is_none_or(|p| p.contains(&m.page))).collect(); + if chosen.is_empty() { + return Err(RedactError::NothingToApply); + } + Snapshot::capture_marks(doc, &chosen) + } + + pub(crate) fn capture_marks(doc: &Document, chosen: &[&Mark]) -> Result { + let mut by_page: Vec = chosen.iter().map(|m| m.page).collect(); + by_page.sort_unstable(); + by_page.dedup(); + let page_list = pdfcraft_model::pages(doc); + let ctx = Ctx::new(HashSet::new()); + let mut regions: Vec = Vec::new(); + let mut pages = BTreeMap::new(); + let mut images = pdfcraft_cos::ImageBudget::new(doc); + // What was cut, each copy once, however many overlapping marks it lay under: the allowance + // for what stays is what the source holds less this. + let (mut cut_text, mut cut_raw) = (String::new(), String::new()); + for &pi in &by_page { + let page = page_list.get(pi).ok_or(RedactError::Unreadable(pi + 1))?; + let page_marks: Vec<&&Mark> = chosen.iter().filter(|m| m.page == pi).collect(); + let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); + let (_, data) = page_streams(doc, &page.dict, pi)?; + let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + // The interpreter's own removal path, on a scratch copy: what it takes out is what + // has to be gone. + let mut scratch = doc.clone(); + let mut sink = Report::default(); + let removed: Vec = { + let mut scope = Scope::new(&rects, Mode::Apply, &mut sink, pdfcraft_cos::DecodeBudget::page(), &mut images); + scope.removed = Some(Vec::new()); + process(&mut scratch, &mut scope, &data, &resources, Matrix::IDENTITY); + scope.removed.take().unwrap_or_default() + }; + let lines = joined(&removed); + let mut seen = HashSet::new(); + for line in lines.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))) { + cut_text.push('\u{1}'); + cut_text.push_str(&squash(&line.text)); + cut_raw.push('\u{1}'); + cut_raw.push_str(&squash(&latin1(&line.raw))); + } + let before = extract_page(doc, &page.dict, &ctx, &[]); + pages.insert( + pi, + PageSnap { + sha: Some(sha256(&data)), + text_ops: before.as_ref().ok().map(|e| e.glyph_ops), + undecodable_ops: before.as_ref().map_or(0, |e| e.undecodable_ops), + }, + ); + let mut k = 0; + for m in &page_marks { + for r in &m.rects { + let hits: Vec<&Removed> = removed.iter().filter(|x| x.rect == k).collect(); + let runs = hits.iter().map(|x| (x.raw.clone(), x.text.clone())).collect(); + let codes = hits.iter().map(|x| x.codes).sum(); + let joined = lines.iter().filter(|l| l.rect == k && l.pieces > 1).map(|l| (l.raw.clone(), l.text.clone())).collect(); + regions.push(RegionSnap { page: pi, region: *r, mark: m.obj, runs, joined, codes }); + k += 1; + } + } + } + // The document's text as the independent extractor reads it, for the allowance of + // strings that legitimately remain elsewhere. + let (mut src_text, mut src_raw) = (String::new(), String::new()); + let mut problems = Vec::new(); + for (i, p) in page_list.iter().enumerate() { + match extract_page(doc, &p.dict, &ctx, &[]) { + Ok(e) => { + src_text.push_str(&e.text); + src_raw.push_str(&e.raw); + } + Err(()) => { + problems.push(format!("page {} of the source can't be read, so strings left on it can't be told from removed ones", i + 1)) + } + } + } + let mut needles: Vec = Vec::new(); + for (ri, r) in regions.iter().enumerate() { + for (raw, text) in r.runs.iter().chain(&r.joined) { + let raw_s = latin1(raw); + for (raw_view, whole) in [(false, text.trim().to_string()), (true, raw_s)] { + let mut shown: Vec = Vec::new(); + if !squash(&whole).is_empty() { + shown.push(whole.clone()); + shown.extend(words(&whole).into_iter().filter(|w| *w != whole).map(str::to_string)); + } + for s in shown { + add_needle(&mut needles, raw_view, s, Some(ri)); + } + } + } + } + for (src, cut, raw) in [(&src_text, &cut_text, false), (&src_raw, &cut_raw, true)] { + match (super::sweep::key_counts(&needles, src, raw, &ctx), super::sweep::key_counts(&needles, cut, raw, &ctx)) { + (Some(source), Some(removed)) => { + let removed: BTreeMap = removed.into_iter().collect(); + for (i, total) in source { + if let Some(n) = needles.get_mut(i) { + n.allowance = total.saturating_sub(removed.get(&i).copied().unwrap_or(0)); + } + } + } + _ => problems.push("the source text allowances could not be counted in full".into()), + } + } + // The names the removed text legitimately sits in (a redacted word that is also a + // `/Subtype`, a `/Filter`, a `/BaseFont`): names are structure redaction never edits, + // so the output may hold the input's own matches, and only a surplus is a survivor. + let names = all_names(doc); + let name_allow: Vec> = needles.iter().map(|n| count_name_matches(&names, n, &ctx)).collect(); + if name_allow.iter().any(Option::is_none) { + problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + for (n, allow) in needles.iter_mut().zip(name_allow) { + n.name_allowance = allow.unwrap_or(0); + } + Ok(Snapshot { regions, pages, needles, problems, src: (src_text, src_raw), names, overlay: HashSet::new() }) + } + + /// Tell the proof which streams this run created to draw the redaction boxes and labels + /// (`created`: references into `doc`, the document after `apply`). Call it before + /// [`prove`](Snapshot::prove) / [`prove_saved`](Snapshot::prove_saved). + /// + /// Those streams paint over the regions on purpose and may show label text, so the proof + /// leaves them out of the text sweep and the paint check. They are recognised by the digest + /// of their decoded content (a save renumbers objects, so references would not survive it), + /// and *only* those: a stream that merely carries a `/PCMark /Redaction` key in the input is + /// not exempt. Streams that can't be read here are an error; nothing is then exempted. + pub fn set_overlay_streams(&mut self, doc: &Document, created: &[ObjRef]) -> Result<(), RedactError> { + let ctx = Ctx::new(HashSet::new()); + let mut digests = HashSet::new(); + for r in created { + let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; + let Object::Stream(s) = &*obj else { continue }; + let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + digests.insert(sha256(&[data])); + } + self.overlay = digests; + Ok(()) + } + + /// Also look for these strings in the output: text that left the document by another route + /// than a page's glyphs (a removed annotation's `/Contents` or `/RC`, a form field's value). + /// Each is searched whole, on every surface. A string that also legitimately remains as page + /// text is counted on the page-content surfaces only (as for the removed glyph runs). + /// Strings that are blank or shorter than the minimum needle are ignored. + pub fn add_removed_strings(&mut self, strings: &[String]) { + for s in strings { + let text = s.trim(); + let key = squash(text); + if key.chars().count() < MIN_NEEDLE { + continue; + } + let known = self.needles.iter().any(|n| !n.raw_view && n.key == key); + // No owner: a survivor fails the proof through its surface; no region carries it. + add_needle(&mut self.needles, false, text.to_string(), None); + if !known { + let ctx = Ctx::new(HashSet::new()); + let matches = self.needles.last().map(|n| count_name_matches(&self.names, n, &ctx)); + if let Some(n) = self.needles.last_mut() { + n.allowance = count(&self.src.0, &n.key); + n.name_allowance = matches.flatten().unwrap_or(0); + } + if matches == Some(None) { + self.problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + } + } + } + + /// Serialize `doc` the way a save would and sweep the result. The output is re-opened with + /// `password`, the credentials the caller opened the document with (`None` tries the empty + /// password): a save of an encrypted document is itself encrypted, and a proof that could + /// not read its own subject would prove nothing. + pub fn prove(&self, doc: &Document, password: Option<&str>, opts: &ProofOptions) -> Proof { + match write_full(doc, &SaveOptions::default()) { + Ok(bytes) => self.prove_inner(&bytes, Some(doc), password, opts), + Err(e) => { + let mut p = self.prove_inner(&[], None, password, opts); + p.failures.push(format!("the document could not be serialized for the sweep: {e}")); + p + } + } + } + + /// Sweep an already saved file (for example an incrementally saved one, whose earlier + /// revision still holds the original content). The file is opened with `password`, the + /// credentials it is read with. + pub fn prove_saved(&self, bytes: &[u8], password: Option<&str>, opts: &ProofOptions) -> Proof { + self.prove_inner(bytes, None, password, opts) + } + + fn prove_inner(&self, bytes: &[u8], live: Option<&Document>, password: Option<&str>, opts: &ProofOptions) -> Proof { + let mut acc = Acc::new(Ctx::new(self.overlay.clone())); + let mut failures: Vec = Vec::new(); + for p in &self.problems { + acc.problem(Surface::ExtractedText, p.clone()); + } + // Always the caller's credentials, never unconditionally the empty password: a + // document whose user password is not empty must be provable with that password, and + // must fail closed (an unswept output, never a false pass) without it. + let parsed = if bytes.is_empty() { None } else { Document::open_with_password(Arc::new(bytes.to_vec()), password).ok() }; + match &parsed { + Some(out) => sweep_doc(&self.needles, out, bytes, 0, "", &mut acc), + None => { + let why = "the output could not be read back".to_string(); + for s in SURFACES { + acc.problem(s, why.clone()); + } + failures.push(why); + } + } + if live.is_some_and(|d| !d.full_save_required()) { + acc.survivor(Surface::Revisions, "the document would be saved as an incremental update, keeping the original revision".into(), None); + } + + // The non-text content under each region, on the output. + let (findings, after) = match &parsed { + Some(out) => self.check_output(out, &acc.ctx), + None => Default::default(), + }; + let mut entries = Vec::with_capacity(self.regions.len()); + for (i, r) in self.regions.iter().enumerate() { + let (entry, failure) = self.entry(i, r, &findings, &after, &acc.failed_needles, opts); + failures.extend(failure); + entries.push(entry); + } + let surfaces = acc.finish(); + for s in &surfaces { + if s.survivors > 0 { + failures.push(format!("{:?}: {} survivor(s) at {}", s.surface, s.survivors, s.locations.join(", "))); + } + for p in &s.problems { + failures.push(format!("{:?} could not be swept: {p}", s.surface)); + } + } + Proof { entries, surfaces, failures } + } + + /// The non-text content under each region of the output (what still paints or shows there), + /// and per page the digest and glyph-operator count of what `apply` left. + fn check_output(&self, out: &Document, ctx: &Ctx) -> (HashMap>, PagesAfter) { + let mut findings: HashMap> = HashMap::new(); + let mut after: PagesAfter = BTreeMap::new(); + let out_pages = pdfcraft_model::pages(out); + for &pi in self.pages.keys() { + let idx: Vec = (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| r.page == pi)).collect(); + let Some(page) = out_pages.get(pi) else { + for &i in &idx { + findings.entry(i).or_default().push(Finding::Missing); + } + continue; + }; + let rects: Vec<[f64; 4]> = idx.iter().filter_map(|&i| self.regions.get(i).map(|r| r.region)).collect(); + let (sha, ops, shown) = page_after(out, &page.dict, ctx, &rects); + after.insert(pi, (sha, ops)); + match shown { + Some(shown) => { + for (k, n) in shown.into_iter().enumerate() { + if let (Some(&i), true) = (idx.get(k), n > 0) { + findings.entry(i).or_default().push(Finding::Text(n)); + } + } + } + None => { + for &i in &idx { + findings + .entry(i) + .or_default() + .push(Finding::Unverifiable("the page text can't be placed, so what still shows under the region can't be checked")); + } + } + } + for (k, f) in check_geometry(out, &page.dict, &rects, ctx) { + if let Some(&i) = idx.get(k) { + findings.entry(i).or_default().push(f); + } + } + } + (findings, after) + } + + /// The manifest entry of region `i`, and why it failed when it did. + fn entry( + &self, + i: usize, + r: &RegionSnap, + findings: &HashMap>, + after: &PagesAfter, + failed_needles: &HashSet, + opts: &ProofOptions, + ) -> (Excision, Option) { + let mut failure = None; + let snap = self.pages.get(&r.page); + let (sha_after, ops_after) = after.get(&r.page).cloned().unwrap_or((None, None)); + let sha_before = snap.and_then(|s| s.sha.clone()); + let owned: Vec<&Needle> = self.needles.iter().filter(|n| n.owners.contains(&i)).collect(); + let survived = self.needles.iter().enumerate().filter(|(ni, n)| n.owners.contains(&i) && failed_needles.contains(ni)).count(); + let mut detail: Vec = Vec::new(); + let mut status = if r.runs.is_empty() { Status::VerifiedNoTextInRegion } else { Status::Verified }; + let fs = findings.get(&i).map(Vec::as_slice).unwrap_or_default(); + if survived > 0 { + status = Status::Failed; + detail.push(format!("{survived} removed string(s) still found")); + } + for f in fs { + match f { + Finding::Text(n) => detail.push(format!("{n} glyph(s) still show inside the region (placed independently of the redaction)")), + Finding::Image => detail.push("an image still carries pixels under the region".into()), + Finding::Path => detail.push("a path still paints inside the region".into()), + Finding::Shading => detail.push("a shading still paints inside the region".into()), + Finding::Missing => detail.push("the page is missing from the output".into()), + Finding::Unverifiable(why) => detail.push((*why).to_string()), + } + if !matches!(f, Finding::Unverifiable(_)) { + status = Status::Failed; + } else if status != Status::Failed { + status = Status::Unverifiable; + } + } + if sha_before.is_none() || sha_after.is_none() { + status = Status::Failed; + detail.push("the page content can't be read on one side, so it can't be compared".into()); + } else if !r.runs.is_empty() && sha_before == sha_after { + status = Status::Failed; + detail.push("glyphs were removed but the page content is unchanged".into()); + } + if r.runs.is_empty() && snap.is_some_and(|s| s.undecodable_ops > 0) && status != Status::Failed { + status = Status::Unverifiable; + detail.push("the page draws text that can't be decoded, so \"no text in the region\" can't be checked".into()); + } + let ambiguous = owned.iter().filter(|n| n.allowance > 0).count(); + if ambiguous > 0 { + detail.push(format!("{ambiguous} string(s) also remain in retained content; only a surplus over those copies counts")); + } + if matches!(status, Status::Failed | Status::Unverifiable) { + failure = Some(format!("page {} region {:?}: {}", r.page + 1, r.region, detail.join("; "))); + } + let entry = Excision { + page: r.page, + region: r.region, + mark: r.mark, + status, + detail: detail.join("; "), + runs_removed: r.runs.len(), + codes_removed: r.codes, + page_sha256_after: sha_after, + text_ops_before: snap.and_then(|s| s.text_ops), + text_ops_after: ops_after, + removed_text: opts.include_plaintext.then(|| r.runs.iter().map(|(_, t)| t.clone()).collect()), + }; + (entry, failure) + } +} + +/// Per page: the digest of its content in the output and its glyph-operator count. +type PagesAfter = BTreeMap, Option)>; + +/// Every name of the document (the values, wherever an object holds them), with how often +/// each occurs. +fn all_names(doc: &Document) -> BTreeMap, usize> { + let mut names: BTreeMap, usize> = BTreeMap::new(); + for num in doc.object_numbers() { + if let Ok(obj) = doc.try_get(num) { + walk_strings(&obj, 0, &mut |o| { + if let Object::Name(n) = o { + let seen = names.entry(n.clone()).or_default(); + *seen = seen.saturating_add(1); + } + }); + } + } + names +} + +/// How many of `names` carry `needle`, by the same search the sweep runs on a name. `None` +/// when the search work ran out: an unknown allowance must not read as "none allowed", so the +/// caller reports that (and sweeps names with no allowance). +fn count_name_matches(names: &BTreeMap, usize>, needle: &Needle, ctx: &Ctx) -> Option { + if needle.short() { + return Some(0); + } + let mut total = 0usize; + for (name, seen) in names { + if !bytes_hits(std::slice::from_ref(needle), name, MIN_NEEDLE, ctx)?.is_empty() { + total = total.saturating_add(*seen); + } + } + Some(total) +} + +/// The page's content digest and glyph-operator count in the output, and per region the glyphs +/// still placed inside it (`None` when the page's text can't be placed). The digest covers the +/// page's content as `apply` left it, the overlay streams included. +pub(super) fn page_after(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> (Option, Option, Option>) { + let sha = page_streams(doc, page, 0).ok().map(|(_, d)| sha256(&d)); + let e = extract_page(doc, page, ctx, regions).ok(); + (sha, e.as_ref().map(|e| e.glyph_ops), e.map(|e| e.shown_in)) +} + +/// One text operator's removed runs under one region, as a reader takes them: joined. +struct Line { + rect: usize, + op: usize, + at: (usize, usize), + raw: Vec, + text: String, + pieces: usize, +} + +/// The runs of `removed` joined by text operator and region, in the order they were shown. +fn joined(removed: &[Removed]) -> Vec { + let mut lines: Vec = Vec::new(); + let mut found: HashMap<(usize, usize), usize> = HashMap::new(); + for r in removed { + let line = found.get(&(r.op, r.rect)).copied().and_then(|i| lines.get_mut(i)); + match line { + Some(l) => { + l.raw.extend_from_slice(&r.raw); + l.text.push_str(&r.text); + l.pieces += 1; + } + None => { + found.insert((r.op, r.rect), lines.len()); + lines.push(Line { rect: r.rect, op: r.op, at: r.at, raw: r.raw.clone(), text: r.text.clone(), pieces: 1 }); + } + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(rect: usize, op: usize, place: usize, text: &str) -> Removed { + Removed { rect, raw: text.as_bytes().to_vec(), text: text.to_string(), codes: text.len(), op, at: (place, 0) } + } + + #[test] + fn a_word_split_across_a_tj_array_is_one_line_for_each_region() { + let lines = joined(&[run(0, 1, 0, "sec"), run(1, 1, 0, "sec"), run(0, 1, 1, "ret"), run(1, 1, 1, "ret"), run(0, 2, 0, "other")]); + let shown: Vec<(usize, usize, &str, usize)> = lines.iter().map(|l| (l.rect, l.op, l.text.as_str(), l.pieces)).collect(); + assert_eq!(shown, [(0, 1, "secret", 2), (1, 1, "secret", 2), (0, 2, "other", 1)]); + } + + #[test] + fn overlapping_marks_are_one_copy_and_two_copies_stay_two() { + // Under two marks at once: the same operator, place and bytes, so one copy was cut. + let both = joined(&[run(0, 1, 0, "word"), run(1, 1, 0, "word")]); + let mut seen = HashSet::new(); + assert_eq!(both.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 1); + // The same word twice in one array, one under each mark: two copies were cut. + let apart = joined(&[run(0, 1, 0, "word"), run(1, 1, 2, "word")]); + let mut seen = HashSet::new(); + assert_eq!(apart.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 2); + } + + #[test] + fn a_spent_decode_budget_is_told_apart() { + assert!(!Ctx::new(HashSet::new()).decode_spent()); + assert!(Ctx::with_decode_left(0).decode_spent()); + } +} diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs new file mode 100644 index 0000000000..73b44c005d --- /dev/null +++ b/crates/redact/src/verify/sweep.rs @@ -0,0 +1,759 @@ +//! Sweeping the serialized output for the removed strings, on every surface a reader could +//! recover them from. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; + +use super::extract::extract_page; +use super::util::{Ctx, decode_text, find, image_codec, spellings, squash, survivor_encodings, xml_unescape}; +use super::{SURFACES, Surface, SurfaceReport, Verdict}; +use crate::verify::limits::{MAX_NESTING, MAX_WALK, MIN_NEEDLE, MIN_STRUCTURED_NEEDLE}; + +/// A string the sweep looks for. +pub(super) struct Needle { + /// The code bytes as Latin-1 text (the "raw view" of a page) rather than decoded text. + pub(super) raw_view: bool, + /// As removed, whitespace included (for the byte encodings). + pub(super) shown: String, + /// `shown` without whitespace (for page text, which is compared whitespace-blind). + pub(super) key: String, + /// What the raw file may spell it as. + pub(super) bytes: Vec>, + pub(super) owners: BTreeSet, + /// Occurrences that legitimately remain in content that was not redacted. The surfaces that + /// hold page content (the page text, the raw file, decoded streams) count occurrences over + /// the whole document and fail on a surplus; every other surface (strings, Info, XMP, + /// attachments) is swept with no allowance. + pub(super) allowance: usize, + /// Names of the input that carry this needle (a redacted word that is also a `/Subtype`, a + /// `/Filter`, a `/BaseFont`). Names are structure redaction never edits, so the name surface + /// is swept against the input's own matches: the output may hold as many as the input did, + /// and only a surplus is a survivor. + pub(super) name_allowance: usize, +} + +impl Needle { + pub(super) fn short(&self) -> bool { + self.key.chars().count() < MIN_NEEDLE + } + + /// Long enough to be looked for in bytes (`min`: the shortest the haystack allows). + pub(super) fn at_least(&self, min: usize) -> bool { + self.key.chars().count() >= min + } +} + +pub(super) fn add_needle(set: &mut Vec, raw_view: bool, shown: String, owner: Option) { + let key = squash(&shown); + if key.is_empty() { + return; + } + if let Some(n) = set.iter_mut().find(|n| n.raw_view == raw_view && n.key == key) { + n.owners.extend(owner); + return; + } + let bytes = if raw_view { + // The code bytes themselves (Latin-1 round-trips them). + let raw: Vec = shown.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).collect(); + spellings([raw]) + } else { + survivor_encodings(&shown) + }; + set.push(Needle { raw_view, shown, key, bytes, owners: owner.into_iter().collect(), allowance: 0, name_allowance: 0 }); +} + +pub(super) struct Acc { + pub(super) reps: BTreeMap, + pub(super) failed_needles: HashSet, + pub(super) ctx: Ctx, +} + +impl Acc { + pub(super) fn new(ctx: Ctx) -> Self { + let mut a = Acc { reps: BTreeMap::new(), failed_needles: HashSet::new(), ctx }; + for s in SURFACES { + a.rep(s); + } + a + } + + pub(super) fn rep(&mut self, surface: Surface) -> &mut SurfaceReport { + self.reps.entry(surface).or_insert_with(|| SurfaceReport { + surface, + verdict: Verdict::Absent, + scanned: 0, + raw_only: 0, + survivors: 0, + locations: Vec::new(), + problems: Vec::new(), + }) + } + + pub(super) fn scanned(&mut self, s: Surface) { + self.rep(s).scanned += 1; + } + + pub(super) fn raw_only(&mut self, s: Surface) { + self.rep(s).raw_only += 1; + } + + pub(super) fn survivor(&mut self, s: Surface, loc: String, needle: Option) { + self.survivors(s, loc, needle, 1); + } + + /// `count` survivors at one place. + pub(super) fn survivors(&mut self, s: Surface, loc: String, needle: Option, count: usize) { + if let Some(n) = needle { + self.failed_needles.insert(n); + } + let r = self.rep(s); + r.survivors = r.survivors.saturating_add(count); + if !r.locations.contains(&loc) && r.locations.len() < 64 { + r.locations.push(loc); + } + } + + pub(super) fn problem(&mut self, s: Surface, msg: String) { + let r = self.rep(s); + if r.problems.len() < 64 { + r.problems.push(msg); + } + } + + /// The needles (at least `min` long) found in `hay`; running out of search work is a + /// problem of `surface`. + pub(super) fn bytes(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec { + bytes_hits(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// How often each needle (at least `min` long) occurs in `hay`, as `(needle, count)`. + pub(super) fn counts(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec<(usize, usize)> { + bytes_counts(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// Survivors and problems so far, over all surfaces. + pub(super) fn totals(&self) -> (usize, usize) { + (self.reps.values().map(|r| r.survivors).sum(), self.reps.values().map(|r| r.problems.len()).sum()) + } + + pub(super) fn finish(mut self) -> Vec { + for r in self.reps.values_mut() { + r.verdict = if r.survivors > 0 { + Verdict::Survivor + } else if !r.problems.is_empty() { + Verdict::Unswept + } else if r.scanned + r.raw_only == 0 { + Verdict::Absent + } else { + Verdict::Clean + }; + } + self.reps.into_values().collect() + } +} + +/// Occurrences of needles on one surface, per place, so that what legitimately remains +/// (a needle's allowance) is taken off the document-wide total once, not once per place. +#[derive(Default)] +pub(super) struct Tally(BTreeMap>); + +impl Tally { + pub(super) fn add(&mut self, needle: usize, loc: &str, count: usize) { + if count > 0 { + self.0.entry(needle).or_default().push((loc.to_string(), count)); + } + } + + /// Report what is left of each needle's occurrences after its allowance (none inside an + /// attachment, where nothing legitimately remains). + pub(super) fn settle(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.allowance) + } + + /// The name surface: the input's own name matches are the allowance (redaction never edits + /// names), so what is left is a surplus of matches over the input. + pub(super) fn settle_names(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.name_allowance) + } + + fn settle_with(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc, allow_of: impl Fn(&Needle) -> usize) { + for (ni, places) in self.0 { + let mut allow = if depth == 0 { needles.get(ni).map_or(0, &allow_of) } else { 0 }; + for (loc, count) in places { + let used = count.min(allow); + allow -= used; + if count > used { + acc.survivors(surface, loc, Some(ni), count - used); + } + } + } + } +} + +/// Search all encodings in one bounded pass, including overlapping matches. +fn pattern_matches<'a>( + patterns: impl IntoIterator, + hay: &[u8], + ctx: &Ctx, + mut found: impl FnMut(usize, usize, usize) -> bool, +) -> Option<()> { + const MAX_PATTERNS: usize = 100_000; + const MAX_PATTERN_BYTES: usize = 16 << 20; + const MAX_MATCHES: usize = 1 << 20; + if !ctx.charge(hay.len()) { + return None; + } + let mut map: BTreeMap, Vec> = BTreeMap::new(); + let (mut bytes, mut links) = (0usize, 0usize); + for (owner, pattern) in patterns { + if pattern.is_empty() { + continue; + } + if !ctx.charge(pattern.len().saturating_add(1)) { + return None; + } + if !map.contains_key(pattern) { + bytes = bytes.checked_add(pattern.len())?; + if bytes > MAX_PATTERN_BYTES || map.len() >= MAX_PATTERNS { + return None; + } + } + let owners = map.entry(pattern.to_vec()).or_default(); + if owners.last() != Some(&owner) { + links = links.checked_add(1)?; + if links > MAX_PATTERNS { + return None; + } + owners.push(owner); + } + } + if map.is_empty() { + return Some(()); + } + let matcher = aho_corasick::AhoCorasick::new(map.keys()).ok()?; + let owners: Vec> = map.into_values().collect(); + let mut matches = 0usize; + for hit in matcher.find_overlapping_iter(hay) { + for &owner in owners.get(hit.pattern().as_usize())? { + matches = matches.checked_add(1)?; + if matches > MAX_MATCHES || !ctx.charge(1) || !found(owner, hit.start(), hit.end()) { + return None; + } + } + } + Some(()) +} + +/// Count normalized text keys in one pass, with the same non-overlap rule as `str::matches`. +pub(super) fn key_counts(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx) -> Option> { + key_counts_from(needles, hay, raw, ctx, 0) +} + +/// [`key_counts`] of the needles at least `min` long, so a one-letter needle can't use up the match +/// limits of a search it takes no part in. +fn key_counts_from(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx, min: usize) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.raw_view == raw && n.at_least(min)).map(|(i, n)| (i, n.key.as_bytes())); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay.as_bytes(), ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut hits)| { + hits.sort_unstable(); + let (mut count, mut end) = (0, 0); + for (a, b) in hits { + if a >= end { + count += 1; + end = b; + } + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn bytes_hits(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut found = BTreeSet::new(); + pattern_matches(patterns, hay, ctx, |i, _, _| { + found.insert(i); + true + })?; + Some(found.into_iter().collect()) +} + +/// A larger count is incomplete evidence, never a silently truncated count. +pub(super) const MAX_COUNTED: usize = 1 << 16; + +pub(super) fn bytes_counts(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay, ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut spans)| { + spans.sort_unstable(); + spans.dedup(); + let (mut count, mut end) = (0usize, 0usize); + for (a, b) in spans { + if count == 0 || a >= end { + count += 1; + } + end = end.max(b); + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn string_hits(needles: &[Needle], value: &PdfString, ctx: &Ctx) -> Option> { + let text = value.to_text(); + let key = squash(&text); + if !ctx.charge(value.bytes.len().saturating_add(key.len())) { + return None; + } + let mut found: BTreeSet = bytes_hits(needles, &value.bytes, MIN_NEEDLE, ctx)?.into_iter().collect(); + let patterns = needles.iter().enumerate().filter(|(_, n)| !n.raw_view && !n.short()).map(|(i, n)| (i, n.key.as_bytes())); + pattern_matches(patterns, key.as_bytes(), ctx, |i, _, _| { + found.insert(i); + true + })?; + let tokens: BTreeSet<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|s| !s.is_empty()).collect(); + for (i, needle) in needles.iter().enumerate().filter(|(_, n)| n.short() && n.allowance == 0 && !n.raw_view) { + if !ctx.charge(needle.shown.len().saturating_add(1)) { + return None; + } + if tokens.contains(needle.shown.as_str()) { + found.insert(i); + } + } + Some(found.into_iter().collect()) +} + +/// Streams whose data is binary rather than text: images, embedded font programs, colour +/// profiles, cross-reference and object streams (the objects in the latter are swept as +/// objects). Their decoded data is swept like any other stream's; the pixels of an image codec +/// stay undecoded (text that exists only as pixels is not text), and one that won't decode (or is +/// past the cap) is searched as stored bytes only. A short needle "found" in stored bytes is +/// chance. +pub(super) fn binary_stream(d: &Dict) -> bool { + let subtype = d.name(b"Subtype"); + subtype == Some(b"Image".as_slice()) + || matches!(d.name(b"Type"), Some(b"XRef" | b"ObjStm")) + || matches!(subtype, Some(b"Type1C" | b"CIDFontType0C" | b"OpenType")) + || [b"Length1".as_slice(), b"Length2", b"Length3"].iter().any(|k| d.contains(k)) + || (d.contains(b"N") && !d.contains(b"Type") && subtype.is_none()) +} + +/// What a file identifier (`/ID` of the trailer) is spelled as in the file. +pub(super) fn id_spellings(doc: &Document) -> Vec> { + let Some(Object::Array(ids)) = doc.trailer().get(b"ID").map(|o| doc.resolve(o)).as_deref().cloned() else { return Vec::new() }; + ids.iter().filter_map(Object::as_string).flat_map(|s| spellings([s.bytes.to_vec()])).collect() +} + +/// Visit `o` and everything nested in it. False when something lay deeper than [`MAX_WALK`] +/// and was not visited: the caller reports that as unswept. +pub(super) fn walk_strings(o: &Object, depth: usize, f: &mut impl FnMut(&Object)) -> bool { + if depth > MAX_WALK { + return false; + } + f(o); + // Every child is visited even after one is too deep: nothing is skipped silently. + let mut ok = true; + match o { + Object::Array(a) => a.iter().for_each(|x| ok &= walk_strings(x, depth + 1, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + _ => {} + } + ok +} + +/// Sweep a document (the output, or an attachment) on every surface. +pub(super) fn sweep_doc(needles: &[Needle], doc: &Document, bytes: &[u8], depth: usize, tag: &str, acc: &mut Acc) { + let (embedded, overlay_raw) = sweep_objects(needles, doc, depth, tag, acc); + sweep_file(needles, doc, bytes, overlay_raw, depth, tag, acc); + sweep_pages(needles, doc, depth, tag, acc); + sweep_info(needles, doc, tag, acc); + sweep_xmp(needles, doc, tag, acc); + + // Revision structure: one revision, no earlier content kept. + acc.scanned(Surface::Revisions); + let revs = doc.revisions().len(); + if revs > 1 || doc.trailer().contains(b"Prev") { + acc.survivor(Surface::Revisions, format!("{tag}{revs} revisions: earlier ones keep the original content"), None); + } + + sweep_attachments(needles, doc, &embedded, depth, tag, acc); +} + +/// Every object: strings, names and stream data (orphans included). Returns the embedded file +/// streams found, and the raw bytes of the overlay streams (to leave out of the raw sweep). +fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) -> (BTreeSet, Vec>) { + let mut embedded: BTreeSet = BTreeSet::new(); + let mut overlay_raw: Vec> = Vec::new(); + let mut in_streams = Tally::default(); + let mut in_names = Tally::default(); + let lookup = lookup_streams(doc); + for num in doc.object_numbers() { + let loc = format!("{tag}object {num}"); + let obj = match doc.try_get(num) { + Ok(o) => o, + Err(_) => { + acc.problem(Surface::ObjectStrings, format!("{loc} can't be read")); + continue; + } + }; + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + // The file identifier is random bytes, never text: a short needle must not "read" it. + let without_id = match &*obj { + Object::Stream(s) if s.dict.name(b"Type") == Some(b"XRef") => { + let mut d = s.dict.clone(); + d.remove(b"ID"); + Some(Object::Dict(d)) + } + _ => None, + }; + let complete = walk_strings(without_id.as_ref().unwrap_or(&obj), 0, &mut |o| match o { + Object::String(s) => match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + }, + Object::Name(n) => match bytes_hits(needles, n, MIN_NEEDLE, ctx) { + // A name the input had its own matches in is structure, not a leftover copy: + // the matches are tallied and settled against the input's count. + Some(h) => h.into_iter().for_each(|ni| in_names.add(ni, &loc, 1)), + None => spent = true, + }, + _ => {} + }); + if !complete { + acc.problem(Surface::ObjectStrings, format!("{loc} is nested too deeply to sweep in full")); + } + if spent { + acc.problem(Surface::ObjectStrings, "the search work limit was reached".into()); + } + acc.scanned(Surface::ObjectStrings); + for n in hits { + acc.survivor(Surface::ObjectStrings, loc.clone(), Some(n)); + } + // File specifications, wherever they sit (a name tree keeps them as separate objects, + // annotations and collections may inline them). + walk_strings(&obj, 0, &mut |o| { + if let Object::Dict(d) = o + && let Some(Object::Dict(ef)) = d.get(b"EF") + { + embedded.extend(ef.iter().filter_map(|(_, v)| v.as_ref()).map(|r| r.num)); + } + }); + if let Object::Stream(s) = &*obj { + if s.dict.name(b"Type") == Some(b"EmbeddedFile") { + embedded.insert(num); + } + acc.scanned(Surface::DecodedStreams); + if binary_stream(&s.dict) && image_codec(&s.dict) { + // The pixels of an image codec are still encoded and are not text: their stored + // bytes are searched for whole strings only. + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } else if binary_stream(&s.dict) { + // A compressed copy of the removed text can hide in a font program or colour + // profile (with /Length1 in its dict, its stored bytes hold no plaintext): the + // stored bytes and the decoded data are both searched. Such a stream that won't + // decode (damaged, or past the cap) is searched as stored bytes only and counted + // as raw only: it holds no page content, and redaction never touches it. A + // cross-reference or object stream that won't decode is a problem of the proof. + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + match acc.ctx.decode(s) { + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + Err(()) if acc.ctx.decode_spent() || matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm")) => { + acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")); + } + Err(()) => acc.raw_only(Surface::DecodedStreams), + } + } else { + match acc.ctx.decode(s) { + // The boxes and labels this run drew are not text to look for. + Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + // Colour-space lookup data and function samples are data tables, not content. + Err(()) if !acc.ctx.decode_spent() && (lookup.contains(&num) || s.dict.contains(b"FunctionType")) => { + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } + Err(()) => acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")), + } + } + } + } + in_names.settle_names(Surface::ObjectStrings, needles, depth, acc); + in_streams.settle(Surface::DecodedStreams, needles, depth, acc); + (embedded, overlay_raw) +} + +/// The streams that hold the lookup table of an `/Indexed` colour space (object numbers). +fn lookup_streams(doc: &Document) -> BTreeSet { + let mut out = BTreeSet::new(); + for num in doc.object_numbers() { + let Ok(obj) = doc.try_get(num) else { continue }; + walk_strings(&obj, 0, &mut |o| { + if let Object::Array(a) = o + && matches!(a.first(), Some(Object::Name(n)) if n == b"Indexed" || n == b"I") + && let Some(Object::Ref(r)) = a.get(3) + { + out.insert(r.num); + } + }); + } + out +} + +/// The raw bytes of the whole file, the overlay streams (stored as they are, or not at all) and +/// the file identifier (random bytes, regenerated on every write) aside. +fn sweep_file(needles: &[Needle], doc: &Document, bytes: &[u8], overlay_raw: Vec>, depth: usize, tag: &str, acc: &mut Acc) { + acc.scanned(Surface::RawBytes); + let mut mask = overlay_raw; + mask.extend(id_spellings(doc)); + let masked = mask_all(bytes, &mask, &acc.ctx); + let mut in_file = Tally::default(); + for (n, c) in acc.counts(Surface::RawBytes, needles, masked.as_deref().unwrap_or(bytes), MIN_STRUCTURED_NEEDLE) { + in_file.add(n, &format!("{tag}file"), c); + } + in_file.settle(Surface::RawBytes, needles, depth, acc); + if masked.is_none() && !mask.is_empty() { + acc.problem(Surface::RawBytes, "the search work limit was reached".into()); + } +} + +/// Embedded files, PDF attachments swept in full. +fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet, depth: usize, tag: &str, acc: &mut Acc) { + for &num in embedded { + let loc = format!("{tag}attachment object {num}"); + acc.scanned(Surface::EmbeddedFiles); + let obj = doc.get(ObjRef::new(num, doc.generation(num))); + let Object::Stream(s) = &*obj else { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is not a stream")); + continue; + }; + let payload = match acc.ctx.decode(s) { + Ok(p) => p, + Err(()) => { + acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); + continue; + } + }; + sweep_payload(needles, &payload, depth, &loc, acc); + } +} + +/// `bytes` with every occurrence of `pats` zeroed (`None` when the work limit ran out, or +/// there is nothing to mask). +pub(super) fn mask_all(bytes: &[u8], pats: &[Vec], ctx: &Ctx) -> Option> { + if pats.iter().all(Vec::is_empty) { + return None; + } + let mut out = bytes.to_vec(); + for p in pats.iter().filter(|p| !p.is_empty()) { + if !ctx.charge(bytes.len()) { + return None; + } + let mut at = 0; + while let Some(w) = bytes.get(at..at + p.len()) { + if w == p.as_slice() { + if let Some(z) = out.get_mut(at..at + p.len()) { + z.fill(0); + } + at += p.len(); + } else { + at += 1; + } + } + } + Some(out) +} + +/// One attachment: a PDF is swept as a document, text is searched, containers are reported as +/// unswept (their content can't be seen), images are searched as raw bytes only. +pub(super) fn sweep_payload(needles: &[Needle], payload: &[u8], depth: usize, loc: &str, acc: &mut Acc) { + let head = payload.get(..1024).unwrap_or(payload); + let tag = format!("{loc} > "); + if find(head, b"%PDF-") { + if depth >= MAX_NESTING { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is nested too deep to sweep")); + return; + } + match Document::open(Arc::new(payload.to_vec())) { + Ok(inner) => { + let before = acc.totals(); + sweep_doc(needles, &inner, payload, depth + 1, &tag, acc); + // What the attachment's own sweep found counts against the attachment too. + let after = acc.totals(); + if after.0 > before.0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), None); + } + if after.1 > before.1 { + acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be swept in full")); + } + } + Err(_) => acc.problem(Surface::EmbeddedFiles, format!("{loc} is a PDF that can't be opened")), + } + return; + } + let starts = |sig: &[u8]| payload.starts_with(sig); + let container = starts(b"PK\x03\x04") + || starts(b"PK\x05\x06") + || starts(&[0x1F, 0x8B]) + || starts(b"BZh") + || starts(&[0xFD, b'7', b'z', b'X', b'Z', 0]) + || starts(&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C]) + || starts(b"Rar!") + || starts(&[0xD0, 0xCF, 0x11, 0xE0]) + || payload.get(257..262) == Some(b"ustar"); + if container { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is an archive or container that can't be opened")); + return; + } + let image = starts(&[0x89, b'P', b'N', b'G']) || starts(&[0xFF, 0xD8, 0xFF]) || starts(b"GIF8"); + if image { + acc.raw_only(Surface::EmbeddedFiles); + } + let min = if image { MIN_STRUCTURED_NEEDLE } else { MIN_NEEDLE }; + for n in acc.bytes(Surface::EmbeddedFiles, needles, payload, min) { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(n)); + } + if !image { + let key = squash(&decode_text(payload)); + match key_counts_from(needles, &key, false, &acc.ctx, MIN_NEEDLE) { + Some(counts) => { + for (i, count) in counts { + if count > 0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(i)); + } + } + } + None => acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be searched in full")), + } + } +} + +pub(super) fn sweep_pages(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) { + let mut shown = Tally::default(); + 'pages: for (i, p) in pdfcraft_model::pages(doc).iter().enumerate() { + acc.scanned(Surface::ExtractedText); + let loc = format!("{tag}page {}", i + 1); + let Ok(e) = extract_page(doc, &p.dict, &acc.ctx, &[]) else { + acc.problem(Surface::ExtractedText, format!("{loc} can't be read")); + continue; + }; + for (hay, raw) in [(&e.text, false), (&e.raw, true)] { + let Some(counts) = key_counts(needles, hay, raw, &acc.ctx) else { + acc.problem(Surface::ExtractedText, "the search work limit was reached".into()); + break 'pages; + }; + for (ni, count) in counts { + shown.add(ni, &loc, count); + } + } + } + // A surplus over what legitimately remains anywhere in the document is a survivor. The + // counts made before the work ran out are settled too: a survivor already seen is + // reported where it sits, not hidden behind the limit. + shown.settle(Surface::ExtractedText, needles, depth, acc); +} + +pub(super) fn sweep_info(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(info) = doc.trailer().get(b"Info").map(|i| doc.resolve(i)) else { return }; + let Some(d) = info.as_dict() else { + acc.problem(Surface::Info, format!("{tag}Info is not a dictionary")); + return; + }; + for (k, v) in d.iter() { + acc.scanned(Surface::Info); + let v = doc.resolve(v); + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + let complete = walk_strings(&v, 0, &mut |o| { + if let Object::String(s) = o { + match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + } + } + }); + if !complete || spent { + acc.problem(Surface::Info, format!("{tag}Info /{} can't be swept in full", String::from_utf8_lossy(k))); + } + for n in hits { + acc.survivor(Surface::Info, format!("{tag}Info /{}", String::from_utf8_lossy(k)), Some(n)); + } + } +} + +pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(root) = doc.root() else { return }; + let Some(meta) = doc.get(root).as_dict().and_then(|c| c.get(b"Metadata")).map(|m| doc.resolve(m)) else { return }; + acc.scanned(Surface::Xmp); + let loc = format!("{tag}XMP metadata"); + let Object::Stream(s) = &*meta else { + acc.problem(Surface::Xmp, format!("{loc} is not a stream")); + return; + }; + let Ok(data) = acc.ctx.decode(s) else { + acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); + return; + }; + let mut hits: BTreeSet = acc.bytes(Surface::Xmp, needles, &data, MIN_NEEDLE).into_iter().collect(); + let key = squash(&xml_unescape(&decode_text(&data))); + for (i, n) in needles.iter().enumerate() { + if n.at_least(MIN_NEEDLE) && !n.raw_view && key.contains(&n.key) { + hits.insert(i); + } + } + for n in hits { + acc.survivor(Surface::Xmp, loc.clone(), Some(n)); + } +} diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs new file mode 100644 index 0000000000..77c4309cdd --- /dev/null +++ b/crates/redact/src/verify/util.rs @@ -0,0 +1,301 @@ +//! Shared helpers of the proof: what one proof may spend, text and byte search, and the +//! spellings a string can have in a PDF. + +use std::cell::Cell; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::sync::Arc; + +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; +use sha2::{Digest, Sha256}; + +use crate::verify::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_NEEDLE}; + +/// What one proof may spend, and what it knows about the document it checks. +pub(super) struct Ctx { + /// Digests of the decoded content of the overlay streams (their boxes and labels are drawn + /// on purpose and are neither text to look for nor paint to flag). + pub(super) overlay: HashSet, + /// Decoded bytes of the proof so far: one allowance for the whole proof, failures charged + /// too (see [`Ctx::decode`]), so a stream referenced many times, or one that fails after + /// decoding part of itself, cannot decode again for free. + pub(super) decoded: Cell, + pub(super) work: Cell, +} + +impl Ctx { + pub(super) fn new(overlay: HashSet) -> Self { + Ctx { overlay, decoded: Cell::new(0), work: Cell::new(0) } + } + + /// Decode `s` strictly: a stream that is damaged, longer than [`MAX_STREAM`], or one more + /// than [`MAX_TOTAL_DECODED`] allows is an error (never silently truncated data). Every + /// attempt is charged to the cumulative allowance, failures included: a decode that fails + /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded + /// before the error — the work of finding out is not work a repeated reference gets for free. + pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); + let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); + let mut budget = DecodeBudget::new(cap, usize::MAX); + let result = budget.decode_strict(s); + // The attempt's own accounting covers a success (what it decoded), a stream cut short + // by corruption (what it decoded first) and one refused past the cap (the cap): charge + // it through, whichever way the attempt went. + let spent = cap.saturating_sub(budget.remaining()); + self.decoded.set(self.decoded.get().saturating_add(spent as u64)); + result.map_err(|_| ()) + } + + /// Is the cumulative decode allowance used up? A decode refused then says nothing about the + /// stream, so it must not pass for a stream that can't be decoded. + pub(super) fn decode_spent(&self) -> bool { + self.decoded.get() >= MAX_TOTAL_DECODED + } + + /// Spend `n` units of search work; false once [`MAX_WORK`] is used up. + pub(super) fn charge(&self, n: usize) -> bool { + let used = self.work.get().saturating_add(n as u64); + self.work.set(used); + used <= MAX_WORK + } + + /// Is this decoded content one of the overlay streams this run created? + pub(super) fn is_overlay(&self, data: &[u8]) -> bool { + !self.overlay.is_empty() && self.overlay.contains(&sha256(&[data])) + } + + /// (tests) A context whose cumulative decode allowance has `left` bytes left. + #[cfg(test)] + pub(super) fn with_decode_left(left: u64) -> Self { + Ctx { overlay: HashSet::new(), decoded: Cell::new(MAX_TOTAL_DECODED.saturating_sub(left)), work: Cell::new(0) } + } +} + +/// Follow `o` to its object; a reference that can't be loaded is an error (a missing object is +/// `Null` for [`Document::resolve`], which would read as "empty"). +pub(super) fn resolve_strict(doc: &Document, o: &Object) -> Result, ()> { + match o { + Object::Ref(r) => doc.try_get(r.num).map_err(|_| ()), + other => Ok(Arc::new(other.clone())), + } +} + +pub(super) fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +pub(super) fn sha256>(parts: &[T]) -> String { + let mut h = Sha256::new(); + for p in parts { + let p = p.as_ref(); + h.update((p.len() as u64).to_be_bytes()); + h.update(p); + } + hex(h.finalize().as_slice()) +} + +/// The text of `s` without whitespace (page text is compared whitespace-blind: a gap in a `TJ` +/// is a space in one extraction and nothing in another). +pub(super) fn squash(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}').collect() +} + +pub(super) fn latin1(b: &[u8]) -> String { + b.iter().map(|&c| char::from(c)).collect() +} + +/// The words of a run worth looking for on their own (a run that survives in pieces). +pub(super) fn words(s: &str) -> Vec<&str> { + s.split_whitespace().filter(|w| w.chars().count() >= MIN_NEEDLE).collect() +} + +pub(super) fn count(hay: &str, needle: &str) -> usize { + if needle.is_empty() { 0 } else { hay.matches(needle).count() } +} + +/// Is `pat` in `hay`? +pub(super) fn find(hay: &[u8], pat: &[u8]) -> bool { + find_from(hay, pat, 0).is_some() +} + +/// Where `pat` first occurs in `hay` at or after `from`. The pattern's prefix function is +/// built once and the haystack is walked once: the search is worst-case linear, so +/// adversarial input (a needle of one letter and a haystack full of it) can neither make it +/// quadratic nor spend more comparisons than the caller charges for. +pub(super) fn find_from(hay: &[u8], pat: &[u8], from: usize) -> Option { + let m = pat.len(); + if m == 0 { + return None; + } + // fail[i]: the longest proper prefix of `pat` that ends at `i` and is also a suffix there. + let mut fail = vec![0usize; m]; + let mut k = 0; + for (i, &c) in pat.iter().enumerate().skip(1) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + } + fail[i] = k; + } + let mut k = 0; + for (at, &c) in hay.iter().enumerate().skip(from) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + if k == m { + return Some(at + 1 - m); + } + } + } + None +} + +pub(super) fn escape_literal(b: &[u8], octal: bool) -> Vec { + let mut out = Vec::with_capacity(b.len() * 2); + for &c in b { + if octal { + out.extend(format!("\\{c:03o}").bytes()); + continue; + } + match c { + b'(' | b')' | b'\\' => out.extend([b'\\', c]), + b'\n' => out.extend(*b"\\n"), + b'\r' => out.extend(*b"\\r"), + b'\t' => out.extend(*b"\\t"), + _ => out.push(c), + } + } + out +} + +/// Every way `text` can be spelled in a PDF: UTF-8, UTF-16BE (with and without a byte order +/// mark) and UTF-16LE, each as raw bytes, as a hex string (either case) and as a backslash- +/// escaped literal string (minimal escapes, and octal for every byte). +pub(crate) fn survivor_encodings(text: &str) -> Vec> { + let utf16 = |be: bool, bom: bool| -> Vec { + let mut v = Vec::new(); + if bom { + v.extend(if be { [0xFE, 0xFF] } else { [0xFF, 0xFE] }); + } + for u in text.encode_utf16() { + v.extend(if be { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + let forms = [text.as_bytes().to_vec(), utf16(true, true), utf16(true, false), utf16(false, true), utf16(false, false)]; + spellings(forms) +} + +/// The spellings of a byte string (see [`survivor_encodings`]). +pub(super) fn spellings(forms: impl IntoIterator>) -> Vec> { + let mut out: Vec> = Vec::new(); + for f in forms { + if f.is_empty() { + continue; + } + let h = hex(&f); + out.push(h.to_uppercase().into_bytes()); + out.push(h.into_bytes()); + out.push(escape_literal(&f, false)); + out.push(escape_literal(&f, true)); + out.push(f); + } + let mut seen = HashSet::new(); + out.retain(|b| seen.insert(b.clone())); + out +} + +/// Text of a payload: UTF-16 when it has a byte order mark, else UTF-8 (lossy). +pub(super) fn decode_text(b: &[u8]) -> String { + let units = |be: bool, rest: &[u8]| -> String { + let u: Vec = rest.as_chunks::<2>().0.iter().map(|c| if be { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) }).collect(); + String::from_utf16_lossy(&u) + }; + match b { + [0xFE, 0xFF, rest @ ..] => units(true, rest), + [0xFF, 0xFE, rest @ ..] => units(false, rest), + _ => String::from_utf8_lossy(b).into_owned(), + } +} + +pub(super) fn xml_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(rest.get(..at).unwrap_or_default()); + rest = rest.get(at..).unwrap_or_default(); + match xml_entity(rest) { + Some((c, len)) => { + out.push(c); + rest = rest.get(len..).unwrap_or_default(); + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + out.push_str(rest); + out +} + +/// The entity starting at `&`: the character it reads as, and how far it runs (through the +/// closing `;`). A numeric reference may carry any number of leading zeros, so its digits +/// are padded without a bound; more significant digits than a character value can hold, or +/// a missing `;`, is not an entity and the `&` stays as it is. Walking only the digits the +/// reference itself holds keeps the decode linear in the input. +fn xml_entity(s: &str) -> Option<(char, usize)> { + for (name, c) in [("&", '&'), ("<", '<'), (">", '>'), (""", '"'), ("'", '\'')] { + if s.starts_with(name) { + return Some((c, name.len())); + } + } + let body = s.strip_prefix("&#")?; + let (radix, digits) = match body.as_bytes().first() { + Some(b'x' | b'X') => (16u32, &body[1..]), + _ => (10, body), + }; + let mut value: u32 = 0; + let mut digits_held = false; + let mut end = None; + for (i, c) in digits.char_indices() { + match c.to_digit(radix) { + Some(d) => { + digits_held = true; + // Leading zeros only pad; past a character's worth of significant digits the + // multiplication overflows and this was never an entity. + value = value.checked_mul(radix)?.checked_add(d)?; + } + None => { + if c == ';' && digits_held { + end = Some(i); + } + break; + } + } + } + let end = end?; + let ch = char::from_u32(value)?; + Some((ch, 2 + usize::from(radix == 16) + end + 1)) +} + +/// Names of the filters of a stream dictionary. +pub(super) fn filter_names(d: &Dict) -> Vec> { + match d.get(b"Filter") { + Some(Object::Name(n)) => vec![n.clone()], + Some(Object::Array(a)) => a.iter().filter_map(|o| o.as_name().map(<[u8]>::to_vec)).collect(), + _ => Vec::new(), + } +} + +/// Image codecs: their data is pixels, still encoded. +pub(super) fn image_codec(d: &Dict) -> bool { + filter_names(d).iter().any(|n| matches!(n.as_slice(), b"DCTDecode" | b"DCT" | b"JPXDecode" | b"CCITTFaxDecode" | b"CCF" | b"JBIG2Decode")) +} From 97cef90e016fc81d6683996e88c6eed13c1b8be7 Mon Sep 17 00:00:00 2001 From: Hamza Naveed <45464935+Hamza0101@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:01:42 +0500 Subject: [PATCH 69/85] Redaction: a pattern nothing paints with any more leaves the page (#3680) * Redaction: a pattern nothing paints with any more leaves the page A tiling pattern's cell is a content stream of its own and can draw anything, text included. Redacting a shape filled with such a pattern removed the fill from the page's content -- the page looked right, a black box -- but left the pattern in /Resources /Pattern, and its cell, with the text that was under the mark, was written into the saved file unchanged. The pass does not enter pattern cells, so it cannot redact inside one. What it can do is notice that nothing paints with the pattern any more. The graphics state now carries the fill and stroke pattern a `scn` or `SCN` selected (and `cs`, `g`, `rg`, `k` and their stroke forms clear it), every painting operator that survives the rewrite records the pattern it drew with, and a pattern the page selected but no surviving paint used leaves the page's resources. Measured on the report's file: before applied: 1, (SECRET) in the saved bytes after applied: 1, no /Pattern on the page, (SECRET) gone A pattern some surviving paint still uses is kept, and so is its cell: with two boxes filled from one pattern and only the first marked, patterns_removed is 0 and the resources keep the entry -- the second box still shows the word, so it is not a leak. Report gains patterns_removed. Fixes #3647. * Keep a pattern every operator that can paint with it still uses Two defects CodeRabbit found in the first version of this change, both of which could remove a pattern the page still draws with. Pattern use was recorded for path painting only. Text shows with the fill pattern too -- and with the stroke pattern in the stroking render modes -- as do image masks and anything a form draws, and a form's content is interpreted with a graphics state of its own, so a pattern it inherits is never seen inside it. Content like /Pattern cs /P0 scn 20 140 100 30 re f BT (KEEP) Tj ET with the box redacted therefore lost the fill of text still on the page. Every operator that can paint with the current pattern now records it: text (except mode 3, which paints nothing, and mode 7, which only clips), inline images, and image or form XObjects. Only path painting knows exactly what it drew and whether the paint survived, so the others are recorded as they are met. That can keep a pattern whose paint was in fact all redacted, which costs an unused resource entry, while dropping one something still draws with takes paint off the page. /Resources /Pattern may be an indirect object, like any value. Reading the reference itself as a dictionary found none and pruned nothing, so on such a file the cell -- with the text that was under the mark -- stayed in the saved bytes, which is the leak this change is for. It is resolved now, and what is written back is a dictionary of this page's own, so pruning never reaches through a /Pattern dictionary another page shares. The controls in the test: without the first fix the surviving text's pattern is removed (left: 1, right: 0), and without the second the indirect file prunes nothing (left: 0, right: 1). * M8: refuse retained unsafe tiling patterns * M8: guard shared appearances and narrow pattern scans * M8: update TLS dependencies for new advisories * Redaction: the comment on prune_patterns said more than the code does `prune_patterns` drops the `/Pattern` resource entry; the `scn` that selected it stays in the content, so the page names a pattern its resources no longer hold. The comment claimed the selector went too (noted in review). ISO 32000-1 8.6.6.2 makes that operand a key of the `/Pattern` subdictionary, so the open question is whether a reader stops reading the stream at the stale name. `a_page_whose_pruned_pattern_is_still_selected_renders` answers it with the renderer rather than with prose: the redacted page is saved, reopened and rasterized, and * it renders with no error and the same 300x300 page box, * the text written *after* the stale selector is still painted -- which is exactly what a reader would drop if it gave up at the bad operand, * the redaction overlay is black over the box the pattern filled. Control: with the drawing after the selector removed, the pixel assertion fails ("the text after the stale selector is still painted"), so it is reading that text and not a stray pixel. The selector is left in place rather than cut out: removing operators is a rewrite of the content this pass does not otherwise do, and nothing surviving paints with that colour. `pdfcraft-render` is a dev dependency of this crate only -- layer 4 on layer 3, which the layering allows. * WIP: pattern guard depth bound sized from the interpreter's * Pattern tests: keep the guard's cases reaching the guard now a hit path is removed whole * Pattern automation tests: keep the pattern in use where they test the guard, since a hit path is now removed whole * Redaction: a form without resources keeps the page pattern it still paints A form with no /Resources of its own paints with its caller's patterns, but its surviving pattern paint was recorded only in the form's own output and never reached the page. With the page's own fill under the mark removed whole, prune_patterns then dropped /P0 from the page while the form outside the mark still painted with it, so that fill lost its paint in the saved file. Pass the painted names up from a form that inherits its caller's resources. Only the painted side goes up, so this can keep a pattern, never prune one. * Redaction: a pruned pattern does not stay named by the page tree prune_patterns writes only the page's own /Resources, and the guard walked only pages' effective resources. A /Pages node that still named the pruned pattern kept its cell reachable from Root, so a full save wrote the cell, text included, while apply returned Ok. Three shapes did it: a page with no /Parent inheriting from the root, a /Parent that disagrees with the /Kids tree, and a root /Resources shadowed by a nearer holder that was retired. Patterns pruned from a page are now unlinked from page-tree holders no page inherits from, as unlink_doomed does for XObjects, when every /Parent agrees with the tree. Whenever a pattern was pruned (or a selected page had any), the guard also inspects the /Resources of every /Pages node Root reaches down /Kids and up each page's /Parent chain, so whatever stays reachable is proven safe or the redaction is refused, leaving the document unchanged. * Redaction: a Type 3 glyph without resources keeps the page pattern it paints A Type 3 font with no /Resources looks the names in its glyph descriptions up in the resources of the page or form showing it, so a glyph outside the mark can paint with the page's pattern. Text recorded only the fill and stroke pattern of the graphics state, so with the page's own fill under the mark removed whole, prune_patterns dropped the pattern and the glyph lost its paint. Showing such a font now keeps every pattern of the scope, as a form without /Resources keeps what it paints. * Redaction: say what text-recorded pattern use keeps Text, inline images and XObjects record their pattern before the hit test, so a pattern painted only by redacted text keeps its cell. The guard refuses a cell with text or images; a vector-only cell stays in the file. The comment said this cost only an unused resource entry. Pin the behaviour in a test. * Redaction: the page tree is followed up every /Parent link A full save writes every node a /Parent link reaches, but the guard followed /Parent only from pages and stopped at the first node it had already seen, and unlinking never followed it. A /Pages node reached only through the /Parent of a /Kids node -- a root /Pages with a /Parent, or an intermediate node whose /Parent leads out of the tree -- kept naming the pruned pattern, and its cell, text included, stayed in the saved file while apply returned Ok. The guard now follows /Parent from every node it visits, page or /Pages, with the same seen-set against cycles. Unlinking treats a root with a /Parent as an inconsistent tree, like a disagreeing link, and leaves the decision to the guard, which inspects every node those links reach and refuses an unsafe cell. * Redaction: a long page tree does not exhaust the pattern guard's budget The page-tree walk counted every node, pages included, against the guard's 16,384-object budget, so redacting a pattern page of a 17,000-page document was refused with a limit error where it had applied before. Only the holder nodes the guard inspects count against that budget now; the walk itself is bounded at a million nodes, as unlink_doomed bounds the same tree. * Redaction: a Type 3 font set by an ExtGState or shown in a form keeps the page patterns The resourceless Type 3 check covered only a font set by Tf in the scope that paints. A Type 3 font without /Resources set through an ExtGState /Font entry was not flagged, and a glyph shown inside a form with /Resources of its own recorded its patterns in the form's scope, which the page never saw. In both cases the page's own fill under the mark was removed whole, prune_patterns dropped the pattern, and the glyph outside the mark lost its paint. An ExtGState font is now checked the same way, and a glyph shown in any nested form marks the page as keeping every pattern it selected. This can only keep patterns; whatever is kept is checked by the guard. * Redaction: a pruned pattern a full save would still write is proven safe or refused The walks that check retained pattern cells follow the references they know -- resource categories, the page tree, appearances -- but a full save writes whatever the trailer reaches by any reference. A direct /Parent dictionary, a /Kids array on a page, or a stray resource entry (/Shading naming the pattern stream) kept a pruned cell, text included, in the saved file while apply returned Ok. pdfcraft_cos gains reachable_objects, which runs the same breadth-first walk write_full uses (factored out as full_save_order, so the two cannot drift). Before apply succeeds, every pruned pattern that walk still reaches is inspected like a retained cell: vector-only passes, anything else refuses with the existing pattern error. It runs only when a pattern was pruned and costs one walk of the reachable objects. * Redaction: a direct /Parent dictionary is part of the page tree too A /Parent given as a dictionary rather than a reference is written inline by a full save, but the page-tree walk followed /Parent only as a reference, and unlinking took such a tree for consistent. A root /Pages whose direct /Parent held /Resources naming the pruned pattern kept its cell in the saved file. The guard now walks a direct /Parent like any node (its /Resources, /Parent and /Kids), and unlinking counts any /Parent that is not the reference /Kids came from as an inconsistent tree, leaving the decision to the guard. * Redaction: page-tree walks queue each node once and borrow /Kids On a hostile tree whose /Pages nodes share one long /Kids array, the guard's page-tree walk and pattern unlinking pushed every /Kids entry of every node before checking whether it had been seen, and copied the array on each visit. Both now queue only nodes not seen yet and read /Kids in place; unlinking also stops at the first node reached twice, since its /Parent can agree with at most one of the nodes above it, and leaves that tree to the guard. * Redaction: a Type 3 font with /Resources null is resourceless too /Resources null is the same as no /Resources, but the check counted any entry, so such a font's glyph outside the mark lost the page pattern it painted with. A Type 3 glyph shown by a kept annotation appearance is a known limit, pinned as an ignored test: appearances are not interpreted for pattern use, and under ISO 32000 the glyph resolves in the appearance's own resources. * Redaction: a pruned pattern is recorded along its whole reference chain A pattern name can lead to a reference-valued object whose value is the cell (/P0 7 0 R, object 7 being 6 0 R). Pruning recorded only object 7, so the reachability backstop checked an object nothing reached while a stray reference kept the cell, object 6, in the saved file. Every object along the chain is recorded now, bounded like the other walks. * Pattern tests: pin the file identifier where two saves are compared main now gives a redacted rewrite a new identifier from fresh entropy, so an incremental and a full save after Apply differ in /ID alone. The test that checks they are the same save passes fixed entropy to both. * Redaction: replies and pop-ups removed on other pages feed the shared-appearance check Apply checks every page for retained unsafe pattern cells when a removed annotation's appearance is shared with one that stays on an unselected page. Only annotations removed under a mark fed that check. Replies and pop-ups that go with them in step 2b (removed_everywhere), often listed on another page, did not, so a text-bearing cell in an appearance a kept annotation shares with them escaped the all-pages check and Apply returned Ok. Their appearance references now go into removed_appearances too, with the same limit. Raised by FredApps' review of the rebase. --------- Co-authored-by: skylar taylor-barrick --- Cargo.lock | 1 + ROADMAP.md | 2 + crates/automation/tests/pattern_redaction.rs | 146 ++++ crates/cos/src/lib.rs | 2 +- crates/cos/src/writer.rs | 40 +- crates/redact/Cargo.toml | 3 + crates/redact/README.md | 9 + crates/redact/src/interp.rs | 127 ++- crates/redact/src/interp/prune.rs | 136 ++++ crates/redact/src/lib.rs | 137 +++- crates/redact/src/pattern_guard.rs | 496 ++++++++++++ crates/redact/src/tests.rs | 194 +++++ crates/redact/tests/pattern_page_tree.rs | 221 +++++ .../redact/tests/pattern_page_tree_shapes.rs | 275 +++++++ crates/redact/tests/pattern_reachability.rs | 304 +++++++ crates/redact/tests/pattern_redaction.rs | 758 ++++++++++++++++++ docs/roadmap.md | 2 +- 17 files changed, 2837 insertions(+), 16 deletions(-) create mode 100644 crates/automation/tests/pattern_redaction.rs create mode 100644 crates/redact/src/pattern_guard.rs create mode 100644 crates/redact/tests/pattern_page_tree.rs create mode 100644 crates/redact/tests/pattern_page_tree_shapes.rs create mode 100644 crates/redact/tests/pattern_reachability.rs create mode 100644 crates/redact/tests/pattern_redaction.rs diff --git a/Cargo.lock b/Cargo.lock index e1f271ed12..8617c45522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4628,6 +4628,7 @@ dependencies = [ "pdfcraft-fonts", "pdfcraft-forms", "pdfcraft-model", + "pdfcraft-render", "pdfcraft-testkit", "pdfcraft-xfa", "sha2 0.11.1", diff --git a/ROADMAP.md b/ROADMAP.md index 4fcdadaee9..820ceb0179 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -357,6 +357,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M5, arrow keys move comments):** With a comment selected, the arrow keys move it a screen pixel and ⌘/Ctrl or Shift and an arrow ten, instead of scrolling the view or turning the page; up is up on screen at any zoom and rotation, and each press is one undo step. Text markup and locked comments stay put. Estimates unchanged. +- **2026-10-20 (M8, #3647):** Apply prunes fully removed tiling patterns, then checks retained pattern resources for text, images and unreadable content before reporting success. Shared patterns, Form-owned and inherited patterns, and cells kept by another page cause an atomic refusal when safe removal cannot be shown. Synthetic saved-file and automation regressions cover refusal, preserved marks and undo, and vector-only controls. Estimates unchanged. + - **2026-10-20 (M1, PR #1143 review):** Repair-aware dictionary/preflight edits and snapshot restoration preserve both accepted generations of a damaged live slot. Such edits use the existing full-save fallback; classic and packed rewrites coalesce aliases so page-tree parent links retain one object identity. Explicit generation replacement remains single-generation. Materialized xref diagnostics inspect retained primary and hybrid rows before folding, retaining the 256-object cap plus summary and discarding abandoned-index notes. Synthetic red/green controls cover preflight, undo restoration, equal values, explicit narrowing, encryption packing and four save variants. Estimates unchanged. - **2026-10-19 (M14, #3373):** The main menu and its long submenus scroll in short windows, keeping their commands reachable. Estimates unchanged. diff --git a/crates/automation/tests/pattern_redaction.rs b/crates/automation/tests/pattern_redaction.rs new file mode 100644 index 0000000000..667779cef6 --- /dev/null +++ b/crates/automation/tests/pattern_redaction.rs @@ -0,0 +1,146 @@ +//! Rectangle redaction must not report a pattern-cell secret safely removed (#3647). + +use std::sync::Arc; + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_engine::DocId; +use serde_json::{Value, json}; + +fn tool(automation: &mut Automation, name: &str, args: Value) -> Value { + match automation.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +/// `retained`: paint with the pattern once more, away from the mark (x 290..340 lies right of it). +/// A path the mark hits is removed whole (F-008) and a pattern nothing surviving paints with is +/// pruned, so only a pattern that some surviving paint still uses reaches the guard. +fn fixture(cell: &[u8], partial: bool, compressed: bool, retained: bool) -> Vec { + let mut pattern = Dict::from_iter([ + (b"Type".to_vec(), Object::name("Pattern")), + (b"PatternType".to_vec(), Object::Int(1)), + (b"PaintType".to_vec(), Object::Int(1)), + (b"TilingType".to_vec(), Object::Int(1)), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"XStep".to_vec(), Object::Int(500)), + (b"YStep".to_vec(), Object::Int(300)), + ]); + pattern.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Font".to_vec(), + Object::Dict(Dict::from_iter([(b"F1".to_vec(), Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))])), + )])), + ); + let width = if partial { 400 } else { 170 }; + let survivor = if retained { " 290 115 50 30 re f" } else { "" }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET /Pattern cs /P0 scn 35 110 {width} 35 re f{survivor}"); + let mut page_stream = Vec::new(); + pdfcraft_cos::serialize(&Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes())), &mut page_stream); + let mut pattern_stream = Vec::new(); + let stream = if compressed { Stream::flate(pattern, cell) } else { Stream::from_raw(pattern, cell.to_vec()) }; + pdfcraft_cos::serialize(&Object::Stream(stream), &mut pattern_stream); + let objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >> >>" + .to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + page_stream, + pattern_stream, + ]; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend(format!("{} 0 obj\n", index + 1).as_bytes()); + bytes.extend(object); + bytes.extend(b"\nendobj\n"); + } + let xref = bytes.len(); + bytes.extend(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes()); + for offset in offsets { + bytes.extend(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes()); + bytes +} + +fn saved_has_secret(bytes: Vec) -> bool { + let document = Document::open(Arc::new(bytes)).unwrap(); + document.object_numbers().into_iter().any(|num| { + let object = document.try_get(num).unwrap(); + matches!(&*object, Object::Stream(stream) if stream.decoded().unwrap().windows(10).any(|part| part == b"CELLSECRET")) + }) +} + +#[test] +fn pattern_redaction_tools_remove_or_refuse_without_leaking_or_changing_undo() { + let dir = tempfile::tempdir().unwrap(); + for (partial, retained) in [(false, false), (true, false), (true, true)] { + for compressed in [false, true] { + let cell = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET BT /F1 20 Tf 300 120 Td (SAFE) Tj ET"; + let bytes = fixture(cell, partial, compressed, retained); + assert!(saved_has_secret(bytes.clone())); + std::fs::write(dir.path().join("input.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "input.pdf"}))["doc"].as_u64().unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let before = automation.session().get(DocId(id)).unwrap().bytes.clone(); + let pixels = automation.render_pixels(id, 1, 1.0).unwrap(); + match automation.call("redact_apply", &json!({"doc": id})) { + Ok(_) => { + assert!(!retained, "a text-bearing cell still in use must not be reported safe (partial: {partial})"); + tool(&mut automation, "doc_save", json!({"doc": id, "path": "redacted.pdf"})); + assert!( + !saved_has_secret(std::fs::read(dir.path().join("redacted.pdf")).unwrap()), + "successful apply/save left the pattern-cell text recoverable" + ); + } + Err(error) => { + assert!(retained, "a pattern nothing surviving paints with should apply (partial: {partial}): {error}"); + assert!(error.to_string().contains("tiling pattern"), "{error}"); + assert!(!error.to_string().contains("CELLSECRET")); + let document = automation.session().get(DocId(id)).unwrap(); + assert_eq!(document.bytes, before); + assert_eq!(document.redaction_marks(), 1); + assert_eq!(document.can_undo(), Some("Add redaction mark")); + assert_eq!(automation.render_pixels(id, 1, 1.0).unwrap(), pixels); + tool(&mut automation, "edit_undo", json!({"doc": id})); + assert_eq!(automation.session().get(DocId(id)).unwrap().redaction_marks(), 0); + } + } + } + } +} + +#[test] +fn pattern_redaction_vector_control_saves_reopens_and_preserves_outside_pixels() { + let dir = tempfile::tempdir().unwrap(); + // The guard's control: the cell is plain vectors and a paint outside the mark still uses it, + // so it is inspected and kept, not refused, and what that paint draws is unchanged. + let bytes = fixture(b"0.2 0.4 0.6 rg 40 120 15 15 re f 300 120 30 20 re f", true, true, true); + std::fs::write(dir.path().join("vectors.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "vectors.pdf"}))["doc"].as_u64().unwrap(); + let (width, height, original) = automation.render_pixels(id, 1, 1.0).unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let applied = tool(&mut automation, "redact_apply", json!({"doc": id})); + assert_eq!(applied["applied"], 1); + let save = tool(&mut automation, "doc_save", json!({"doc": id, "path": "vectors-redacted.pdf"})); + assert_eq!(save["incremental"], false); + let reopened = tool(&mut automation, "doc_open", json!({"path": "vectors-redacted.pdf"}))["doc"].as_u64().unwrap(); + let (new_width, new_height, actual) = automation.render_pixels(reopened, 1, 1.0).unwrap(); + assert_eq!((new_width, new_height), (width, height)); + for (index, (before, after)) in original.as_chunks::<4>().0.iter().zip(actual.as_chunks::<4>().0).enumerate() { + let (x, y) = (index % width as usize, index / width as usize); + if x >= 280 { + assert_eq!(before, after, "unmarked pattern pixels changed at {x},{y}"); + } + if (35..205).contains(&x) && (155..190).contains(&y) { + assert_eq!(after, &[0, 0, 0, 255], "marked interior must be black"); + } + } +} diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 8e051e4578..32a6f6d8b3 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::{Lexer, is_whitespace, parse_indirect}; pub use pdfcraft_crypt::{ Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, }; -pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental}; +pub use writer::{SaveOptions, pdf_date, reachable_objects, serialize, write_full, write_incremental}; #[derive(Debug, thiserror::Error, Clone, PartialEq)] pub enum CosError { diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index e0762612dc..0af74f7bab 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -273,16 +273,12 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, Ok(file) } -/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. -pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - // A full traversal must not fill the editor's shared caches (which undo snapshots also - // own). Parsed objects and decoded object streams are temporary working data here. - let mut reader = doc.object_reader(); - stamp_mod_date(&mut reader.document, opts); - let trailer_in = reader.document.trailer().clone(); +/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and +/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets +/// (aliases of one slot included) and, in visit order, whether it is a stream. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>); + +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -326,6 +322,30 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro order.push((canonical, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); } + Ok((map, order)) +} + +/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] +/// reaches from the trailer, object streams included. For a check that something removed from a +/// document cannot reach the saved file through a reference nobody looked for. +pub fn reachable_objects(doc: &Document) -> Result, CosError> { + let mut reader = doc.object_reader(); + let trailer_in = reader.document.trailer().clone(); + let (_, order) = full_save_order(&mut reader, &trailer_in)?; + Ok(order.into_iter().map(|(r, _)| r.num).collect()) +} + +/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. +pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + // A full traversal must not fill the editor's shared caches (which undo snapshots also + // own). Parsed objects and decoded object streams are temporary working data here. + let mut reader = doc.object_reader(); + stamp_mod_date(&mut reader.document, opts); + let trailer_in = reader.document.trailer().clone(); + let (map, order) = full_save_order(&mut reader, &trailer_in)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index c02335e3c4..149d8d8295 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -25,3 +25,6 @@ sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } +# The redacted page is rendered in `tests::a_page_whose_pruned_pattern_is_still_selected_renders`: +# layer 3 from layer 4, a dev dependency only. +pdfcraft-render = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index db9f46b8c9..f78f819c03 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -60,6 +60,15 @@ resume after it would be a guess. A Redact annotation with neither a usable `/Qu `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. +Tiling-pattern cells are separate streams. Applying a mark removes a pattern resource when no +surviving paint uses it. If a cell remains reachable after rewriting the selected pages and may +contain text, images, replacement descriptions, or unreadable content, Apply refuses with a +`RedactError::Pattern` and restores the document and pending marks. This includes patterns owned +by Forms and patterns inherited by Forms. Vector-only retained cells remain supported. The +bounded check is conservative: an unrelated retained pattern on a selected page can also cause +refusal when its content cannot be proven safe. If a selected page had a pattern resource, the +check covers all pages after rewriting so another page cannot keep the same cell in the saved PDF. + Before changing any page, `apply` prepares the redaction appearances with a shared 8 MiB budget for their uncompressed PDF content across the selected pages. It bounds repeated-text counts and working line buffers before allocating them, and measures and encodes each repeated diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 7a2cd26eca..b0a7257cb3 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -31,7 +31,7 @@ mod prune; use layers::strip_hidden_layers; use prune::{clip_out, extgstate_font, prune_form_resources}; -pub(crate) use prune::{prune, prune_properties, resource_ops, unlink_doomed}; +pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns}; /// How deep this pass follows form XObjects. A form past it is removed rather than walked, which /// is the fail-safe choice for a redaction -- marked content inside a form we cannot read would @@ -85,6 +85,17 @@ struct Gs { scale: f64, leading: f64, rise: f64, + /// The tiling or shading pattern selected as the fill colour (`/Name scn`), and as the stroke + /// colour (`/Name SCN`). Part of the graphics state, so `q`/`Q` save and restore them with + /// everything else. A pattern's cell is a content stream of its own and this pass does not + /// enter it, so a pattern whose only painting was removed has to leave the page's resources + /// or its drawing stays in the file (#3647). + fill_pattern: Option>, + stroke_pattern: Option>, + /// The current font is a Type 3 font without `/Resources`: its glyph descriptions look their + /// names up in this scope's resources (ISO 32000-2 §9.6.4), so a shown glyph may paint with + /// any of this scope's patterns. + type3_inherits: bool, } /// What processing a scope produced. @@ -102,6 +113,16 @@ pub(crate) struct Output { /// dictionaries may name them too (an inherited or shared /Resources, or the caller's when a /// form has none of its own): [`unlink_doomed`] removes those entries. pub doomed: Vec, + /// Pattern resource names a painting operator that survived the rewrite still uses. A name + /// this scope saw selected but never painted with afterwards is pruned from the page's + /// `/Resources /Pattern`, so a cell whose only drawing was redacted does not stay in the + /// saved file (#3647). + pub patterns_seen: Vec>, + pub patterns_painted: Vec>, + /// A Type 3 glyph without resources of its own was shown in this scope or a nested form, + /// through a font set by `Tf` or an ExtGState: which of the page's patterns it paints with is + /// not tracked, so the page keeps them all. + pub keeps_all_patterns: bool, /// Original property values retired by this scope or a nested form. Other resource holders /// may still name them; unlink only entries no surviving content uses. pub retired_properties: Vec, @@ -264,6 +285,22 @@ struct Glyph { rects: Vec, } +/// Record the patterns an operator paints with, so [`prune_patterns`] keeps them. Only path +/// painting knows exactly what it draws and whether the paint survived; text, images and forms +/// are recorded as they are met, which can keep a pattern whose paint was in fact all redacted, +/// and with it the pattern's cell. A cell that carries text or images is then refused by +/// `pattern_guard`; a vector-only cell stays in the file though nothing shows it +/// (`a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell`). Dropping a pattern something +/// still draws with would lose paint from the page instead. +fn uses_patterns(out: &mut Output, gs: &Gs, fills: bool, strokes: bool) { + if fills && let Some(n) = &gs.fill_pattern { + out.patterns_painted.push(n.clone()); + } + if strokes && let Some(n) = &gs.stroke_pattern { + out.patterns_painted.push(n.clone()); + } +} + /// Process a scope's content streams (state flows from one stream to the next). pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], resources: &Dict, ctm: Matrix) -> Output { let gs = Gs { @@ -277,6 +314,9 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< scale: 1.0, leading: 0.0, rise: 0.0, + fill_pattern: None, + stroke_pattern: None, + type3_inherits: false, }; process_from(doc, scope, streams, resources, gs) } @@ -377,6 +417,8 @@ struct Walk { fonts_res: Dict, ext_gstates: Dict, xobjects: Dict, + /// This scope's pattern names, which a Type 3 glyph without resources may paint with. + patterns: Vec>, fallback: Rc, /// The current path: its operators, its bounding box in user space, and whether it clips. path: Vec, @@ -395,6 +437,7 @@ impl Walk { fonts_res: res_dict(doc, resources, b"Font"), ext_gstates: res_dict(doc, resources, b"ExtGState"), xobjects: res_dict(doc, resources, b"XObject"), + patterns: res_dict(doc, resources, b"Pattern").iter().map(|(n, _)| n.to_vec()).collect(), fallback: Rc::new(Metrics::fallback()), path: Vec::new(), path_box: None, @@ -433,6 +476,39 @@ impl Walk { b"BI" => Self::inline_image(&self.gs, scope, &mut self.out, op, piece), b"sh" if scope.mode == Mode::Apply => Self::shading(&self.gs, scope, op, piece), b"Do" => self.draw_xobject(doc, scope, resources, op, piece), + // `/Name scn` selects a pattern as the fill colour; numbers select a plain colour + // and clear it. `cs` with a space other than /Pattern clears it too. + b"scn" | b"SCN" => { + let fill = o == b"scn"; + let picked = op.operands.last().and_then(Object::as_name).map(<[u8]>::to_vec); + if let Some(name) = &picked { + self.out.patterns_seen.push(name.clone()); + } + if fill { + self.gs.fill_pattern = picked; + } else { + self.gs.stroke_pattern = picked; + } + Some(op) + } + b"cs" | b"CS" => { + if op.name(0) != Some(b"Pattern") { + if o == b"cs" { + self.gs.fill_pattern = None; + } else { + self.gs.stroke_pattern = None; + } + } + Some(op) + } + b"g" | b"rg" | b"k" => { + self.gs.fill_pattern = None; + Some(op) + } + b"G" | b"RG" | b"K" => { + self.gs.stroke_pattern = None; + Some(op) + } _ => Some(op), }; piece.ops.extend(kept); @@ -464,6 +540,12 @@ impl Walk { let clips = std::mem::replace(&mut self.clip, false); let hit = scope.mode == Mode::Apply && op.op.as_slice() != b"n" && bbox.is_some_and(|b| scope.hits(b)); if !hit { + // The paint survives, so the pattern it draws with is still in use. `n` paints + // nothing, so it uses none. A hit path is removed whole (F-008), pattern and all. + let o = op.op.as_slice(); + let strokes = matches!(o, b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*"); + let fills = matches!(o, b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*"); + uses_patterns(&mut self.out, &self.gs, fills, strokes); piece.ops.extend(body); piece.ops.push(op); return; @@ -497,6 +579,8 @@ impl Walk { _ => { if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); + // As for `Tf`: a Type 3 font without /Resources paints through this scope's. + self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } } } @@ -507,6 +591,7 @@ impl Walk { let Some(name) = op.name(0) else { return }; let font_dict = self.fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); self.gs.vertical = font_dict.as_ref().is_some_and(|d| is_vertical(doc, d)); + self.gs.type3_inherits = font_dict.as_ref().is_some_and(|d| type3_without_resources(doc, d)); self.gs.font = match scope.fonts.get(name) { Some(f) => f.clone(), None => { @@ -555,6 +640,16 @@ impl Walk { /// `Tj`, `TJ`, `'` and `"`: glyphs under a region are cut out (or counted, when verifying). fn show_text(&mut self, scope: &mut Scope<'_>, op: Op, piece: &mut Piece) -> Option { + // Text paints with the fill pattern, and with the stroke pattern in the stroking render + // modes. Mode 3 paints nothing and mode 7 only clips. + let mode = self.gs.render_mode; + uses_patterns(&mut self.out, &self.gs, matches!(mode, 0 | 2 | 4 | 6), matches!(mode, 1 | 2 | 5 | 6)); + // A Type 3 glyph without resources of its own paints through this scope's: keep every + // pattern it could select, as with a form that has no /Resources. + if self.gs.type3_inherits { + self.out.patterns_painted.extend(self.patterns.iter().cloned()); + self.out.keeps_all_patterns = true; + } let o = op.op.as_slice(); if o == b"\"" { self.gs.word_spacing = op.num(0).unwrap_or(self.gs.word_spacing); @@ -608,6 +703,8 @@ impl Walk { /// An inline image under a region is removed (or counted, when verifying). fn inline_image(gs: &Gs, scope: &mut Scope<'_>, out: &mut Output, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, pattern included. + uses_patterns(out, gs, true, false); if !scope.hits(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { return Some(op); } @@ -632,6 +729,9 @@ impl Walk { } fn draw_xobject(&mut self, doc: &mut Document, scope: &mut Scope<'_>, resources: &Dict, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, and a form's content is interpreted with a + // graphics state of its own, so a pattern it inherits from here is never recorded inside it. + uses_patterns(&mut self.out, &self.gs, true, true); let Some(new) = xobject(doc, scope, &op, &self.gs, resources, &mut self.xobjects, &mut self.out) else { return Some(op) }; piece.changed = true; piece.ops.extend(new); @@ -924,8 +1024,18 @@ fn xobject( out.ambiguous |= inner.ambiguous; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); + // A Type 3 glyph without resources shown inside the form may paint with the page's + // patterns (the renderer resolves its names there), whether or not the form has + // /Resources of its own. + out.keeps_all_patterns |= inner.keeps_all_patterns; out.tag_changes.extend(inner.tag_changes); out.error = inner.error.or(out.error.take()); + // A form without /Resources of its own paints with its caller's patterns, so what it + // still paints keeps the caller's entry. Only the painted side is passed up: a name + // the form selected but no longer paints with is left for the caller to keep. + if own.is_none() { + out.patterns_painted.extend(inner.patterns_painted); + } if scope.mode == Mode::Verify { return None; } @@ -992,3 +1102,18 @@ fn xobject( _ => None, } } + +/// Whether the ExtGState `name` sets a Type 3 font that has no `/Resources` of its own. +fn extgstate_type3_inherits(doc: &Document, ext_gstates: &Dict, name: &[u8]) -> bool { + let Some(state) = ext_gstates.get(name).map(|s| doc.resolve(s)) else { return false }; + let Some(font) = state.as_dict().and_then(|d| d.get(b"Font")).map(|f| doc.resolve(f)) else { return false }; + let Some(reference) = font.as_array().and_then(|a| a.first()) else { return false }; + let dict = doc.resolve(reference); + dict.as_dict().is_some_and(|d| type3_without_resources(doc, d)) +} + +/// A Type 3 font whose glyph descriptions use the resources of whatever shows them: no +/// `/Resources`, or `/Resources null`, which is the same thing. +fn type3_without_resources(doc: &Document, font: &Dict) -> bool { + font.name(b"Subtype") == Some(b"Type3") && font.get(b"Resources").is_none_or(|r| matches!(&*doc.resolve(r), Object::Null)) +} diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 6ca5ea2803..188af03abe 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -355,3 +355,139 @@ pub(super) fn extgstate_font( }; Some((metrics, vertical, size)) } + +/// Drop every `/Pattern` entry this scope saw selected but no surviving painting operator drew +/// with. The `scn` that selected it stays in the content, so the page names a pattern its +/// resources no longer hold; nothing surviving paints with that colour, and +/// `a_page_whose_pruned_pattern_is_still_selected_renders` pins that the page still renders and +/// that operators after the stale selector still draw. A pattern's cell is a content stream of its own which +/// this pass does not enter, so a pattern whose only painting was redacted would otherwise keep +/// its drawing -- text included -- in the saved file (#3647). A pattern some surviving paint +/// still uses, or one the scope never saw, is left alone. +/// `/Pattern` may be an indirect object like any other value, so it is resolved: reading the +/// reference itself as a dictionary found none and pruned nothing, leaving the cell in the file. +/// What is written back is a dictionary of this page's own, so pruning never reaches through a +/// `/Pattern` dictionary another page shares. The pattern objects dropped are added to `pruned`, +/// for [`unlink_pruned_patterns`]. +pub(crate) fn prune_patterns(doc: &Document, res: &mut Dict, seen: &[Vec], painted: &[Vec], pruned: &mut Vec) -> usize { + let Some(mut pat) = res.get(b"Pattern").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()) else { return 0 }; + let mut dropped = 0; + for n in seen { + if !painted.contains(n) + && let Some(value) = pat.get(n) + { + // Every object along the reference chain: a name can lead to a reference-valued + // object whose value is the cell, and the cell is what must not stay reachable. + let mut next = value.as_ref(); + for _ in 0..MAX_DEPTH { + let Some(reference) = next else { break }; + pruned.push(reference); + next = match &*doc.get(reference) { + Object::Ref(r) => Some(*r), + _ => None, + }; + } + pat.remove(n); + dropped += 1; + } + } + if dropped > 0 { + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + } + dropped +} + +/// Unlink the patterns a page's rewrite pruned (`pruned`) from page-tree nodes whose +/// `/Resources` no page inherits any more: a full save keeps whatever Root still reaches, so a +/// `/Pages` node above the page that still names the cell would keep it in the file. That happens +/// when the page's nearest holder was retired but a holder further up, which it shadowed, still +/// names the pattern. A holder some page still inherits from is left alone; whatever stays +/// reachable is checked by `pattern_guard`, which refuses a retained cell it cannot prove safe. +/// When any node's `/Parent` disagrees with where the `/Kids` tree puts it (or the root has +/// one), which holder a page inherits from is ambiguous and a node may be reached only through +/// `/Parent`, so nothing is unlinked and the guard, which follows `/Parent` too, decides. +pub(crate) fn unlink_pruned_patterns(doc: &mut Document, pruned: &[ObjRef]) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let pruned: HashSet = pruned.iter().copied().collect(); + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(()) }; + // Every node with a /Resources of its own, and whether any page inherits from it (it is the + // nearest holder above a page, or the page itself). Cycles are cut as in `unlink_doomed`. + let mut holders: Vec<(ObjRef, bool)> = Vec::new(); + let mut seen = HashSet::new(); + let mut consistent = true; + let mut stack: Vec<(ObjRef, Option, Option)> = vec![(top, None, None)]; + seen.insert(top); + while let Some((node, parent, mut holder)) = stack.pop() { + let obj = doc.get(node); + let Some(d) = obj.as_dict() else { continue }; + // Every node's /Parent must be the node /Kids reached it from, and the root has none. + // A node reached only through a /Parent link is then impossible here; when the links + // disagree, the guard inspects every node they reach instead. A /Parent that is not a + // reference (a direct dictionary, written inline by a full save) never agrees. + let agrees = match d.get(b"Parent") { + None => parent.is_none(), + Some(Object::Ref(r)) => Some(*r) == parent, + Some(_) => false, + }; + if !agrees { + consistent = false; + break; + } + if d.contains(b"Resources") { + holders.push((node, false)); + holder = Some(holders.len() - 1); + } + let is_pages = d.name(b"Type") == Some(b"Pages") || (d.contains(b"Kids") && d.name(b"Type") != Some(b"Page")); + if is_pages { + // Borrowed, not copied: a /Kids array can be shared by many hostile nodes. + let kids_object = d.get(b"Kids").map(|k| doc.resolve(k)); + for kid in kids_object.as_deref().and_then(Object::as_array).into_iter().flatten().filter_map(Object::as_ref) { + // A node reached twice (a shared /Kids entry or a cycle) has a /Parent that agrees + // with at most one of the nodes above it. + if !seen.insert(kid) || seen.len() > 1_000_000 { + consistent = false; + break; + } + stack.push((kid, Some(node), holder)); + } + if !consistent { + break; + } + } else if let Some(h) = holder.and_then(|i| holders.get_mut(i)) { + h.1 = true; + } + } + if !consistent { + return Ok(()); + } + for (node, inherited) in holders { + if inherited { + continue; + } + let Some(mut res) = doc.get(node).as_dict().and_then(|d| d.get(b"Resources")).and_then(|r| doc.resolve(r).as_dict().cloned()) else { + continue; + }; + let mut pat = res_dict(doc, &res, b"Pattern"); + let names: Vec> = pat.iter().filter(|(_, v)| v.as_ref().is_some_and(|r| pruned.contains(&r))).map(|(n, _)| n.to_vec()).collect(); + if names.is_empty() { + continue; + } + for n in &names { + pat.remove(n); + } + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + // A private copy on this node: the dictionary object may be shared with other nodes. + doc.update_dict(node, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + } + Ok(()) +} diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 6bb8e5cc06..6d323004ad 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -22,6 +22,7 @@ pub mod codes; mod image; mod interp; mod overlay; +mod pattern_guard; pub mod patterns; pub mod sanitize; mod tags; @@ -34,6 +35,7 @@ pub(crate) mod type3_tables; pub mod verify; use interp::{Mode, Scope, process}; +pub use pattern_guard::PatternProblem; pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -53,6 +55,8 @@ pub enum RedactError { #[error("redaction could not be verified: {0} item(s) were still found under the marks, so nothing was changed")] Residue(usize), /// A mark's overlay text has a character Helvetica can't draw; nothing was redacted. + #[error("page {page} cannot be redacted safely: {reason}; nothing was changed")] + Pattern { page: usize, reason: PatternProblem }, #[error("{0}")] Undrawable(String), #[error( @@ -146,6 +150,8 @@ fn overlay_look(doc: &Document, d: &Dict) -> pdfcraft_annot::OverlayLook { /// What [`apply`] removed. #[derive(Clone, Debug, Default, PartialEq)] pub struct Report { + /// Patterns dropped from a page's resources because nothing painted with them any more. + pub patterns_removed: usize, pub marks: usize, pub pages: usize, pub glyphs: usize, @@ -397,6 +403,38 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result Option> { + const MAX_REFERENCES: usize = 16_384; + let mut references = HashSet::new(); + let Some(ap) = annotation.get(b"AP") else { return Some(references) }; + if let Some(reference) = ap.as_ref() { + references.insert(reference); + } + let resolved_ap = doc.resolve(ap); + let Some(appearances) = resolved_ap.as_dict() else { return Some(references) }; + for key in [b"N".as_slice(), b"R", b"D"] { + let Some(value) = appearances.get(key) else { continue }; + if let Some(reference) = value.as_ref() { + references.insert(reference); + } + let resolved = doc.resolve(value); + if let Object::Dict(states) = &*resolved { + for (_, state) in states.iter() { + if let Some(reference) = state.as_ref() { + references.insert(reference); + if references.len() > MAX_REFERENCES { + return None; + } + } + } + } + } + Some(references) +} + /// [`apply`] without the copy: an error can leave `doc` part redacted. fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result { let images = pdfcraft_cos::ImageBudget::new(doc); @@ -429,6 +467,16 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // content rewriting and verification instead of scanning all marks for every page. let groups: Vec<_> = chosen.chunk_by(|a, b| a.page == b.page).filter_map(|group| group.first().map(|mark| (mark.page, group))).collect(); let by_page: Vec<_> = groups.iter().map(|(page, _)| *page).collect(); + // A pattern pruned from a selected page can remain reachable through another page's + // resources. When a selected page had patterns, inspect every page after the rewrite. + let source_pages = pdfcraft_model::pages(doc); + let shared_pattern_possible = by_page.iter().any(|&pi| { + source_pages + .get(pi) + .and_then(|page| page.dict.get(b"Resources")) + .map(|resources| doc.resolve(resources)) + .is_some_and(|resources| resources.as_dict().is_some_and(|dict| dict.get(b"Pattern").is_some())) + }); // Budget failures must leave even a direct library caller's document untouched. let mut remaining = overlay::MAX_BYTES; let overlays = groups @@ -448,6 +496,9 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co widget_owner, removed_everywhere: HashSet::new(), images, + removed_appearances: HashSet::new(), + appearance_scan_limit: false, + pruned_patterns: Vec::new(), }; for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -458,7 +509,18 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if applying.images.refused() > 0 { return Err(RedactError::ImagesPastAllowance); } - let Applying { mut report, doomed_fields, doomed_xobjects, retired_properties, mut removed_everywhere, mut images, .. } = applying; + let Applying { + mut report, + doomed_fields, + doomed_xobjects, + retired_properties, + mut removed_everywhere, + mut images, + mut removed_appearances, + mut appearance_scan_limit, + pruned_patterns, + .. + } = applying; // 2b. Replies and pop-ups of a removed annotation can be listed on another page than their // parent: their `/IRT` or `/Parent` would keep the removed comment reachable. @@ -481,6 +543,19 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); if removed_everywhere.contains(&r) || is_reply || is_popup { + // A reply or pop-up removed here can share its appearance with an annotation + // that stays on an unselected page, like one removed under a mark: the + // all-pages pattern check must see it. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } if removed_everywhere.insert(r) { grew = true; if is_reply { @@ -541,6 +616,8 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // A removed image or form must not stay reachable through another resource dictionary (an // inherited or shared /Resources): a full save keeps whatever is still referenced. interp::unlink_doomed(doc, &doomed_xobjects, &retired_properties)?; + // Likewise a pruned pattern: a /Pages node above the page can still name its cell. + interp::unlink_pruned_patterns(doc, &pruned_patterns)?; report.images_restored = restore_shared_jbig2(doc, &doomed_xobjects, &mut images)?; // 5. Verify: nothing readable may remain under a region. @@ -563,6 +640,29 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if residue > 0 { return Err(RedactError::Residue(residue)); } + // The interpreter can remove a pattern's paint while another use keeps the cell reachable. + // Inspect the finished selected pages, including rewritten Forms, and all pages if a removed + // pattern or appearance may still be shared. Retained unsafe cells refuse the redaction; + // `apply` runs on a copy, so the refusal leaves the caller's document as it was. + let shared_appearance_possible = appearance_scan_limit + || !removed_appearances.is_empty() + && pdfcraft_model::pages(doc).iter().enumerate().any(|(index, page)| { + !by_page.contains(&index) + && annots_of(doc, &page.dict).iter().any(|annotation| { + doc.resolve(annotation).as_dict().is_some_and(|dict| { + appearance_refs(doc, dict).is_none_or(|refs| refs.iter().any(|reference| removed_appearances.contains(reference))) + }) + }) + }); + let first_page = by_page.first().map_or(1, |p| p + 1); + let check_pages = + if shared_pattern_possible || shared_appearance_possible { (0..pdfcraft_model::pages(doc).len()).collect::>() } else { by_page }; + // A pruned pattern may also stay named by a /Pages node no page inherits from, or by one a + // page with a broken /Parent link still reaches: then every page-tree node is inspected too. + pattern_guard::check(doc, &check_pages, shared_pattern_possible || !pruned_patterns.is_empty())?; + // Whatever reference keeps a pruned pattern in the file -- one no walk above follows -- the + // full save would write it: prove it safe or refuse. + pattern_guard::check_pruned_unreachable(doc, &pruned_patterns, first_page)?; // The previous revision still holds the removed content: the next save must rewrite, and the // rewritten file is a different file from its source, so it gets a new identifier. doc.require_full_save_with_new_id(); @@ -650,6 +750,13 @@ struct Applying { removed_everywhere: HashSet, /// What clearing images may spend, across the run's pages. images: pdfcraft_cos::ImageBudget, + /// Appearance streams of the annotations removed, on any page: one shared with a kept + /// annotation keeps a pruned pattern's cell reachable (see `pattern_guard`). + removed_appearances: HashSet, + /// Too many appearance references to track: the pattern guard inspects every page. + appearance_scan_limit: bool, + /// Pattern objects pruned from a page's resources, which page-tree nodes may still name. + pruned_patterns: Vec, } /// Step 1 for one page: rewrites its content so nothing readable stays under a mark, and cleans the tags it no longer shows. @@ -660,7 +767,7 @@ fn redact_page_content( page: &pdfcraft_model::Page, rects: &[[f64; 4]], ) -> Result<(), RedactError> { - let Applying { report, doomed_xobjects, retired_properties, images, .. } = applying; + let Applying { report, doomed_xobjects, retired_properties, images, pruned_patterns, .. } = applying; // 1. Content. let mut budget = pdfcraft_cos::DecodeBudget::page(); let (list, data) = page_streams_within(doc, &page.dict, pi, &mut budget)?; @@ -733,6 +840,11 @@ fn redact_page_content( }, pi, )?; + // A pattern this page selected but no surviving paint draws leaves its resources, so + // its cell -- a content stream this pass does not enter -- cannot stay in the file. + // A Type 3 glyph without resources paints with patterns nobody tracked: keep them all. + let painted = if out.keeps_all_patterns { &out.patterns_seen } else { &out.patterns_painted }; + report.patterns_removed += interp::prune_patterns(doc, &mut res, &out.patterns_seen, painted, pruned_patterns); retired_properties.extend(interp::prune_properties(doc, &mut res, &cleaned_data)?); type3::prune_type3_from_resources(doc, &mut res, &cleaned_data)?; doc.update_dict(page.obj, |d| { @@ -761,7 +873,7 @@ fn remove_page_annotations( rects: &[[f64; 4]], page_marks: &[&Mark], ) -> Result<(), RedactError> { - let Applying { report, doomed_fields, widget_owner, removed_everywhere, .. } = applying; + let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); let mut removed: Vec = Vec::new(); @@ -819,6 +931,25 @@ fn remove_page_annotations( } } removed_everywhere.extend(removed.iter().copied()); + // A removed annotation's appearance streams can be shared with a kept one on another page, + // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. + for reference in &removed { + if *appearance_scan_limit { + break; + } + if let Some(annotation) = doc.get(*reference).as_dict() { + match appearance_refs(doc, annotation) { + Some(references) => { + removed_appearances.extend(references); + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } + } + None => *appearance_scan_limit = true, + } + } + } // Pop-ups dropped with their parent below count too: a reply on another page may answer one. for a in &annots { let Some(r) = a.as_ref() else { continue }; diff --git a/crates/redact/src/pattern_guard.rs b/crates/redact/src/pattern_guard.rs new file mode 100644 index 0000000000..96b865e3da --- /dev/null +++ b/crates/redact/src/pattern_guard.rs @@ -0,0 +1,496 @@ +//! Refuse pattern content that remains reachable after redaction (#3647). +//! +//! Pattern cells paint through a path, not `Do`, so clipping/removing the path does not remove +//! their bytes. Inspect the rewritten pages' resource graphs before committing the +//! working copy. This is deliberately conservative: an unused text/image-bearing pattern is +//! refused too. A plain vector pattern remains supported. The bounded read-only pass follows +//! the fail-closed approach proposed in PR #162. + +use std::collections::HashSet; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use crate::RedactError; + +/// How deep the walk may recurse. Every step down (a reference, an array, a dictionary, a stream, +/// a resources map) is one frame, and a nested form costs about four of them: the 50-deep chain +/// in `a_deeply_nested_form_keeps_the_text_nobody_marked` peaks at 200. The interpreter follows +/// forms as deep as the renderer and Edit text read (`interp::MAX_DEPTH`, #3645), so this must +/// reach at least that far, or the guard would refuse a document the redaction itself handles. +/// Total work is bounded separately by `MAX_OBJECTS`, and 512 frames stay far inside a thread's +/// stack. +const MAX_DEPTH: usize = 512; +const _: () = assert!(MAX_DEPTH >= 8 * crate::interp::MAX_DEPTH, "the guard must walk past the deepest form the interpreter follows"); +const MAX_OBJECTS: usize = 16_384; +const MAX_STREAM_BYTES: usize = 1 << 20; +const MAX_TOTAL_BYTES: usize = 8 << 20; +const MAX_FILTERS: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum PatternProblem { + #[error("a tiling pattern carries text or images that cannot be removed safely")] + Content, + #[error("pattern resources or content cannot be read completely")] + Unreadable, + #[error("pattern inspection exceeded its resource, depth or decoded-content budget")] + Limit, +} + +struct Guard { + left: usize, + bytes_left: usize, + seen: HashSet<(ObjRef, bool)>, + active: HashSet<(ObjRef, bool)>, +} + +impl Guard { + fn new() -> Self { + Self { left: MAX_OBJECTS, bytes_left: MAX_TOTAL_BYTES, seen: HashSet::new(), active: HashSet::new() } + } + + fn charge(&mut self, depth: usize) -> Result<(), PatternProblem> { + self.left = self.left.checked_sub(1).ok_or(PatternProblem::Limit)?; + if depth > MAX_DEPTH { + return Err(PatternProblem::Limit); + } + Ok(()) + } + + fn walk(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + match object { + Object::Ref(reference) => { + let key = (*reference, inside); + if inside && self.active.contains(&key) { + return Err(PatternProblem::Unreadable); + } + if !self.seen.insert(key) { + return Ok(()); + } + let value = doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?; + self.active.insert(key); + let result = self.walk(doc, &value, inside, depth + 1); + self.active.remove(&key); + result + } + Object::Array(values) => { + for value in values { + self.walk(doc, value, inside, depth + 1)?; + } + Ok(()) + } + Object::Dict(dict) => self.dictionary(doc, dict, inside, false, depth + 1), + Object::Stream(stream) => { + if inside { + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, stream.dict.get(b"PatternType"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + if subtype.as_name() == Some(b"Form") { + if resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null { + return Err(PatternProblem::Unreadable); + } + self.content(doc, stream)?; + } else if pattern_type.as_int() == Some(1) { + self.content(doc, stream)?; + } + } + self.dictionary(doc, &stream.dict, inside, true, depth + 1) + } + _ => Ok(()), + } + } + + fn dictionary(&mut self, doc: &Document, dict: &Dict, inside: bool, stream: bool, depth: usize) -> Result<(), PatternProblem> { + // Stream dictionaries include appearance and soft-mask streams whose consumers + // may infer Form semantics without /Subtype. Resource-map names, in contrast, + // are arbitrary and must not be mistaken for dictionary field names. + let subtype = resolved(doc, dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, dict.get(b"PatternType"))?; + let owns_resources = stream || matches!(subtype.as_name(), Some(b"Form" | b"Type3")) || pattern_type.as_int() == Some(1); + for (key, value) in dict.iter() { + if owns_resources && key.as_slice() == b"Resources" { + self.resources(doc, value, inside, depth + 1)?; + } else if inside || !owns_resources { + self.walk(doc, value, inside, depth + 1)?; + } + } + Ok(()) + } + + fn resources(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + let object = resolved(doc, Some(object))?; + // A Form's null Resources is equivalent to an absent entry: it uses the already + // inspected caller resources. Tiling-pattern resources are required separately. + if object == Object::Null { + return Ok(()); + } + let Some(resources) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (category, value) in resources.iter() { + // Resource dictionaries may include large embedded fonts and colour tables. Only + // structures that can own another pattern cell need recursive inspection. + if !matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Font" | b"Properties" | b"ExtGState") { + continue; + } + if category.as_slice() == b"Font" { + let fonts = resolved(doc, Some(value))?; + let Some(fonts) = fonts.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, font) in fonts.iter() { + self.charge(depth)?; + let font = resolved(doc, Some(font))?; + let Some(dict) = font.as_dict() else { return Err(PatternProblem::Unreadable) }; + if resolved(doc, dict.get(b"Subtype"))?.as_name() == Some(b"Type3") + && let Some(resources) = dict.get(b"Resources") + { + self.resources(doc, resources, inside, depth + 1)?; + } + } + continue; + } + if matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Properties" | b"ExtGState") && resolved(doc, Some(value))? == Object::Null { + continue; + } + if category.as_slice() == b"Pattern" { + let object = resolved(doc, Some(value))?; + let Some(patterns) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, pattern) in patterns.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(pattern))?; + if object == Object::Null { + continue; + } + let Some(dict) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + let kind = resolved(doc, dict.get(b"PatternType"))?; + match (kind.as_int(), &object) { + (Some(1), Object::Stream(_)) => { + // Do not guess a fallback for a malformed cell, which different + // readers may resolve differently. Pattern resources are mandatory. + if resolved(doc, dict.get(b"Resources"))?.as_dict().is_none() { + return Err(PatternProblem::Unreadable); + } + } + (Some(2), Object::Dict(_)) => {} + _ => return Err(PatternProblem::Unreadable), + } + self.walk(doc, pattern, true, depth + 1)?; + } + } else { + if (inside && matches!(category.as_slice(), b"XObject" | b"Properties")) || category.as_slice() == b"ExtGState" { + let object = resolved(doc, Some(value))?; + let Some(entries) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if object == Object::Null { + continue; + } + if category.as_slice() == b"Properties" { + let Some(properties) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + replacement_text(doc, properties)?; + } else if category.as_slice() == b"ExtGState" { + let Some(state) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + match resolved(doc, state.get(b"SMask"))? { + Object::Null => {} + Object::Name(name) if name == b"None" => {} + Object::Dict(mask) => { + let group = resolved(doc, mask.get(b"G"))?; + let Object::Stream(group) = group else { return Err(PatternProblem::Unreadable) }; + readable_form(doc, &group)?; + if let Some(value) = mask.get(b"G") { + self.walk(doc, value, inside, depth + 1)?; + } + } + _ => return Err(PatternProblem::Unreadable), + } + } else { + let Object::Stream(stream) = object else { return Err(PatternProblem::Unreadable) }; + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + readable_form(doc, &stream)?; + } + } + } + if category.as_slice() == b"XObject" { + let entries = resolved(doc, Some(value))?; + let Some(entries) = entries.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if let Object::Stream(stream) = object + && resolved(doc, stream.dict.get(b"Subtype"))?.as_name() == Some(b"Form") + { + self.walk(doc, entry, inside, depth + 1)?; + } + } + } + } + } + Ok(()) + } + + fn content(&mut self, doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // Strict decoding with a small bound, unlike the tolerant general reader. Normalize + // indirect filter declarations before using Stream::filters, which accepts direct data. + let normalized = normalized_stream(doc, stream)?; + let chain = normalized.filters(); + let max = MAX_STREAM_BYTES.min(self.bytes_left); + if chain.is_empty() && normalized.raw.len() > max { + return Err(PatternProblem::Limit); + } + let data = pdfcraft_filters::decode(&chain, &normalized.raw, max).map_err(|error| match error { + pdfcraft_filters::FilterError::LimitExceeded(_) => PatternProblem::Limit, + _ => PatternProblem::Unreadable, + })?; + self.bytes_left = self.bytes_left.checked_sub(data.len()).ok_or(PatternProblem::Limit)?; + let parsed = pdfcraft_content::parse(&data); + if parsed.skipped != 0 { + return Err(PatternProblem::Unreadable); + } + for op in parsed.ops { + self.charge(0)?; + if matches!(op.op.as_slice(), b"BDC" | b"DP") { + for operand in &op.operands { + if let Some(properties) = resolved(doc, Some(operand))?.as_dict() { + replacement_text(doc, properties)?; + } + } + } + match op.op.as_slice() { + b"Tj" | b"TJ" | b"'" | b"\"" | b"BI" => return Err(PatternProblem::Content), + // Unknown operators cannot establish that no hidden content was skipped. + b"q" | b"Q" | b"cm" | b"w" | b"J" | b"j" | b"M" | b"d" | b"ri" | b"i" | b"gs" | b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" + | b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n" | b"W" | b"W*" | b"BT" | b"ET" | b"Tc" | b"Tw" | b"Tz" + | b"TL" | b"Tf" | b"Tr" | b"Ts" | b"Td" | b"TD" | b"Tm" | b"T*" | b"d0" | b"d1" | b"CS" | b"cs" | b"SC" | b"SCN" | b"sc" | b"scn" + | b"G" | b"g" | b"RG" | b"rg" | b"K" | b"k" | b"sh" | b"Do" | b"MP" | b"DP" | b"BMC" | b"BDC" | b"EMC" | b"BX" | b"EX" => {} + _ => return Err(PatternProblem::Unreadable), + } + } + Ok(()) + } +} + +fn readable_form(doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // PS, Form/Subtype2 PS, reference XObjects and unknown/missing subtypes cannot be + // proven free of recoverable pattern content by the PDF content parser. + if resolved(doc, stream.dict.get(b"Subtype"))?.as_name() != Some(b"Form") + || resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null + || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null + { + return Err(PatternProblem::Unreadable); + } + Ok(()) +} + +fn replacement_text(doc: &Document, properties: &Dict) -> Result<(), PatternProblem> { + for key in [b"ActualText".as_slice(), b"Alt", b"E"] { + if resolved(doc, properties.get(key))? != Object::Null { + return Err(PatternProblem::Content); + } + } + Ok(()) +} + +fn resolved(doc: &Document, value: Option<&Object>) -> Result { + let mut value = value.cloned().unwrap_or(Object::Null); + for _ in 0..MAX_DEPTH { + let Object::Ref(reference) = value else { return Ok(value) }; + value = (*doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?).clone(); + } + Err(PatternProblem::Limit) +} + +fn normalized_stream(doc: &Document, stream: &Stream) -> Result { + if stream.dict.contains(b"F") || stream.dict.contains(b"FFilter") || stream.dict.contains(b"FDecodeParms") || stream.dict.contains(b"DP") { + return Err(PatternProblem::Unreadable); + } + let mut normalized = stream.clone(); + let filter = resolved(doc, stream.dict.get(b"Filter"))?; + let names = match &filter { + Object::Null => Vec::new(), + Object::Name(_) => vec![filter], + Object::Array(values) if values.len() <= MAX_FILTERS => values.iter().map(|v| resolved(doc, Some(v))).collect::, _>>()?, + _ => return Err(PatternProblem::Unreadable), + }; + if names.iter().any(|name| name.as_name().is_none()) { + return Err(PatternProblem::Unreadable); + } + let parms = resolved(doc, stream.dict.get(b"DecodeParms"))?; + let parms = match parms { + Object::Null => vec![Object::Null; names.len()], + Object::Dict(_) if names.len() == 1 => vec![parms], + Object::Array(values) if values.len() == names.len() => values, + _ => return Err(PatternProblem::Unreadable), + }; + let parms = parms + .iter() + .map(|value| { + let value = resolved(doc, Some(value))?; + let Object::Dict(mut dict) = value else { + return if value == Object::Null { Ok(value) } else { Err(PatternProblem::Unreadable) }; + }; + for key in [b"Predictor".as_slice(), b"Colors", b"BitsPerComponent", b"Columns", b"EarlyChange"] { + if let Some(value) = dict.get(key) { + let value = resolved(doc, Some(value))?; + if value.as_int().is_none() { + return Err(PatternProblem::Unreadable); + } + dict.set(key.to_vec(), value); + } + } + if !matches!(dict.int(b"Predictor").unwrap_or(1), 1 | 2 | 10..=15) + || dict.int(b"Colors").unwrap_or(1) <= 0 + || dict.int(b"Columns").unwrap_or(1) <= 0 + || !matches!(dict.int(b"BitsPerComponent").unwrap_or(8), 1 | 2 | 4 | 8 | 16) + || !matches!(dict.int(b"EarlyChange").unwrap_or(1), 0 | 1) + { + return Err(PatternProblem::Unreadable); + } + Ok(Object::Dict(dict)) + }) + .collect::, _>>()?; + normalized.dict.set(b"Filter".to_vec(), Object::Array(names)); + normalized.dict.set(b"DecodeParms".to_vec(), Object::Array(parms)); + Ok(normalized) +} + +/// Page-tree nodes a walk of the tree may visit, pages included, before it refuses: the page tree +/// is walked whole elsewhere too (`pdfcraft_model::pages`, `unlink_doomed`), so a long document +/// must not refuse here. +const MAX_TREE_NODES: usize = 1_000_000; + +/// Every node a full save reaches through the page tree -- down `/Kids` and up the `/Parent` of +/// every node visited, page or `/Pages`, so a node reached only through a `/Parent` link counts +/// too -- that has `/Resources` of its own and is not one of the document's pages: those are +/// walked with their effective resources. Cycles are cut by the seen-set. Holders are bounded by +/// `MAX_OBJECTS`, pages are not. +fn tree_holders(doc: &Document) -> Result, PatternProblem> { + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(Vec::new()) }; + let pages: HashSet = pdfcraft_model::pages(doc).iter().map(|p| p.obj).collect(); + // The `/Resources` values to inspect. A direct `/Parent` dictionary is written inline by a + // full save, so it is a node like any other: its resources, `/Parent` and `/Kids` count. + let mut holders = Vec::new(); + let mut seen = HashSet::new(); + seen.insert(top); + let mut stack = vec![top]; + let mut inline: Vec = Vec::new(); + // One node: queue what it links to that was not seen yet (so a /Kids array shared by many + // nodes costs its length once per node, never a copy), and keep its resources. + let mut visit = + |dict: &Dict, is_page: bool, stack: &mut Vec, inline: &mut Vec, holders: &mut Vec| -> Result<(), PatternProblem> { + match dict.get(b"Parent") { + Some(Object::Ref(parent)) => { + if seen.insert(*parent) { + stack.push(*parent); + } + } + Some(Object::Dict(parent)) => { + if inline.len() >= MAX_DEPTH { + return Err(PatternProblem::Limit); + } + inline.push(parent.clone()); + } + _ => {} + } + if let Some(resources) = dict.get(b"Resources") + && !is_page + { + holders.push(resources.clone()); + if holders.len() > MAX_OBJECTS { + return Err(PatternProblem::Limit); + } + } + let is_pages = dict.name(b"Type") == Some(b"Pages") || (dict.contains(b"Kids") && dict.name(b"Type") != Some(b"Page")); + if is_pages { + let kids_object = match dict.get(b"Kids") { + Some(Object::Ref(reference)) => Some(doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?), + _ => None, + }; + let kids = match &kids_object { + Some(object) => object.as_array(), + None => dict.get(b"Kids").and_then(Object::as_array), + }; + for kid in kids.into_iter().flatten().filter_map(Object::as_ref) { + if seen.insert(kid) { + stack.push(kid); + } + } + } + if seen.len() > MAX_TREE_NODES { + return Err(PatternProblem::Limit); + } + Ok(()) + }; + loop { + if let Some(node) = stack.pop() { + let object = doc.try_get(node.num).map_err(|_| PatternProblem::Unreadable)?; + if let Some(dict) = object.as_dict() { + visit(dict, pages.contains(&node), &mut stack, &mut inline, &mut holders)?; + } + } else if let Some(dict) = inline.pop() { + visit(&dict, false, &mut stack, &mut inline, &mut holders)?; + } else { + break; + } + } + Ok(holders) +} + +/// Validate the requested pages before `apply` commits its working copy. With `tree`, the +/// `/Resources` of every page-tree node above the pages is inspected as well: a node no page +/// inherits from, or one a page with a broken `/Parent` link still reaches, can keep a pruned +/// cell in the saved file. +pub(crate) fn check(doc: &Document, selected: &[usize], tree: bool) -> Result<(), RedactError> { + if tree && let Some(&first) = selected.first() { + let page = first.saturating_add(1); + let holders = tree_holders(doc).map_err(|reason| RedactError::Pattern { page, reason })?; + for resources in holders { + let mut guard = Guard::new(); + guard.resources(doc, &resources, false, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + let pages = pdfcraft_model::pages(doc); + for &index in selected { + let Some(page) = pages.get(index) else { return Err(RedactError::Unreadable(index.saturating_add(1))) }; + let mut guard = Guard::new(); + let checked = (|| { + if let Some(resources) = page.dict.get(b"Resources") { + guard.resources(doc, resources, false, 0)?; + } + // Kept appearance streams may own their own pattern resources. + for annotation in crate::annots_of(doc, &page.dict) { + let object = resolved(doc, Some(&annotation))?; + if let Some(appearance) = object.as_dict().and_then(|dict| dict.get(b"AP")) { + guard.walk(doc, appearance, false, 0)?; + } + } + Ok(()) + })(); + checked.map_err(|reason| RedactError::Pattern { page: index.saturating_add(1), reason })?; + } + Ok(()) +} + +/// The backstop after every other check: a full save writes whatever the trailer reaches, by any +/// reference at all -- a stray resource category, a key on a page, a direct /Parent, `/AcroForm +/// /DR`, `/PieceInfo` -- and the walks above follow only the references they know. A pattern +/// pruned from a page that a full save would still write must therefore be proven safe like any +/// retained cell (vector-only), or the redaction is refused. Costs one walk of the reachable +/// objects, and only when a pattern was pruned. +pub(crate) fn check_pruned_unreachable(doc: &Document, pruned: &[ObjRef], page: usize) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let reachable = pdfcraft_cos::reachable_objects(doc)?; + let mut guard = Guard::new(); + let mut checked = HashSet::new(); + for &pattern in pruned { + if reachable.contains(&pattern.num) && checked.insert(pattern.num) { + guard.walk(doc, &Object::Ref(pattern), true, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + Ok(()) +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 78cff889d3..8978e0e7d6 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -290,6 +290,200 @@ fn vectors_are_removed_or_clipped_and_inline_images_go() { assert!(c.contains("150 150 m"), "the line elsewhere stays"); } +/// A tiling pattern's cell is a content stream of its own and can draw text. Redacting a shape +/// filled with such a pattern removed the fill from the page but left the pattern in the page's +/// resources, so its cell -- with the text that was under the mark -- stayed in the saved file +/// (#3647). A pattern nothing paints with any more now leaves the resources; one some surviving +/// paint still uses stays. +#[test] +fn a_pattern_whose_only_painting_was_redacted_leaves_the_page() { + fn page_with(fills: &str) -> Document { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let content = format!("BT /F1 10 Tf 10 280 Td (PUBLIC) Tj ET {fills}"); + one_page(content.as_bytes(), "/Pattern << /P0 7 0 R >>", vec![cell]) + } + let has_pattern = |doc: &Document| { + pdfcraft_model::pages(doc)[0] + .dict + .get(b"Resources") + .and_then(|r| doc.resolve(r).as_dict().cloned()) + .is_some_and(|res| res.get(b"Pattern").is_some()) + }; + + // One box, marked: the pattern is no longer painted, so it goes with the fill. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "the pattern left the page"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + + // Two boxes share a text-bearing cell. The second still needs its pattern, but it is not + // safe to infer that it visibly displays every byte of the cell. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f /Pattern cs /P0 scn 20 40 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal keeps the pending mark and original bytes"); + + // Paint is not only path painting: text shows with the fill pattern too, and the text here + // survives the mark. Dropping the pattern would take the fill away from text still on the + // page, so a pattern is kept for every operator that can paint with it. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert!(has_pattern(&doc), "the refused operation keeps the original pattern"); + assert!(content(&doc, 0).contains("(KEEP)"), "the text is still there: {}", content(&doc, 0)); + + // Text in render mode 3 paints nothing, so it keeps no pattern: the unpainted pattern goes. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT 3 Tr /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "invisible text paints no pattern"); + + // `/Pattern` may be an indirect object, like any value. Reading the reference itself as a + // dictionary found none, so nothing was pruned and the cell stayed in the saved file. + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Pattern cs /P0 scn 20 140 100 30 re f", "/Pattern 8 0 R", vec![cell, b"<< /P0 7 0 R >>".to_vec()]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "an indirect /Pattern is resolved before it is pruned"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); +} + +/// Pruning takes the pattern out of `/Resources` but leaves the `/P0 scn` that selected it, so +/// the saved page names a pattern that is no longer there. ISO 32000-1 (8.6.6.2) says that +/// operand is a key of the `/Pattern` subdictionary, so this is what a reader is handed: the +/// question is whether it stops reading the stream there. Asked in the review of #3647. +#[test] +fn a_page_whose_pruned_pattern_is_still_selected_renders() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + // `0 g` after the fill puts the text back on a plain colour: with the pattern still selected + // the text would paint with it, and a pattern something still paints with is never pruned. + // The text is *after* the selector on purpose -- that is what a reader would lose if it gave + // up on the stale `/P0`. + let content_stream = b"/Pattern cs /P0 scn 20 140 100 30 re f 0 g BT /F1 24 Tf 10 40 Td (PUBLIC) Tj ET"; + let mut doc = one_page(content_stream, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "the pattern is pruned"); + + // The state this test is about: the selector outlived the resource it names. + let after = content(&doc, 0); + assert!(after.contains("/P0 scn"), "the selector is still in the content: {after}"); + assert!(!after.contains("20 140 100 30 re"), "the fill that used it is gone: {after}"); + + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + let mut r = pdfcraft_render::PageRenderer::new(std::sync::Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(out.error.is_none(), "the page still renders: {:?}, warnings {:?}", out.error, out.warnings); + assert_eq!((out.width, out.height), (300, 300), "the page box is unchanged"); + + // The text after the stale selector is still drawn: 24 pt at y 40, so the top-left origin + // puts it in the rows around 240..265. Without it the reader gave up at the bad operand. + let painted = |rows: std::ops::Range, cols: std::ops::Range| { + rows.flat_map(|y| cols.clone().map(move |x| (y, x))).any(|(y, x)| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] != [255, 255, 255] + }) + }; + assert!(painted(236..268, 8..120), "the text after the stale selector is still painted"); + // The mark itself is drawn over the area the pattern filled -- `apply` writes a black + // overlay there (`q 0 0 0 rg 15 135 115 40 re f Q`), so the cell cannot show through it + // even if a reader kept the stale selector. + let black = |y: u32, x: u32| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] == [0, 0, 0] + }; + assert!(black(145, 70), "the redaction overlay is painted over the pattern's box"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_when_pattern_is_used_elsewhere() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"0.9 g 0 0 300 300 re f 0 g BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let content = b"/Pattern cs /P0 scn 20 140 100 30 re f 20 40 100 30 re f"; + let mut doc = one_page(content, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must be atomic"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_on_an_unselected_page() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 8 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 140 100 30 re f"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + cell, + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 9 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 40 100 30 re f"), + ]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, Some(&[0])) { + Ok(_) => { + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the unselected page kept the hidden cell text"); + } + Err(RedactError::Pattern { reason: PatternProblem::Content, .. }) => { + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must keep both pages and the mark"); + } + Err(error) => panic!("unexpected refusal: {error}"), + } +} + +#[test] +fn redacted_form_must_not_retain_its_own_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Pattern << /P0 7 0 R >> >>", + b"/Pattern cs /P0 scn 20 140 100 30 re f", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "form-owned pattern refusal must be atomic"); +} + +#[test] +fn redacted_form_must_not_retain_inherited_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"20 140 100 30 re f"); + let mut doc = one_page(b"/Pattern cs /P0 scn /Fm Do", "/Pattern << /P0 7 0 R >> /XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "inherited pattern refusal must be atomic"); +} + #[test] fn partially_covered_vector_paths_do_not_leak_drawing_commands() { // A vector path with detailed drawing commands (e.g. outlined text or confidential geometry) diff --git a/crates/redact/tests/pattern_page_tree.rs b/crates/redact/tests/pattern_page_tree.rs new file mode 100644 index 0000000000..c2b68a4485 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree.rs @@ -0,0 +1,221 @@ +//! Pattern redaction through the page tree and through glyphs: synthetic files whose pattern +//! cells could stay reachable, or whose surviving paint could lose its pattern, after Apply. +//! Each test reads the saved bytes back (full and incremental, with and without object +//! streams) and searches every decoded stream. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +/// Public text outside the mark, and a pattern fill wholly under it. +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +/// The needle in any decoded stream, with and without object streams, and in an incremental save. +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// A successful apply must leave no trace of the cell; a refusal must leave the document as it was. +fn assert_confidential_or_refused(mut doc: Document) { + mark(&mut doc, 0); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The page inherits its resources, but its `/Parent` does not lead back to the node it inherits +/// from (missing here). Readers resolve inheritance down the `/Kids` tree, so the page draws with +/// the root's `/Pattern`; that root keeps its `/Resources` after the page's copy is pruned. +#[test] +fn a_page_whose_parent_link_is_missing_does_not_keep_the_pruned_cell_in_the_page_tree() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + assert_confidential_or_refused(doc); +} + +/// As above with a `/Parent` that names a different, resource-less `/Pages` node. +#[test] +fn a_page_whose_parent_link_disagrees_with_the_tree_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + ]); + assert_confidential_or_refused(doc); +} + +/// Two ancestors both hold `/Resources` naming the pattern; the page inherits from the nearer one, +/// which is retired once the page gets its own pruned copy. The root's dictionary, which no page +/// inherits, still names the cell. +#[test] +fn a_shadowed_ancestor_resource_dictionary_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + // The tree is consistent and no page inherits the root's dictionary any more, so the pruned + // pattern is unlinked from it and the redaction applies rather than refusing. + let mut doc = doc; + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "the root's shadowed /Resources still names the cell ({report:?})"); +} + +/// Control: the same shape with only the nearer ancestor holding resources applies cleanly. +#[test] +fn control_a_single_inheriting_ancestor_is_retired_with_the_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert_confidential_or_refused(doc); +} + +/// A hostile chain of tiling patterns, each cell selecting the next, read from a file: the guard +/// must stop with an error (or apply), never overflow the stack. +#[test] +fn a_hostile_chain_of_nested_patterns_is_bounded() { + const N: usize = 3000; + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + // The fill outside the mark keeps /P0 in use, so the guard must inspect the chain. + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f /P0 scn 300 200 100 40 re f\n"), + ]; + for i in 0..N { + let num = 6 + i; + let (res, body) = if i + 1 < N { + (format!("/Resources << /Pattern << /Q {} 0 R >> >>", num + 1), b"/Pattern cs /Q scn 0 0 10 10 re f\n".to_vec()) + } else { + ("/Resources << /Font << /F1 4 0 R >> >>".to_string(), TEXT_CELL.to_vec()) + }; + objs.push(stream_obj(&format!("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 {res}"), body)); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Limit | PatternProblem::Content, .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A Type 3 font without `/Resources` draws its glyph with the page's `/P0` (ISO 32000-1 9.6.5: +/// names are then looked up in the page's resources). The glyph is outside the mark; the page's +/// own fill with `/P0` is under it. Pruning `/P0` loses the glyph's paint. +#[test] +fn a_type3_glyph_without_resources_keeps_the_page_pattern_it_paints_with() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, b"0 0 1 rg 0 0 500 300 re f\n"), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>".to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]); + let mut doc = doc; + let blue_glyph = |doc: &Document| { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + // The glyph covers x 300..340, y 230..270 (rows 30..70 from the top). + let i = ((50 * out.width + 320) * 4) as usize; + out.rgba[i..i + 3].to_vec() + }; + let before = blue_glyph(&doc); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(blue_glyph(&doc), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} + +/// Text, inline images and XObjects record their fill pattern as they are met, before the hit +/// test, so a pattern whose only paint was text wholly under the mark is kept, and so is its cell. +/// That is the documented direction of error (keeping paint over losing it): a cell that carries +/// text or images is then refused by the guard, so what can stay is vector drawing nothing shows. +#[test] +fn a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell() { + const DRAWING: &[u8] = b"0.1 0.2 0.3 rg 111.11 22.22 m 133.33 44.44 l 155.55 22.22 l f\n"; + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nBT /F1 20 Tf /Pattern cs /P0 scn 40 120 Td (HIDDEN) Tj ET\n"), + stream_obj(PATTERN, DRAWING), + ]); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert!(report.glyphs >= 6, "{report:?}"); + assert_eq!(report.patterns_removed, 0, "text paint keeps its pattern"); + assert!(!saved_contains(&doc, b"(HIDDEN)"), "the glyphs are gone"); + assert!(saved_contains(&doc, b"133.33 44.44 l"), "the vector cell is kept, as documented"); +} diff --git a/crates/redact/tests/pattern_page_tree_shapes.rs b/crates/redact/tests/pattern_page_tree_shapes.rs new file mode 100644 index 0000000000..dafedd56b4 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree_shapes.rs @@ -0,0 +1,275 @@ +//! Pattern redaction over unusual page trees -- cyclic, deep, wide, inconsistent, packed in +//! object streams, or reached only through `/Parent` -- and over Type 3 fonts reached other than +//! by `Tf` in the scope that paints. Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Round-trip through a full save with object streams, so every dictionary lives in one. +fn packed(doc: &Document) -> Document { + Document::open(Arc::new(write_full(doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap() +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The root `/Pages` node carries a `/Parent` to a node outside the `/Kids` tree that names the +/// pattern. A full save follows that `/Parent`, so the node and the cell are written. +#[test] +fn a_node_reached_only_through_the_roots_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 7 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// As above, one level down: an intermediate node in `/Kids` whose `/Parent` leads elsewhere. +#[test] +fn a_node_reached_only_through_an_intermediate_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [8 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 8 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Pages /Parent 7 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// `/Parent` cycles: the root's parent is itself, and the page's chain loops through it. +#[test] +fn a_parent_cycle_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 2 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A page 5,000 levels below the root that holds the pattern (past `resource_source`'s 4,096). +#[test] +fn a_very_deep_tree_is_confidential_or_refused() { + const DEPTH: usize = 5000; + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Page /Parent {} 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>", 7 + DEPTH - 1).into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for i in 0..DEPTH { + let num = 7 + i; + let parent = if i == 0 { 2 } else { num - 1 }; + let kid = if i + 1 == DEPTH { 3 } else { num + 1 }; + objs.push(format!("<< /Type /Pages /Parent {parent} 0 R /Kids [{kid} 0 R] /Count 1 >>").into_bytes()); + } + confidential_or_refused(doc_or(objs), 0); +} +fn doc_or(objs: Vec>) -> Document { + open(objs) +} +/// The shadowed-root and missing-parent shapes again, with every dictionary in object streams. +#[test] +fn page_tree_shapes_in_object_streams_are_confidential_or_refused() { + let shadowed = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert!(confidential_or_refused(packed(&shadowed), 0), "the shadowed root applies"); + let missing = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(packed(&missing), 0); +} +/// The shadowed root's `/Resources` object is also a form's resources, drawn outside the mark on +/// the redacted page: unlinking from the root must not reach through to the form. +#[test] +fn a_root_resource_dictionary_shared_with_a_form_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources 9 0 R >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\nq 1 0 0 1 300 200 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm 8 0 R >> >> >>".to_vec(), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 50] /Resources 9 0 R", b"0 0 1 rg 0 0 50 20 re f\n"), + RES.as_bytes().to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// A document of 17,000 pages whose first page names a pattern: only that page has patterns. +/// Redacting it should apply as it does in a short document. +#[test] +fn a_long_document_with_a_pattern_on_one_page_still_applies() { + const PAGES: usize = 17_000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let result = apply(&mut doc, None); + assert!(result.is_ok(), "a long document refuses: {result:?}"); + assert!(!saved_contains(&doc, SECRET)); +} +/// Control for the above: the same with 1,000 pages. +#[test] +fn control_a_shorter_document_with_a_pattern_on_one_page_applies() { + const PAGES: usize = 1000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + assert!(apply(&mut doc, None).is_ok()); + assert!(!saved_contains(&doc, SECRET)); +} +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The Type 3 font without `/Resources` is selected through an ExtGState `/Font` entry, not `Tf`. +#[test] +fn a_type3_font_set_by_extgstate_keeps_the_page_pattern_it_paints_with() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /ExtGState << /G 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/G gs BT 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + b"<< /Type /ExtGState /Font [7 0 R 1] >>".to_vec(), + ]); + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font without `/Resources` is shown inside a form that has its own `/Resources` +/// (with no `/Pattern`); the page paints `/P0` under the mark. +#[test] +fn a_type3_glyph_in_a_form_with_resources_renders_the_same_after_the_redaction() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph renders differently ({report:?}, before {before:?})"); +} diff --git a/crates/redact/tests/pattern_reachability.rs b/crates/redact/tests/pattern_reachability.rs new file mode 100644 index 0000000000..9ca4be0c27 --- /dev/null +++ b/crates/redact/tests/pattern_reachability.rs @@ -0,0 +1,304 @@ +//! Pattern redaction against references no walk follows -- a direct `/Parent`, `/Kids` on a +//! page, a stray resource entry -- Type 3 fonts reached in other ways, and hostile wide trees. +//! Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} + +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The root's `/Parent` is a direct dictionary, not a reference: a full save writes it inline +/// with the root, and the pattern it names with it. +#[test] +fn a_direct_parent_dictionary_on_the_root_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent << /Type /Pages /Kids [] /Count 0 /Resources {RES} >> /Resources {RES} >>") + .into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A leaf (`/Type /Page`) that also carries `/Kids`: readers stop at the leaf, a full save does +/// not, and the hidden node names the pattern. +#[test] +fn a_node_hidden_under_a_leaf_s_kids_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} /Kids [7 0 R] >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 3 0 R /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// A stray reference to the cell from a resource category the guard does not inspect +/// (`/Shading`): the class #3999 tracks, shown here for patterns. +#[test] +fn a_stray_resource_reference_to_the_cell_does_not_keep_it() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +fn type3_page(page_res_extra: &str, content: &[u8], extra: Vec>) -> Document { + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> {page_res_extra} >> >>").into_bytes(), + FONT.to_vec(), + stream_obj("", content), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]; + objs.extend(extra); + open(objs) +} +fn assert_glyph_kept(mut doc: Document) { + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font is shown in a form without `/Resources`, which draws through the page's. +#[test] +fn a_type3_glyph_in_a_resourceless_form_keeps_the_page_pattern() { + assert_glyph_kept(type3_page( + "/XObject << /Fm 9 0 R >>", + b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n")], + )); +} +/// The Type 3 font is shown inside a pattern cell the page paints outside the mark: the cell +/// carries text, so the guard refuses rather than dropping anything. +#[test] +fn a_type3_glyph_inside_a_pattern_cell_is_refused() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R /P1 9 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Pattern cs /P1 scn 300 200 100 40 re f\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A hostile wide tree: `K` `/Pages` nodes that all share one `/Kids` array of `M` references +/// back to themselves, beside the real page. Time and memory are measured from outside +/// (`PDF_WIDE_K`, `PDF_WIDE_M`). Ignored by default. +#[test] +#[ignore] +fn hostile_wide_tree_probe() { + let k: usize = std::env::var("PDF_WIDE_K").ok().and_then(|v| v.parse().ok()).unwrap_or(200); + let m: usize = std::env::var("PDF_WIDE_M").ok().and_then(|v| v.parse().ok()).unwrap_or(500_000); + let mut array = String::with_capacity(m * 9); + for i in 0..m { + array.push_str(&format!("{} 0 R ", 10 + i % k)); + } + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R 10 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("[{array}]").into_bytes(), + b"null".to_vec(), + b"null".to_vec(), + ]; + for _ in 0..k { + objs.push(b"<< /Type /Pages /Kids 7 0 R /Count 0 >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let start = std::time::Instant::now(); + let result = apply(&mut doc, None); + eprintln!("WIDE k={k} m={m} apply={:?} in {:?}", result.as_ref().map(|r| r.patterns_removed), start.elapsed()); +} +/// The Type 3 font has `/Resources null`, which readers treat as absent. +#[test] +fn a_type3_font_with_null_resources_keeps_the_page_pattern() { + let mut doc = type3_page("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n", vec![]); + doc.update_dict(pdfcraft_cos::ObjRef::new(7, 0), |d| d.set(b"Resources".to_vec(), Object::Null)).unwrap(); + assert_glyph_kept(doc); +} +/// Known limit (fidelity only, never a leak: a missed use can only prune more). Kept annotation +/// appearances are not interpreted for pattern use, so a Type 3 font without resources shown in +/// one does not keep the page's patterns. ISO 32000 resolves the glyph's names in the +/// appearance's own resources, where `/P0` does not exist, so only a lenient renderer draws it. +/// The Type 3 font is shown by a kept annotation's appearance stream (outside the mark). +#[test] +#[ignore = "known limit: kept annotation appearances are not interpreted, so a resourceless Type 3 glyph in one does not keep the page pattern; ISO 32000 resolves it in the appearance's own resources, which pdfcraft's renderer does not"] +fn a_type3_glyph_in_an_annotation_appearance_renders_the_same() { + let mut doc = type3_page( + "", + b"/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![ + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 40 40] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 0 0 Td (a) Tj ET\n"), + b"<< /Type /Annot /Subtype /Stamp /Rect [300 230 340 270] /F 4 /AP << /N 9 0 R >> >>".to_vec(), + ], + ); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |d| { + d.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(pdfcraft_cos::ObjRef::new(10, 0))])) + }) + .unwrap(); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the annotation renders differently ({report:?}, before {before:?})"); +} +/// A form wholly under the mark is removed; its own `/Pattern` cell carries text. Nothing prunes +/// that pattern, so the backstop never looks, and a stray reference (`/Shading`) keeps the cell. +#[test] +fn a_removed_form_s_pattern_does_not_survive_through_a_stray_reference() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nq 1 0 0 1 35 110 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 170 35] /Resources << /Pattern << /P0 6 0 R >> >>", b"/Pattern cs /P0 scn 0 0 170 35 re f\n"), + ]); + let mut doc = doc; + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the removed form's cell is in the saved file ({report:?})"), + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + } + } +} +/// `/P0` names an indirect object whose value is a reference to the cell; the cell is also named +/// by a stray reference. The backstop checks the number it pruned, not what it points at. +#[test] +fn a_pruned_pattern_named_through_a_reference_object_does_not_dodge_the_backstop() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"6 0 R".to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// The backstop's reachability is a full save's: after Apply an incremental save must be that +/// full save, byte for byte, never an appended revision over the old bytes. +#[test] +fn an_incremental_save_after_apply_is_the_full_save() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + let mut doc = doc; + mark(&mut doc, 0); + apply(&mut doc, None).unwrap(); + for object_streams in [true, false] { + // A redacted rewrite gets a new identifier from fresh entropy; pin it so the two saves + // can be compared byte for byte. + let opts = SaveOptions { object_streams, id_entropy: Some([7; 16]), ..SaveOptions::default() }; + assert_eq!(write_incremental(&doc, &opts).unwrap(), write_full(&doc, &opts).unwrap()); + } + // A later edit after a save-and-reopen starts from the clean file. + let reopened = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!reopened.bytes().windows(SECRET.len()).any(|w| w == SECRET)); +} diff --git a/crates/redact/tests/pattern_redaction.rs b/crates/redact/tests/pattern_redaction.rs new file mode 100644 index 0000000000..4fd898c9ff --- /dev/null +++ b/crates/redact/tests/pattern_redaction.rs @@ -0,0 +1,758 @@ +//! Saved-output confidentiality regressions for #3647, using only synthetic PDFs. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"CELLSECRET"; +const CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; + +fn fixture(cell: &[u8], compressed: bool, partial: bool, inherited: bool) -> Document { + let resources = "/Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; + let parent_res = if inherited { resources } else { "" }; + let own_res = if inherited { "" } else { resources }; + let width = if partial { 400 } else { 170 }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 {width} 35 re f\n"); + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 {parent_res} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R {own_res} >>").into_bytes(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", content.as_bytes()), + stream_obj(PATTERN, cell), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + if compressed { + let pattern = doc.get(ObjRef::new(6, 0)); + let Object::Stream(pattern) = &*pattern else { panic!("fixture pattern must be a stream") }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(pattern.dict.clone(), cell))); + } + doc +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn saved(doc: &Document) -> Vec { + write_full(doc, &SaveOptions::default()).unwrap() +} + +fn has_decoded_secret(bytes: Vec) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap().windows(SECRET.len()).any(|w| w == SECRET)) + }) +} + +/// Success must remove the payload from the saved object graph. Refusal must be atomic even +/// for the direct library caller, not just the engine's working clone. +fn assert_confidential_or_refused(mut doc: Document, expect_refusal: bool) { + mark(&mut doc, 0); + let before = saved(&doc); + assert!(has_decoded_secret(before.clone()), "the fixture must carry the secret"); + match apply(&mut doc, None) { + Ok(report) => { + assert!(!expect_refusal, "a retained text-bearing cell must refuse"); + assert_eq!(report.patterns_removed, 1); + assert!(!has_decoded_secret(saved(&doc)), "successful redaction retained the pattern cell's secret in the saved PDF"); + } + Err(error) => { + assert!(expect_refusal, "a fully removed pattern should apply: {error}"); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Content, .. }), "{error}"); + assert_eq!(saved(&doc), before, "a refused pattern must leave the entire document unchanged"); + assert_eq!(marks(&doc).len(), 1, "the pending mark must survive refusal"); + } + } +} + +#[test] +fn pattern_whole_fill_saved_output_is_confidential_or_refused() { + for compressed in [false, true] { + assert_confidential_or_refused(fixture(CELL, compressed, false, false), false); + } +} + +#[test] +fn pattern_partial_fill_saved_output_is_confidential_or_refused() { + // The mark hits part of the patterned path, which is removed whole (F-008), so nothing paints + // with the pattern any more: it is pruned with its cell, and the redaction is confidential + // rather than refused. Once a hit path was clipped instead, its paint kept the cell in use. + let cell = [CELL, b"BT /F1 20 Tf 300 120 Td (SAFE) Tj ET\n"].concat(); + for compressed in [false, true] { + assert_confidential_or_refused(fixture(&cell, compressed, true, false), false); + } +} + +#[test] +fn pattern_inherited_resources_saved_output_is_confidential_or_refused() { + assert_confidential_or_refused(fixture(CELL, true, false, true), false); +} + +#[test] +fn pattern_vector_only_control_still_applies_and_preserves_the_cell() { + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let mut doc = fixture(cell, true, true, false); + // The guard's control: a plain vector cell some surviving paint still uses is inspected and + // kept, not refused. The hit path is removed whole (F-008), so keep another paint in use. + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!((report.paths_removed, report.paths_clipped, report.patterns_removed), (1, 0, 0)); + assert!(marks(&doc).is_empty()); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + let pattern = reopened.resolve(patterns.get(b"P0").unwrap()); + let Object::Stream(pattern) = &*pattern else { panic!("pattern must be preserved") }; + assert_eq!(pattern.decoded().unwrap(), cell); +} + +#[test] +fn unrelated_font_widths_do_not_exhaust_pattern_budget() { + let mut doc = fixture(b"40 120 15 15 re f\n", false, true, false); + doc.update_dict(ObjRef::new(4, 0), |font| { + font.set(b"W".to_vec(), Object::Array(vec![Object::Int(500); 17_000])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn shared_annotation_appearance_on_unselected_page_is_refused() { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + for page in pages { + let annotation = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + } + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_on_second_page_does_not_change_the_first_page() { + let mut doc = two_pages(); + // The second page's pattern must stay in use for it to refuse: its hit path alone would be + // removed whole (F-008) and the pattern pruned. + keep_pattern_in_use(&mut doc, 1); + mark(&mut doc, 0); + mark(&mut doc, 1); + let before = saved(&doc); + let result = apply(&mut doc, None); + assert!(result.is_err(), "text-carrying pattern must refuse the whole apply"); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 2); +} + +fn two_pages() -> Document { + let mut doc = fixture(CELL, false, true, false); + let source_page = pdfcraft_model::pages(&doc)[0].dict.clone(); + let mut clean = source_page.clone(); + let mut clean_resources = doc.dict(clean.get(b"Resources").unwrap()).unwrap(); + clean_resources.remove(b"Pattern"); + clean.set(b"Resources".to_vec(), Object::Dict(clean_resources)); + let content = doc.add(Object::Stream(Stream::from_raw(Default::default(), b"BT /F1 20 Tf 40 120 Td (DIRECTSECRET) Tj ET".to_vec()))); + clean.set(b"Contents".to_vec(), Object::Ref(content)); + let first = doc.add(Object::Dict(clean)); + doc.update_dict(ObjRef::new(2, 0), |pages| { + pages.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(3, 0))])); + pages.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + doc +} + +fn pattern_stream(doc: &Document) -> Stream { + let object = doc.get(ObjRef::new(6, 0)); + let Object::Stream(stream) = &*object else { panic!("fixture pattern must be a stream") }; + stream.clone() +} + +/// Paint with the pattern once more, well away from the mark. A path the mark hits is removed +/// whole (F-008), and a pattern nothing surviving paints with is pruned along with its cell, so +/// the guard only ever inspects a pattern that some surviving paint still uses. This keeps one, +/// so the guard has to inspect this cell rather than the pattern simply leaving the file. +fn keep_pattern_in_use(doc: &mut Document, page: usize) { + let page = pdfcraft_model::pages(doc).swap_remove(page); + let extra = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"/Pattern cs /P0 scn 300 200 100 40 re f".to_vec()))); + let mut list = match page.dict.get(b"Contents") { + Some(Object::Array(items)) => items.clone(), + Some(other) => vec![other.clone()], + None => Vec::new(), + }; + list.push(Object::Ref(extra)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(list))).unwrap(); +} + +fn assert_refused(mut doc: Document, reason: PatternProblem) { + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, None).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 1, reason }); + assert!(!error.to_string().contains("CELLSECRET"), "errors must not disclose page text"); + assert_eq!(saved(&doc), before, "even direct callers must be unchanged"); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_does_not_block_redacting_other_selected_pages() { + let mut doc = two_pages(); + mark(&mut doc, 0); + mark(&mut doc, 1); + let report = apply(&mut doc, Some(&[0])).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!(marks(&doc).len(), 1); + assert_eq!(marks(&doc)[0].page, 1); + assert!(has_decoded_secret(saved(&doc)), "the unselected page must remain intact"); +} + +#[test] +fn pattern_nested_form_and_nested_pattern_text_are_refused() { + for nested_pattern in [false, true] { + let mut doc = fixture(CELL, false, true, false); + let mut inner = pattern_stream(&doc); + if !nested_pattern { + inner.dict.remove(b"PatternType"); + inner.dict.set(b"Type".to_vec(), Object::name("XObject")); + inner.dict.set(b"Subtype".to_vec(), Object::name("Form")); + } + let reference = doc.add(Object::Stream(inner)); + let resources = Dict::from_iter([( + if nested_pattern { b"Pattern".to_vec() } else { b"XObject".to_vec() }, + Object::Dict(Dict::from_iter([(b"Inner".to_vec(), Object::Ref(reference))])), + )]); + let mut outer = pattern_stream(&doc); + outer.dict.set(b"Resources".to_vec(), Object::Dict(resources)); + let content = if nested_pattern { b"/Pattern cs /Inner scn 35 110 400 35 re f".as_slice() } else { b"/Inner Do" }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(outer.dict, content))); + assert_refused(doc, PatternProblem::Content); + } +} + +#[test] +fn pattern_inside_form_resources_is_refused() { + let mut doc = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&doc)[0].clone(); + let resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(resources)), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let resources = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Fm".to_vec(), Object::Ref(form))])))]); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"/Fm Do".to_vec()))); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_images_and_uncolored_text_are_refused() { + let inline = b"q 10 0 0 10 40 120 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q"; + assert_refused(fixture(inline, true, true, false), PatternProblem::Content); + let mut uncolored = fixture(CELL, false, true, false); + uncolored.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"PaintType".to_vec(), Object::Int(2))).unwrap(); + assert_refused(uncolored, PatternProblem::Content); +} + +#[test] +fn pattern_indirect_filter_and_parameters_are_decoded_not_skipped() { + let mut doc = fixture(CELL, true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let predictor = doc.add(Object::Int(1)); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Ref(predictor))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Ref(filter)); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_damaged_filters_and_unparsed_content_fail_closed() { + for bad in [b"BT (unterminated".as_slice(), b"] BT /F1 20 Tf (CELLSECRET) Tj ET", b"BX UnknownPaint EX"] { + assert_refused(fixture(bad, false, true, false), PatternProblem::Unreadable); + } + let mut damaged = fixture(CELL, true, true, false); + let stream = pattern_stream(&damaged); + let cut = stream.raw.get(..stream.raw.len() / 2).unwrap().to_vec(); + damaged.set(ObjRef::new(6, 0), Object::Stream(Stream::from_raw(stream.dict, cut))); + assert_refused(damaged, PatternProblem::Unreadable); + let mut parameters = fixture(CELL, true, true, false); + parameters + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"DecodeParms".to_vec(), Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(99))]))); + }) + .unwrap(); + assert_refused(parameters, PatternProblem::Unreadable); +} + +#[test] +fn pattern_oversized_decoding_and_resource_cycles_fail_closed() { + let large = vec![b' '; (1 << 20) + 1]; + assert_refused(fixture(&large, true, true, false), PatternProblem::Limit); + let mut cycle = fixture(b"0 0 20 20 re f", false, true, false); + cycle + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"Self".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ); + }) + .unwrap(); + assert_refused(cycle, PatternProblem::Unreadable); +} + +#[test] +fn pattern_plain_nested_form_control_still_applies() { + let mut doc = fixture(b"/Plain Do", false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"40 120 15 15 re f".to_vec(), + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Plain".to_vec(), Object::Ref(form))])))])), + ); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + assert!(marks(&doc).is_empty()); +} + +#[test] +fn pattern_unused_and_annotation_appearance_resources_are_checked() { + let mut unused = fixture(CELL, false, false, false); + unused.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + assert_refused(unused, PatternProblem::Content); + + let mut appearance = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&appearance)[0].clone(); + let mut resources = appearance.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = resources.remove(b"Pattern").unwrap(); + appearance.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + appearance.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + let ap = appearance.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), patterns)]))), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let annotation = appearance.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 250.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + appearance.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + let mut implicit = appearance.clone(); + implicit + .update_dict(ap, |dict| { + dict.remove(b"Subtype"); + }) + .unwrap(); + assert_refused(implicit, PatternProblem::Content); + assert_refused(appearance, PatternProblem::Content); +} + +#[test] +fn pattern_image_xobjects_and_missing_resources_are_refused() { + let mut doc = fixture(b"/Picture Do", false, true, false); + let image = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Image")), + (b"Width".to_vec(), Object::Int(1)), + (b"Height".to_vec(), Object::Int(1)), + (b"ColorSpace".to_vec(), Object::name("DeviceGray")), + (b"BitsPerComponent".to_vec(), Object::Int(8)), + ]), + vec![0], + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Picture".to_vec(), Object::Ref(image))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); + let mut missing = fixture(b"0 0 20 20 re f", false, true, false); + missing + .update_dict(ObjRef::new(6, 0), |dict| { + dict.remove(b"Resources"); + }) + .unwrap(); + assert_refused(missing, PatternProblem::Unreadable); +} + +#[test] +fn pattern_indirect_vector_filter_control_still_applies() { + let mut doc = fixture(b"40 120 15 15 re f", true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(1))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::Ref(filter)])); + dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Ref(parms)])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn pattern_depth_and_total_decoded_budgets_fail_closed() { + let mut deep = fixture(b"0 0 20 20 re f", false, true, false); + // Two frames a level (the array, then the reference): 300 levels run past the 512-frame + // depth budget long before the object budget, so it is the depth budget that refuses, and it + // does so without overflowing the stack. + let mut nested = Object::Int(1); + for _ in 0..300 { + nested = Object::Array(vec![Object::Ref(deep.add(nested))]); + } + deep.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Extra".to_vec(), nested)).unwrap(); + assert_refused(deep, PatternProblem::Limit); + + let mut total = fixture(b"0 0 20 20 re f", false, true, false); + let blank = vec![b' '; 1 << 20]; + let template = pattern_stream(&total).dict; + let mut patterns = Dict::new(); + for index in 0..9 { + let cell = total.add(Object::Stream(Stream::flate(template.clone(), &blank))); + patterns.set(format!("P{index}").into_bytes(), Object::Ref(cell)); + } + total + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Dict(patterns))]))); + }) + .unwrap(); + assert_refused(total, PatternProblem::Limit); +} + +#[test] +fn pattern_replacement_text_without_text_showing_is_refused() { + assert_refused(fixture(b"/Span << /ActualText (CELLSECRET) >> BDC 40 120 20 20 re f EMC", false, true, false), PatternProblem::Content); + let mut named = fixture(b"/Span /Description BDC 40 120 20 20 re f EMC", true, true, false); + named + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Properties".to_vec(), + Object::Dict(Dict::from_iter([( + b"Description".to_vec(), + Object::Dict(Dict::from_iter([(b"Alt".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"CELLSECRET".to_vec())))])), + )])), + )])), + ); + }) + .unwrap(); + assert_refused(named, PatternProblem::Content); +} + +#[test] +fn pattern_resource_names_are_not_dictionary_categories_or_descriptions() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for has_pattern in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let font = Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(ObjRef::new(4, 0)))])); + if has_pattern { + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + } else { + doc.update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"35 110 400 35 re f".to_vec()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "resource named {name}, pattern: {has_pattern}"); + } + } +} + +#[test] +fn pattern_opaque_and_external_xobjects_are_refused() { + for kind in ["PS", "Bogus", "missing", "Form-PS", "Form-Ref"] { + let mut doc = fixture(b"/X Do", false, true, false); + let mut dict = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject"))]); + match kind { + "missing" => {} + "Form-PS" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set(b"Subtype2".to_vec(), Object::name("PS")); + } + "Form-Ref" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set( + b"Ref".to_vec(), + Object::Dict(Dict::from_iter([(b"F".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"external.pdf".to_vec())))])), + ); + } + subtype => { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + } + // Reference XObjects can have an empty fallback: that cannot establish safety of + // the external representation. No external file is read by this test or the guard. + let data = if kind == "Form-Ref" { b"".as_slice() } else { b"/Helvetica findfont 20 scalefont setfont (CELLSECRET) show" }; + let object = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"X".to_vec(), Object::Ref(object))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Unreadable); + } +} + +#[test] +fn pattern_named_form_resources_still_apply() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for inside in [false, true] { + let content = format!("/{name} Do"); + let mut doc = fixture(content.as_bytes(), false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"35 110 400 35 re f".to_vec(), + ))); + let resources = Object::Dict(Dict::from_iter([( + b"XObject".to_vec(), + Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(form))])), + )])); + doc.update_dict(ObjRef::new(if inside { 6 } else { 3 }, 0), |dict| dict.set(b"Resources".to_vec(), resources)).unwrap(); + if !inside { + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "form named {name}, inside: {inside}"); + } + } +} + +#[test] +fn pattern_soft_mask_groups_are_checked_by_resource_context() { + for subtype in [None, Some("Bogus"), Some("Form")] { + for text in [false, true] { + let mut doc = fixture(b"q /GS gs 40 120 20 20 re f Q", false, true, false); + let mut dict = Dict::from_iter([ + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Group".to_vec(), Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Transparency"))]))), + (b"Resources".to_vec(), Object::Dict(Dict::new())), + ]); + if let Some(subtype) = subtype { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + let group = doc.add(Object::Stream(Stream::from_raw(dict, if text { CELL.to_vec() } else { b"40 120 20 20 re f".to_vec() }))); + let state = Object::Dict(Dict::from_iter([( + b"SMask".to_vec(), + Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Luminosity")), (b"G".to_vec(), Object::Ref(group))])), + )])); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"ExtGState".to_vec(), Object::Dict(Dict::from_iter([(b"GS".to_vec(), state)])))])), + ); + }) + .unwrap(); + match (subtype, text) { + (Some("Form"), false) => { + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + (Some("Form"), true) => assert_refused(doc, PatternProblem::Content), + _ => assert_refused(doc, PatternProblem::Unreadable), + } + } + } +} + +#[test] +fn pattern_null_resource_categories_and_entries_are_absent() { + for null_entries in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let mut resources = Dict::new(); + for category in ["Pattern", "XObject", "Properties", "ExtGState"] { + let value = if null_entries { Object::Dict(Dict::from_iter([(b"Unused".to_vec(), Object::Null)])) } else { Object::Null }; + resources.set(category.as_bytes().to_vec(), value); + } + doc.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + let mut plain = fixture(b"", false, false, false); + plain + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Null)]))); + }) + .unwrap(); + plain.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"40 120 20 20 re f".to_vec()))); + mark(&mut plain, 0); + assert_eq!(apply(&mut plain, None).unwrap().marks, 1); +} + +#[test] +fn a_pattern_whose_only_paint_was_redacted_leaves_the_file_with_its_cell() { + // The mark hits the patterned path, which is removed whole (F-008). Nothing paints with the + // pattern any more, so it leaves the page's resources and its cell -- text included -- leaves + // the saved file: the redaction applies rather than refusing. + for compressed in [false, true] { + let mut doc = fixture(CELL, compressed, true, false); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "compressed: {compressed}"); + assert!(!has_decoded_secret(saved(&doc)), "the cell's text is not in the saved file (compressed: {compressed})"); + } +} + +#[test] +fn a_form_without_resources_keeps_the_page_pattern_it_still_paints() { + // The page fills a box under the mark with /P0, then draws a form that has no /Resources of + // its own, so it paints with the page's: its /P0 fill lies outside the mark and survives. + // The page's own fill is removed whole, but the pattern must stay for the form's. + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm0 7 0 R >> >> >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f\n0 g\n/Fm0 Do\n"), + stream_obj(PATTERN, cell), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"/Pattern cs /P0 scn 300 200 100 40 re f\n"), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!((report.paths_removed, report.patterns_removed), (1, 0)); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + assert!(patterns.get(b"P0").is_some(), "the form still paints with /P0 outside the mark"); +} + +#[test] +fn a_shared_appearance_of_a_reply_or_pop_up_removed_on_another_page_is_refused() { + // The marked comment on page 1 has no appearance; its reply (or pop-up) is listed on page 2, + // goes with it, and shares its appearance with a stamp that stays on page 2. That stamp keeps + // the text-bearing cell reachable, so the all-pages check must run and refuse. + for link in ["IRT", "Parent"] { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let comment = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Text")), + (b"Rect".to_vec(), Object::Array(vec![40.into(), 110.into(), 60.into(), 130.into()])), + ]))); + let follower = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name(if link == "IRT" { "Text" } else { "Popup" })), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 260.into()])), + (link.as_bytes().to_vec(), Object::Ref(comment)), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + let stamp = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(pages[0].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(comment)]))).unwrap(); + doc.update_dict(pages[1].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(follower), Object::Ref(stamp)]))).unwrap(); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }, "/{link}"); + assert_eq!(saved(&doc), before, "/{link}"); + assert_eq!(marks(&doc).len(), 1, "/{link}"); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 023688ddc4..736d53bb7a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -245,7 +245,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. Renaming a field renames it in the calculations that read it (field lists, notation, literal `getField` names), in one undo step, and says how many changed. Prepare a form ▸ Calculation order, a page of its own: each calculated field's formula and value, what is wrong (fields that calculate too early, loops, fields missing from `/CO`) and Fix order, Up/Down, Alt+↑/↓ and drag, and on the page each field's step and arrows to and from the field in focus (`form_calculation_order` headlessly). | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). Retained text/image-bearing or unreadable tiling-pattern cells now make Apply refuse atomically after a bounded check of rewritten resources; fully removed cells still apply, and cross-page retention is checked. The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178); black-and-white images stored losslessly as CCITT Group 4 when smaller, averaged down to 300 ppi above 450 (halftones aside), Group 3 stored again as Group 4, and black-and-white RGB and indexed images that use only palette entries 0 and 1 packed to one bit. Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | From af76b5ca89ed0902c30f4da97677cea38b58709d Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:10:16 +0200 Subject: [PATCH 70/85] Delete key removes selected content object in edit mode (#6384) * Delete key removes selected content object in edit mode The reporter says Delete does nothing for a selected textbox or image in edit mode; the only Delete/Backspace handlers in the UI are for organize pages and comments, so a selected content object has no delete path. Add Delete/Backspace handling to the object selection input in object_ui.rs that deletes the currently selected object(s), and cover it with a unit test that selects an object and presses Delete. Test: delete_key_removes_selected_object failed before the change and passes after. * Tests: the Delete key test takes PageToolContext and imports PagePointer Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/ui-egui/src/tools/object_ui.rs | 72 +++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/crates/ui-egui/src/tools/object_ui.rs b/crates/ui-egui/src/tools/object_ui.rs index 67a077bc7b..a35bf795e1 100644 --- a/crates/ui-egui/src/tools/object_ui.rs +++ b/crates/ui-egui/src/tools/object_ui.rs @@ -218,6 +218,22 @@ pub(crate) fn page_input(cx: PageToolContext<'_>, view: &mut DocView) -> bool { } return false; } + // Delete or Backspace removes the selected objects on this page as one edit, + // like Edit::DeletePages removes pages. The key is consumed even when the pointer + // is off the page, because canvas keys belong to the selection while it exists. + if view.objects.page == Some(page) + && !view.objects.selected.is_empty() + && view.objects.gesture.is_none() + && view.pending_edit.is_none() + && !ui.ctx().egui_wants_keyboard_input() + { + let deleted = + ui.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Delete) || i.consume_key(egui::Modifiers::NONE, egui::Key::Backspace)); + if deleted { + view.pending_edit = delete_edit(page, view.objects.selected.iter().map(|o| o.target).collect()); + return true; + } + } let mut selection = std::mem::take(&mut view.objects); let result = input(&mut selection, cx, view); view.objects = selection; @@ -527,4 +543,60 @@ mod tests { assert!(matches!(delete_edit(2, vec![t(ObjectKind::Image, 4)]), Some(Edit::EditPageImage { page: 2, index: 4, .. }))); assert!(delete_edit(0, Vec::new()).is_none()); } + + /// Selecting content objects on a page and pressing Delete must queue one edit that removes + /// them: `page_input` selects and moves content objects but never consumes Delete or + /// Backspace, so pressing Delete currently does nothing (issue #3780). + #[test] + fn delete_key_removes_selected_object() { + use crate::canvas::page_pointer::PagePointer; + use crate::{PdfCraftApp, canvas::ViewDefaults}; + use pdfcraft_engine::{DocId, ImageEdit}; + use pdfcraft_render::PageInfo; + + let mut app = PdfCraftApp::new(); + let bytes = app.session.create_blank(300.0, 400.0, 1).unwrap(); + app.open_bytes("blank.pdf", None, bytes.as_ref().clone()).unwrap(); + let doc = app.session.get(app.views[0].id).unwrap(); + + let info = pdfcraft_render::DocInfo { + pages: vec![PageInfo { width: 300.0, height: 400.0, label: String::new(), crop: [0.0, 0.0, 300.0, 400.0], rotation: 0 }], + ..Default::default() + }; + let mut view = DocView::new(DocId(1), &info, ViewDefaults::default()); + view.objects.page = Some(0); + view.objects.selected.push(EditableObject { + target: ObjectTarget { kind: ObjectKind::Image, index: 0 }, + rect: [50.0, 60.0, 100.0, 110.0], + content_type: "image", + text: None, + }); + + let ctx = egui::Context::default(); + let xf = PageXform { rect: Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0)), rot: 0, pw: 800.0, ph: 600.0 }; + let ptr = PagePointer { pos: None, ..Default::default() }; + let mut output = ctx.run_ui( + egui::RawInput { + screen_rect: Some(Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0))), + events: vec![egui::Event::Key { + key: egui::Key::Delete, + physical_key: None, + pressed: true, + repeat: false, + modifiers: egui::Modifiers::NONE, + }], + ..Default::default() + }, + |ui| { + let resp = ui.interact(ui.available_rect_before_wrap(), egui::Id::unique("page"), egui::Sense::click()); + page_input(PageToolContext { ui, resp: &resp, xf: &xf, ptr: &ptr, page: 0, doc, info: &info }, &mut view); + }, + ); + output.textures_delta.clear(); + + assert!( + matches!(&view.pending_edit, Some(Edit::EditPageImage { page: 0, index: 0, change: ImageEdit::Delete })), + "pressing Delete with one object selected must queue that object's deletion as one edit" + ); + } } From a0811650dec177caf06a1b143a6cc7590c85dff6 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:29:00 +0200 Subject: [PATCH 71/85] Add Pages range field to Compare Files dialog (#6336) * Add Pages range field to Compare Files dialog Users cannot limit a comparison to selected pages, so the dialog needs a validated Pages field that blocks Compare on invalid ranges. Add a Pages text field under New file, keep its text in pages_text, parse comma-separated numbers and ranges to 0-based pages, show an error and disable Compare for zero, reversed, or out-of-range pages, and leave the comparison itself unchanged. Test: compare_dialog_pages_field_rejects_out_of_range failed before the change and passes after. * Compare Files: the Pages field limits the comparison, and says what is wrong with a bad range Co-Authored-By: Claude Opus 5.5 * Compare: a change across a page break counts on both pages; a bad range is refused, not read as all pages Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/compare/src/lib.rs | 14 ++++++++ crates/engine/src/compare.rs | 23 ++++++++++++-- crates/engine/src/tests.rs | 4 +++ crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/lib.rs | 2 ++ crates/ui-egui/src/workflows/compare_ui.rs | 35 ++++++++++++++++---- crates/ui-egui/tests/it/compare.rs | 37 ++++++++++++++++++++++ 7 files changed, 107 insertions(+), 9 deletions(-) diff --git a/crates/compare/src/lib.rs b/crates/compare/src/lib.rs index 91245cbc21..26a50eaf71 100644 --- a/crates/compare/src/lib.rs +++ b/crates/compare/src/lib.rs @@ -41,6 +41,8 @@ pub struct Side { /// The page the change is on (for a pure insertion or deletion, where the other side's /// words would have been). pub page: usize, + /// The page its last word is on: words changed across a page break are one change. + pub last_page: usize, pub rects: Vec<[f64; 4]>, /// For an empty side (the other side's words were inserted or deleted): the box of the /// word next to where they would be. @@ -245,6 +247,7 @@ fn side(words: &[&Word], neighbour: Option<&Word>) -> Side { Side { text: words.iter().map(|w| w.text.as_str()).collect::>().join(" "), page: words.first().or(neighbour.as_ref()).map_or(0, |w| w.page), + last_page: words.last().or(neighbour.as_ref()).map_or(0, |w| w.page), rects: rects(words), near: if words.is_empty() { neighbour.map(|w| w.rect) } else { None }, } @@ -349,6 +352,17 @@ mod tests { assert_eq!(c.count(Kind::Inserted), 2); } + #[test] + fn a_change_across_a_page_break_knows_both_pages() { + let mut old = words("alpha beta", 0); + old.extend(words("gamma delta", 1)); + let mut new = words("alpha one", 0); + new.extend(words("two delta", 1)); + let c = compare(&old, &new); + assert_eq!(c.changes.len(), 1); + assert_eq!((c.changes[0].new.page, c.changes[0].new.last_page), (0, 1)); + } + #[test] fn deletions_across_pages_keep_their_pages() { let mut old = words("alpha beta gamma", 0); diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..ec5cf3faf3 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -63,6 +63,15 @@ impl Session { Ok(pdfcraft_compare::compare(&a.words(), &b.words())) } + /// [`Session::compare`], keeping the changes on `pages` of `new` (0-based; empty: every page). + pub fn compare_pages(&self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let mut c = self.compare(old, new)?; + if !pages.is_empty() { + c.changes.retain(|ch| pages.iter().any(|page| (ch.new.page..=ch.new.last_page).contains(page))); + } + Ok(c) + } + /// Visual compare: regions where page n of `new` looks different from page n of `old` /// (rendered at `dpi`), as (page, user-space box in `new`). pub fn compare_visual(&self, old: DocId, new: DocId, dpi: f32) -> Result, EditError> { @@ -95,7 +104,12 @@ impl Session { /// The compare report as a new PDF (not opened). pub fn compare_report(&self, old: DocId, new: DocId) -> Result>, EditError> { - let c = self.compare(old, new)?; + self.compare_report_of(old, new, &[]) + } + + /// [`Session::compare_report`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn compare_report_of(&self, old: DocId, new: DocId, pages: &[usize]) -> Result>, EditError> { + let c = self.compare_pages(old, new, pages)?; let name = |id| self.get(id).map(|d| d.name.clone()).unwrap_or_default(); self.create_report("Compare Report", &pdfcraft_compare::report(&c, &name(old), &name(new))) } @@ -104,7 +118,12 @@ impl Session { /// (blue, green) and a note where text was deleted (red), authored "Compare". One undoable /// step; returns how many comments were added. pub fn mark_differences(&mut self, old: DocId, new: DocId) -> Result { - let c = self.compare(old, new)?; + self.mark_differences_of(old, new, &[]) + } + + /// [`Session::mark_differences`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn mark_differences_of(&mut self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let c = self.compare_pages(old, new, pages)?; let mut edits = Vec::new(); for ch in &c.changes { let color = colour(ch.kind); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 688dcec049..5b6fb01b9e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5841,6 +5841,10 @@ fn comparing_two_versions_of_a_document() { let got: Vec<(compare::Kind, &str, &str)> = c.changes.iter().map(|x| (x.kind, x.old.text.as_str(), x.new.text.as_str())).collect(); assert_eq!(got, [(compare::Kind::Replaced, "Monday.", "Tuesday."), (compare::Kind::Replaced, "days.", "days of invoice.")], "{got:?}"); assert!(s.compare(old, old).unwrap().identical()); + // limited to pages of the new file: both changes are on the first + assert_eq!(s.compare_pages(old, new, &[]).unwrap().changes.len(), 2); + assert_eq!(s.compare_pages(old, new, &[0]).unwrap().changes.len(), 2); + assert!(s.compare_pages(old, new, &[1]).unwrap().identical()); let report = s.compare_report(old, new).unwrap(); let rid = s.open("report.pdf", None, report, None).unwrap(); diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 3b3d5dd036..04dbd19447 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -210,6 +210,7 @@ impl PdfCraftApp { action_files_override: None, pdfa: Default::default(), compare_old: None, + compare_pages: String::new(), compare: None, js_console: Default::default(), doc_js: Default::default(), diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2f393c61d1..380f84a644 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -526,6 +526,8 @@ pub struct PdfCraftApp { pub pdfa: standards_ui::PdfaState, /// Compare files: the chosen older document and the last result. pub compare_old: Option, + /// The Compare Files dialog's Pages field: the new file's pages to compare, empty for all. + pub compare_pages: String, pub compare: Option, /// The JavaScript console and the Document JavaScripts draft. pub js_console: js_ui::JsConsole, diff --git a/crates/ui-egui/src/workflows/compare_ui.rs b/crates/ui-egui/src/workflows/compare_ui.rs index 939d5fed3e..7bf4c9f24b 100644 --- a/crates/ui-egui/src/workflows/compare_ui.rs +++ b/crates/ui-egui/src/workflows/compare_ui.rs @@ -14,6 +14,8 @@ pub struct CompareState { pub old: DocId, pub new: DocId, pub result: Comparison, + /// The pages of the new file this comparison covers (0-based; empty: every page). + pub pages: Vec, /// Regions that look different (page, user-space box), page n against page n. pub visual: Vec<(usize, [f64; 4])>, pub selected: Option, @@ -52,13 +54,14 @@ pub fn colour(kind: Kind) -> Color32 { pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { ui.label(egui::RichText::new(tl!("Compare Files")).font(theme::semibold(18.0))); ui.add_space(8.0); - let Some((_, new)) = app.active_ids() else { return (false, true) }; + let Some((active_i, new)) = app.active_ids() else { return (false, true) }; let others: Vec<(DocId, String)> = app.views.iter().filter(|v| v.id != new).filter_map(|v| app.session.get(v.id).map(|d| (d.id, d.name.clone()))).collect(); if app.compare_old.is_none_or(|o| !others.iter().any(|(id, _)| *id == o)) { app.compare_old = others.first().map(|(id, _)| *id); } let new_name = app.session.get(new).map(|d| d.name.clone()).unwrap_or_default(); + let page_count = app.views.get(active_i).map_or(0, |v| v.page_count()); egui::Grid::new("compare-files").num_columns(2).spacing([12.0, 10.0]).show(ui, |ui| { ui.label(tl!("Old file:")); let shown = @@ -72,14 +75,23 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.label(tl!("New file:")); ui.label(egui::RichText::new(new_name).strong()); ui.end_row(); + ui.label(tl!("Pages:")); + let field = ui.add(egui::TextEdit::singleline(&mut app.compare_pages).desired_width(320.0).hint_text(format!("1-{page_count}"))); + field.widget_info(|| egui::WidgetInfo::labeled(egui::Role::TextInput, ui.is_enabled(), tl!("Pages"))); + ui.end_row(); }); + let pages_error = crate::page_selection_ui::parse(&app.compare_pages, page_count).err(); + if let Some(e) = &pages_error { + ui.label(e.said(&|s| crate::i18n::t(s).to_string())); + } ui.add_space(4.0); ui.label(egui::RichText::new(tl!("Text is compared word by word; the differences are shown on the new file.")).small().color(t.text_muted)); ui.add_space(10.0); let (mut go, mut cancel) = (false, false); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(app.compare_old.is_some(), |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { + let ready = app.compare_old.is_some() && pages_error.is_none(); + if ui.add_enabled_ui(ready, |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -191,9 +203,17 @@ impl PdfCraftApp { pub fn run_compare(&mut self) { let Some((i, new)) = self.active_ids() else { return }; let Some(old) = self.compare_old else { return }; - match self.session.compare(old, new) { + let count = self.views.get(i).map_or(0, |v| v.page_count()); + let pages = match crate::page_selection_ui::parse(&self.compare_pages, count) { + Ok(pages) => pages, + Err(e) => return self.notify_error(e.said(&|s| crate::i18n::t(s).to_string())), + }; + match self.session.compare_pages(old, new, &pages) { Ok(result) => { - let visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + let mut visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + if !pages.is_empty() { + visual.retain(|(page, _)| pages.contains(page)); + } let n = result.changes.len(); // Side by side when the old version is open in a tab and the window has the room. let side_by_side = self.views.iter().any(|v| v.id == old) && self.ctx.as_ref().is_none_or(|c| c.content_rect().width() >= 1000.0); @@ -201,6 +221,7 @@ impl PdfCraftApp { old, new, result, + pages, visual, selected: None, side_by_side, @@ -225,7 +246,7 @@ impl PdfCraftApp { pub(crate) fn compare_action(&mut self, index: usize, a: PanelAction) { let Some(s) = self.compare.as_mut() else { return }; - let (old, new) = (s.old, s.new); + let (old, new, pages) = (s.old, s.new, s.pages.clone()); match a { PanelAction::Select(i) => { s.selected = Some(i); @@ -252,7 +273,7 @@ impl PdfCraftApp { s.show_visual = visual; self.shade_compare(index); } - PanelAction::Mark => match self.session.mark_differences(old, new) { + PanelAction::Mark => match self.session.mark_differences_of(old, new, &pages) { Ok(n) => { if let Some(info) = self.session.get(new).map(|d| d.info.clone()) { self.views[index].document_changed(&info); @@ -265,7 +286,7 @@ impl PdfCraftApp { } Err(e) => self.notify_error(e), }, - PanelAction::Report => match self.session.compare_report(old, new) { + PanelAction::Report => match self.session.compare_report_of(old, new, &pages) { Ok(bytes) => { if let Err(e) = self.open_bytes("Compare Report.pdf", None, bytes.to_vec()) { self.notify_error(e); diff --git a/crates/ui-egui/tests/it/compare.rs b/crates/ui-egui/tests/it/compare.rs index 52acae545f..25a4616727 100644 --- a/crates/ui-egui/tests/it/compare.rs +++ b/crates/ui-egui/tests/it/compare.rs @@ -160,6 +160,43 @@ fn compare_filters_hide_text_or_visual_changes() { assert_eq!(kinds(&h, old_i), (text_old, 0)); } +/// The Compare Files dialog's Pages field rejects an out-of-range page: the +/// comparison does not run and the dialog stays open with the typed range kept. +#[test] +fn compare_dialog_pages_field_rejects_out_of_range() { + let s = Session::new(); + let v1 = s.create_from_text("t", "Delivery within five days. Returns accepted.").unwrap().to_vec(); + let v2 = s.create_from_text("t", "Delivery within three days. Returns accepted for a week.").unwrap().to_vec(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(move |_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("v1.pdf", None, v1.clone()).unwrap(); + app.open_bytes("v2.pdf", None, v2.clone()).unwrap(); + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("doc.compare")); + h.run_steps(2); + h.get_by_label("Compare Files"); + let pages = h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages"); + assert!(pages.is_some(), "the dialog has a Pages text input"); + assert!(h.query_by_label("Pages:").is_some(), "the dialog has a Pages field"); + let Some(pages) = pages else { return }; + pages.focus(); + pages.type_text("99"); + h.run_steps(2); + assert_eq!( + h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages").and_then(|n| n.value()), + Some("99".to_string()), + "the typed range is kept" + ); + h.get_by_label("Compare").click(); + h.run_steps(3); + assert!(h.query_by_label("Compare Files").is_some(), "the dialog stays open for an out-of-range page"); + assert!(h.state().compare.is_none(), "the comparison did not run"); + assert_ne!(h.state().right, Some(RightPanel::Compare)); +} + /// Writes a screenshot of the Compare panel when PDFCRAFT_SHOT is set (for review). #[test] fn compare_panel_screenshot() { From 81b0833b245fee444e4703b4298aea949995a9aa Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:50:02 +0100 Subject: [PATCH 72/85] Copy the shared house rules into AGENTS.md. The blocks come from craft-contrib at dde09ea31e2e457f5b4653a6e76f7cdf55944bdf. CLAUDE.md imports AGENTS.md. The house-rules workflow checks the copy. --- .github/workflows/house-rules.yml | 28 ++++++++++++ AGENTS.md | 71 ++++++++++++++++++++++++++++--- CLAUDE.md | 66 +--------------------------- 3 files changed, 95 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/house-rules.yml diff --git a/.github/workflows/house-rules.yml b/.github/workflows/house-rules.yml new file mode 100644 index 0000000000..de64ad3ee4 --- /dev/null +++ b/.github/workflows/house-rules.yml @@ -0,0 +1,28 @@ +name: House rules + +on: + push: + branches: [main] + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + pull_request: + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: house-rules-${{ github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + uses: storytold/craft-common-actions/.github/workflows/house-rules.yml@main + with: + rust: true diff --git a/AGENTS.md b/AGENTS.md index 3ab900aa95..f55ab534a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,56 @@ + +## House rules + +These apply in every Craft repository. A later rule may be stricter, not weaker. The source is `standards/house-rules-core.md` in `storytold/craft-contrib`. Propose changes there, not in a copy. + +1. **A person is accountable.** Whoever runs an agent answers for what it opens and for its review comments. +2. **Check, then claim.** Before writing code, read the issue comments, search pull requests for `#N` and the keywords, and check recent commits on `main`. Comment on the issue first. Release a claim you will not finish. +3. **Verify, don't guess.** Open a definition before using it. Confirm the cause in the code, even when someone names a location. +4. **Keep fixes narrow and tested.** Fix what the issue describes, where the bug is. Add a test that fails without the fix. +5. **Review the diff before opening a pull request.** Identifiers exist as used, the test fails without the fix, nothing unrelated changed, and the description is true. +6. **Look for a better pull request.** If another fixes the same thing, close yours when it is better or equal, and say why. +7. **Report only what ran.** Say what you tested and what you could not run. Without a local build or test, say so and rely on CI before marking ready. +8. **`main` changes through a pull request.** Do not push to it. Branch protection enforces this. There is no exception. +9. **Stop and ask only about** licensing outside the allowlist, publishing, releases, a new remote, secrets, spending money, destructive git, and anything else hard to undo. Otherwise keep working. `plan/` is a gitignored notebook, not rules. Without it, do not ask for it and do not stop. +10. **Keep secrets and personal details out** of the repository, logs and requests. +11. **CI stays shared.** Workflows call `storytold/craft-common-actions`. Do not change CI in a feature or fix pull request. If CI cannot cover a change, open an issue there. +12. **Guards say why.** A check that looks removable says what it prevents. Read that before changing it. +13. **Instructions must be readable by everyone.** No notes about one machine, and no links to files a reader cannot open. `craftrules` is retired. Do not link to it. +14. **Keep pull requests moving.** Concurrent pull requests are fine; unanswered actionable feedback is not. Respond in 7 days, a reminder at 14, closure eligible at 30, from when an actionable request waits on the author. Suggestions and unconfirmed bot findings do not start the clock. A fix, explanation or agreed plan resets it; unrelated activity does not. Waiting on a reviewer or maintainer pauses it and does not count against the author. A maintainer decides closure; it is never automatic. Reuse search results, do not poll, and stop at a rate limit. +15. **No agent attribution.** A pull request does not say an agent wrote it. A repository rule may not require that. + + +## Rust Craft repositories + +These apply to native Craft apps and to shared Rust libraries. Select them by repository type. A repository that merely contains some Rust does not take this block. Rules later in a repository's file may be stricter, but not weaker. The source is `standards/house-rules-rust.md` in [storytold/craft-contrib](https://github.com/storytold/craft-contrib). + +1. **Never crash.** Non-test code does not panic and has no `unsafe`, except where the repository file names an exception. Return errors, treat input as hostile, and bound recursion and allocations. New production crate roots copy the `deny` attribute from an existing crate: `clippy::unwrap_used`, `clippy::expect_used`, `clippy::panic`, `clippy::unimplemented`, `clippy::todo`, `clippy::unreachable`. Every crash fix comes with a test that panicked before the fix. +2. **Clean room.** Learn from observed behaviour and public specifications. Do not copy another product's code, assets, strings or artwork. Check a dependency's licence before adding it. +3. **Assets and fonts.** Every asset is original or suitably licensed, and is recorded in `ATTRIBUTION.md` in the same change. Font files live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), not in the app repository. +4. **Rust only for native Craft apps.** No handwritten JavaScript or TypeScript, and no Tauri, Electron or webview shell, in those apps. Shared tooling follows its documented technical scope and maintainer-approved exceptions. Craft PR Review's existing Tauri and TypeScript UI is one such exception. It does not relax this rule for native apps. Preserve any existing web build. +5. **Time goes through `chrono`.** Do not hand-roll calendar conversion. For dates beyond what `chrono` can represent, follow [standards/time.md](https://github.com/storytold/craft-contrib/blob/main/standards/time.md). +6. **Contributor credits.** Add or edit only your own person's entry in `contributors/people.toml`, and only after asking them. Never guess a name or hand-edit `contributors.json`. + +## Licence allowlist + +Dependencies and assets use SPDX identifiers. An expression is allowed when every identifier is in this list. A repository may name a shorter list. It may not add an identifier here by editing its own copy of these rules. + +- `ISC` +- `MIT` +- `BSD-2-Clause` +- `BSD-3-Clause` +- `Apache-2.0` +- `OFL-1.1` +- `Ubuntu-font-1.0` +- `Bitstream-Vera` +- `CC0-1.0` +- `CC-BY-4.0` +- `CC-BY-SA-4.0` +- `LicenseRef-Public-Domain` + +`LicenseRef-Storyteller-Trademark` is not in this list. It covers Storyteller brand marks only, where a repository already says so. + + # AGENTS.md — rules for every agent and contributor These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** @@ -85,7 +138,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi --- -## 2. Clean-room rules (summary; details in `CLAUDE.md` and `plan/adr/0001`) +## 2. Clean-room rules (summary; `plan/adr/0001` is an optional local note) - Never read, disassemble or copy anything inside the Acrobat bundle. Only names and listings are allowed, and **never** open `Contents/Resources/JavaScripts/`. - Behaviour comes only from public specifications (ISO 32000-2, the Arlington model), public documentation and black-box observation. @@ -106,7 +159,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi PdfCraft opens files from strangers, and people trust it with their work. A malformed PDF, a bad tool argument, a corrupt settings file or a full disk must produce an error the user (or agent) can act on, never a crash and never lost work. **This outranks feature work:** don't ship a feature by adding a panic path, and fix a crash before building on top of it. The rules below are the ones that bind this repository. -- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity in `CLAUDE.md`). +- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity below). - **No panicking shortcuts.** No `unwrap()`, `expect()`, `panic!`, `unreachable!`, `todo!` or `unimplemented!` outside tests, unless the call is provably infallible (e.g. writing to a `Vec`), with a comment saying why. `assert!` only for internal invariants that input cannot reach. - **No `unsafe`.** The workspace sets `unsafe_code = "forbid"`. If a platform API needs it, use a maintained safe wrapper crate; a real exception needs the owner's sign-off. - **Treat every input-derived number as hostile.** Use `get()` instead of `[i]` or `[a..b]`. Slice strings only at char boundaries. Use checked/saturating arithmetic for lengths, offsets and counts. Guard against division by zero and NaN/inf casts, and never cast a negative value to `usize`. Cap allocations sized by the file. @@ -115,10 +168,18 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Prove it.** Every crash fix comes with a small synthetic regression test that panicked before the fix. Turn fuzzer findings (`cargo xtask fuzz`) into such tests before fixing them. - **Enforced by clippy.** Root `clippy.toml` lets tests use `unwrap`/`expect`/`panic!`. A crate that is clean carries `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`; new crates start with it. -## 5. Everything else +## 5. Page interaction, fidelity and gates - -Follow `CLAUDE.md`. It covers quality gates, layering, commit rules and how to run and look at the app. +- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd. `cos`, `filters`, `crypt` and `arlington` stay standalone. +- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state. `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. +- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim`, `crates/ui-egui/src/canvas/page_cursor.rs`) and never set it themselves. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. +- **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. +- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. +- **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. +- `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. +- `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. ## Contributor credits (About window) diff --git a/CLAUDE.md b/CLAUDE.md index 5c7034b7a4..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,65 +1 @@ -# PdfCraft — instructions for agents - -PdfCraft is a clean-room, open-source, Rust-native PDF application targeting Adobe Acrobat Pro parity. It runs natively on macOS, Windows, Linux and FreeBSD, and on the web via WASM. It is the sibling of `../photocraft` (a Photoshop-class editor) and follows the same conventions. - -## Start every session here -1. Read `plan/STATUS.md`: the current milestone, the next unchecked task and any blockers. `ROADMAP.md` is the one-page summary (stage, numbers, progress log) and `docs/roadmap.md` the milestone table; at the end of every session update the milestone row and add a line to the ROADMAP.md progress log. - - Read `docs/gaps.md` (the ranked gaps: Acrobat interop, the open-issue backlog, the renderer and a fidelity harness, editing existing content, Pro workflows, 1.0 polish) and `docs/target-app-parity.md` (the numbers and how they are measured) before choosing work. Prefer the gaps over new P2/P3 features, and keep both documents true when things change (`docs/` progress docs carry a status line and a revision history). - - "Shipped" in `parity/` means "exists and tested", not "as good as Acrobat". Don't mark a feature shipped on a generic test, and say in its notes what is still missing. -2. Read that task in `plan/execution-plan.md` §3, the relevant section of `plan/architecture.md`, and the README of the crate you're touching. -3. Follow the **autonomous operation protocol** in `plan/execution-plan.md` §7 (orient → plan → implement + test → verify → record → commit). Don't stop to ask unless §7 lists the decision as the user's. - -`plan/` is gitignored (local-only, like PhotoCraft). The machine-readable parity checklist lives in `parity/` (committed). - -## Non-negotiables -- **Assets: read `AGENTS.md` §1 before adding or showing any icon, image, font or document.** No assets from Adobe products, ever. Only openly licensed or contributor-original assets are allowed, each with an entry in `ATTRIBUTION.toml`. `cargo xtask assets` enforces this. `AGENTS.md` overrides this file. -- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts).** Never commit font files here (`AGENTS.md` §1.4). It is the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts && CRAFT_FONTS_DIR=../craft-fonts cargo test --workspace` embeds the Japanese fonts (UI fallback, Japanese text in edited PDFs) and runs their tests, which otherwise skip. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Clean-room.** - - Never read, disassemble or copy anything inside the Acrobat bundle (names and listings only). **Never open `Contents/Resources/JavaScripts/`.** - - Behaviour comes from public docs, specs (ISO 32000-2, the Arlington model) and black-box observation (`plan/acrobat/`). - - Never copy GPL/AGPL code. MuPDF, Ghostscript, Poppler, veraPDF and DSS run only as external oracle processes. - - See `plan/README.md` §Clean-room and `plan/adr/0001`. -- **Privacy.** When observing Acrobat, use synthetic fixtures only. Never capture the Home view, recent files or account info. Capture by window id (`plan/acrobat/tools/`). Never commit Acrobat outputs, corpus files or personal data. -- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd (`plan/architecture.md` §3). `cos`/`filters`/`crypt`/`arlington` stay standalone. -- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state (`hover_pos`, `interact_pos`, `latest_pos`, `press_origin`, button reads); `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. -- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim(Layer::…, icon)`, `crates/ui-egui/src/canvas/page_cursor.rs`), never set it (`set_cursor_icon`, `cursor_icon =`, `on_hover_cursor`); `document_area` sets it once, from the claim on the highest layer the active tool reaches. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. -- **Page presses.** A page module decides what of its own is under a point in one hit test, which returns a `Hit` (owner, layer, cursor, tip; `crates/ui-egui/src/canvas/page_router.rs`) and gives its cursor, its tip and its press. It acts on a press only when the press is its own or, for the active tool, nobody's: the page view tells it which (`PagePress`: its own, nobody's, or another page's), and it never tests the press point again or asks another module whether it wants it. Escape cancels any press the pages hold (`cancel_on_escape`), and that press is ignored until the button is up. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. -- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. -- **Never panic.** Every PDF, script, MCP call, settings file and keystroke is untrusted input, and none of it may crash the app or lose the user's work. This outranks feature work. Return `Result` (the crate's error enum) and propagate with `?`, or fall back leniently and record it. In non-test code: - - No `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` unless it is provably infallible, with a comment saying why. - - No indexing or slicing with input-derived positions (use `get`, and slice strings only at char boundaries). - - Use checked or saturating arithmetic on input-derived numbers, and guard against division by zero and NaN/inf casts. - - Cap allocations sized by input, and bound recursion with depth limits or seen-sets. - - Handle lock poisoning. - - No `unsafe` (`unsafe_code = "forbid"`). - - Every crash fix gets a synthetic regression test. See `AGENTS.md` §4. -- **Rust only** in the product and build (`xtask`). No handwritten JS/TS. -- **Time goes through `chrono`** (workspace dependency). Never hand-roll UTC/calendar conversion (days-to-date algorithms, leap-year math); a crate owns only its output format string. The one exception is range extension: past the years chrono can represent, shift the date by whole 400-year Gregorian cycles (146,097 days, after which the calendar repeats), let chrono compute the month and day, and add the cycles back to the year (splitting seconds into hours, minutes and seconds is plain arithmetic). -- **Review every update as you make it, at four zooms.** Not an end-of-task pass — a per-update habit, so a defect never gets built on. **Clause:** checked arithmetic, `get()` not `[i]`, char-boundary slicing, the right error through `?`, `<` against `<=`, the 1-based/0-based seam, capped allocations. **Function:** is it total — name an input that panics; walk empty, zero, max, malformed, cyclic; is the return type honest. **File/module:** two sources of truth, dead or duplicated code, is the public surface still smallest and the doc comment still true. **Project:** layering, wasm, undo/redo, save fidelity, encryption and permissions, a matching `automation` tool or registry entry, and whether a `parity/` status or note just became untrue. Scale the zoom to the change, and fix what a level turns up before widening. - - Tests mirror the zooms: property → unit → crate integration → end-to-end. - - **Prove a test fails before the fix by actually reverting the fix.** A test nobody has seen fail is not evidence. This has caught two worthless tests in this repo: one confinement test passed with path resolution switched off entirely, because the tools failed on a nonexistent document id before reaching the path. - - **Rebasing is a review step, not a mechanical one.** Read every hunk you resolve, and the auto-merged hunks in the files you touched too — `main` moves fast enough that a file can change by hundreds of lines under a branch. -- **External PRs are held to safety, security and completeness before style.** Provenance first (no Adobe-derived asset, no committed font, no copied GPL/AGPL code, behaviour from the spec and not from reading another implementation), then no new panic path or `unsafe`, then `--root` confinement on every path including ones reached by nested dispatch, opt-in-only MCP and control channel, no new outbound network, then a headless `automation` tool, tests at the right zoom and an honest `parity/` entry. Provenance is the part a contributor cannot be expected to know, and the only part that cannot be undone after merge. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check once `xtask ci` exists (M0). - - New workspace crates must include `[lints] workspace = true` in `Cargo.toml` to inherit `[workspace.lints]`. Production crate roots must declare the required crate-level lint attributes before modules or other crate items; crate-level `//!` documentation may precede them. - - **Scope builds and tests to the crates you touched** — `cargo clippy -p -j 2 --all-targets` and `cargo test -p -j 2`, under a `timeout`. A full `--workspace` build has locked up a development machine's CPU; it is CI's job and CI runs it on every PR. Run the workspace sweep locally only when you have a reason to and the machine is free. -- **Check for existing work before starting a fix or feature.** Search the issues and PRs, open and closed, with `gh issue list -R storytold/pdfcraft --state all --search ""` and the same with `gh pr list` (try a few phrasings), and check whether a merged fix is already on `main` (`git merge-base --is-ancestor origin/main`). If one is, the report is likely from an older build: say so rather than fixing it again. If an open PR covers it, review or build on that PR instead of opening a competing one. If an issue exists, reference it in the PR (`Fixes #123`). -- **Commits and PRs:** `main` is the only integration branch and is **never pushed to directly**. Branch from an up-to-date `main` as `/` (`feat/`, `fix/`, `docs/`, `chore/`, `refactor/`), one task id per commit (e.g. `M1.4: xref stream reader`), commit only green states, and open a PR into `main` using `.github/pull_request_template.md`. Merge only when CI is green. - - **`main` moves fast enough that PRs rot.** It took 396 commits in the three days to 2026-10-12, and most open PRs are conflicting at any given time. Rebase your own branch the same day `main` moves under it, and ask the author to rebase theirs — never rewrite a contributor's branch. Conflict risk tracks how *contested* the files are, not how large the change is: tooling, tests and docs are nearly free, while `crates/fonts`, `ui-egui` and the i18n catalogues are where several people collide at once. - -## Running and looking at the app -- `cargo run -p pdfcraft -- ` opens the desktop app. -- `cargo run -p pdfcraft-cli -- run --script steps.json --root DIR` drives the engine headlessly through the automation tools (`pdfcraft-cli tools` lists them). Use it, together with `page_render`, to check engine changes. `pdfcraft-cli mcp` is the opt-in MCP server (AGENTS.md §3). -- `cargo xtask fuzz --time 300` mutation-fuzzes open/render/edit/save in child processes. Findings land in `fuzz-out/findings/` (git-ignored; never commit corpus-derived files). Turn every real finding into a small synthetic regression test before fixing it. -- `cargo xtask parity [--partial]` reports Acrobat-parity progress from `parity/acrobat-features.toml`. Update the entry when a feature ships. -- `cargo xtask demo-pdf` builds `dist/demo/pdfcraft-showcase.pdf` (needs Chrome) for visual checks. -- For UI work, **look at the result**. Either launch `pdfcraft --control FILE doc.pdf` and use `pdfcraft-cli ui --control FILE screenshot --out x.png` (plus `inspect`, `click`, `key`, `type`, `command`, `set`), or take a headless shot with `cargo run -p pdfcraft-ui-egui --example shot`. Compare against `plan/acrobat/02-ui-ux.md`. Control-channel tests use kittest (`crates/ui-egui/tests/control.rs`). -- Parallel agents: use a separate `CARGO_TARGET_DIR` per agent and separate git worktrees. - -## Current bootstrap debt (tracked in STATUS.md) -- `pdfcraft-render` renders through the `hayro` crate directly and inspects documents through `lopdf`. Both get replaced by `cos` / `model` / the DisplayList device (M1–M2, ADR-0004). - -## CI workflows -- Workflows in `.github/workflows/` call the shared workflows in [`storytold/craft-common-actions`](https://github.com/storytold/craft-common-actions) at `@main`, with their inputs and secrets. No other jobs: if no shared workflow does what's needed, it doesn't go here. -- Don't add or change CI in a feature or fix PR. If the existing workflows can't build or test a change, link an existing craft-common-actions issue or PR, or open a new issue, or a PR only for a small fix that is safe for every app and does not increase CI time. -- Some workflow files here still run their own jobs. Don't add to them. Moving a job into craft-common-actions is its own task, never part of another PR, and the job stays here until a shared workflow runs it. +@AGENTS.md From 465783fc47a3cb98ceaabaa1cc851c0c0a1b4ea3 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:37:25 +0100 Subject: [PATCH 73/85] docs: preserve local guidance in house-rules rollout --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f55ab534a8..c228c4185b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Dependencies and assets use SPDX identifiers. An expression is allowed when ever # AGENTS.md — rules for every agent and contributor -These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** +These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. Day-to-day working instructions are in this file; `CLAUDE.md` imports it. --- @@ -176,7 +176,7 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. - **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. - **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `workspace = true` under `[lints]`. Scope `cargo clippy -p --all-targets -- -D warnings` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. - **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. - `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. - `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. From 1d71d6039f43a645ec9a1b711d6f080f3e5b6cca Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:16:36 +0200 Subject: [PATCH 74/85] Add PCX image support to Create PDF conversion (#6396) * Add PCX image support to Create PDF conversion PCX is listed as an accepted Create source elsewhere but PdfCraft rejects it, so users cannot convert PCX images to PDF. Detect PCX by its header in is_image and decode it in embed, adding pcx to CONVERTIBLE so existing from_images and source_kind paths handle it. Test: pcx_images_become_pages failed before the change and passes after. * Create: PCX scanlines hold every plane's row, 8-bit indexed images, older versions, standard test files Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/create/src/lib.rs | 134 ++++++++++++++++++++++++++++++++++++- crates/create/src/tests.rs | 76 +++++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index 53a587728f..149bd99d97 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -516,6 +516,129 @@ fn decoded(name: &str, bytes: &[u8], format: image::ImageFormat) -> Result Option { + b.get(at..at + 2).map(|x| u16::from_le_bytes([x[0], x[1]])) +} + +/// How a PCX image holds its pixels: bits per plane and planes. +#[derive(Clone, Copy, PartialEq)] +enum PcxLayout { + /// One byte per pixel per colour: each scanline has a red, a green and a blue row. + Rgb, + /// One byte per pixel, an index into the palette after the image (grey without one). + Indexed, + /// One bit per pixel in each of 1 to 4 planes, an index into the header's palette. + Bits(usize), +} + +impl PcxLayout { + fn of(bits: u8, planes: usize) -> Option { + match (bits, planes) { + (8, 3) => Some(PcxLayout::Rgb), + (8, 1) => Some(PcxLayout::Indexed), + (1, 1..=4) => Some(PcxLayout::Bits(planes)), + _ => None, + } + } +} + +/// The palette of a PCX image, and where its run-length coded scanlines end. An 8-bit image ends +/// with a 0x0C byte and 256 colours; a one-plane bit image is black and white whatever its header +/// holds; other bit images use the 16 colours at byte 16. +fn pcx_palette(bytes: &[u8], layout: PcxLayout) -> (Vec, usize) { + match layout { + PcxLayout::Rgb => (Vec::new(), bytes.len()), + PcxLayout::Indexed => match bytes.len().checked_sub(769).filter(|at| *at >= 128 && bytes.get(*at) == Some(&0x0C)) { + Some(at) => (bytes.get(at + 1..).unwrap_or_default().to_vec(), at), + None => ((0..=255u8).flat_map(|v| [v, v, v]).collect(), bytes.len()), + }, + PcxLayout::Bits(1) => (vec![0, 0, 0, 255, 255, 255], bytes.len()), + PcxLayout::Bits(_) => (bytes.get(16..64).unwrap_or_default().to_vec(), bytes.len()), + } +} + +/// The `rows` scanlines of `line` bytes each, decoded. A run may cross from one plane's row into the +/// next, but not out of its scanline. +fn pcx_scanlines(mut body: &[u8], rows: usize, line: usize) -> Option> { + let mut out = Vec::new(); + out.try_reserve_exact(rows.checked_mul(line)?).ok()?; + for row in 1..=rows { + let end = row * line; + while out.len() < end { + let (&c, rest) = body.split_first()?; + body = rest; + if c < 0xC0 { + out.push(c); + continue; + } + let (&value, rest) = body.split_first()?; + body = rest; + let count = usize::from(c & 0x3F); + if count > end - out.len() { + return None; + } + out.resize(out.len() + count, value); + } + } + Some(out) +} + +/// The RGB samples of the `w` pixels of scanline `row` (planes `stride` bytes apart). +fn pcx_row(out: &mut Vec, row: &[u8], w: usize, stride: usize, layout: PcxLayout, palette: &[u8]) { + let at = |i: usize| row.get(i).copied().unwrap_or(0); + for x in 0..w { + let index = match layout { + PcxLayout::Rgb => { + out.extend([at(x), at(stride + x), at(2 * stride + x)]); + continue; + } + PcxLayout::Indexed => usize::from(at(x)), + PcxLayout::Bits(planes) => (0..planes).fold(0, |index, k| index | (usize::from(at(k * stride + x / 8) >> (7 - x % 8) & 1) << k)), + }; + out.extend((0..3).map(|c| palette.get(index * 3 + c).copied().unwrap_or(0))); + } +} + +/// A PCX image: a 128-byte header (size at 4-11, resolution at 12 and 14, plane count at 65, bytes +/// per plane row at 66), then run-length coded scanlines, each holding one row of every plane. +fn pcx(name: &str, bytes: &[u8]) -> Result { + let bad = |m: &str| CreateError::Image(name.into(), m.into()); + let header = bytes.get(..128).ok_or_else(|| bad("a damaged PCX header"))?; + let word = |at: usize| le16(header, at).unwrap_or(0); + let byte = |at: usize| header.get(at).copied().unwrap_or(0); + let (bits, planes, stride) = (byte(3), usize::from(byte(65)), usize::from(word(66))); + let (Some(w), Some(h)) = (word(8).checked_sub(word(4)), word(10).checked_sub(word(6))) else { + return Err(bad("a damaged PCX header")); + }; + let (w, h) = (usize::from(w) + 1, usize::from(h) + 1); + if planes == 0 || stride == 0 { + return Err(bad("a damaged PCX header")); + } + let layout = PcxLayout::of(bits, planes).ok_or_else(|| bad("this PCX plane layout is not supported"))?; + if stride < (w * usize::from(bits)).div_ceil(8) { + return Err(bad("a damaged PCX header")); + } + let line = planes * stride; + if line.saturating_mul(h) > MAX_DECODED_BYTES || w.saturating_mul(h).saturating_mul(3) > MAX_DECODED_BYTES { + return Err(bad("unusual PCX image size")); + } + let (palette, end) = pcx_palette(bytes, layout); + let body = bytes.get(128..end).unwrap_or_default(); + let scanlines = pcx_scanlines(body, h, line).ok_or_else(|| bad("a damaged PCX body"))?; + let mut data = Vec::new(); + data.try_reserve_exact(w * h * 3).map_err(|_| bad("not enough memory for this image"))?; + for row in scanlines.chunks_exact(line) { + pcx_row(&mut data, row, w, stride, layout, &palette); + } + let dpi = (f64::from(word(12)), f64::from(word(14))); + let dpi = if dpi.0 > 0.0 && dpi.1 > 0.0 { dpi } else { (72.0, 72.0) }; + let mut d = Dict::new(); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let px = (u32::try_from(w).unwrap_or(0), u32::try_from(h).unwrap_or(0)); + Ok(Embedded { dict: d, data, filtered: false, smask: None, icc: None, px, dpi }) +} + /// Largest GIF logical screen, in pixels (4096×4096). const MAX_GIF_PIXELS: u64 = 1 << 24; @@ -623,7 +746,13 @@ pub enum SourceKind { } /// File extensions Create converts to PDF (images and plain text). -pub const CONVERTIBLE: [&str; 12] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "txt", "text"]; +pub const CONVERTIBLE: [&str; 13] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx", "txt", "text"]; + +/// A PCX file: its manufacturer byte, a version (1 was never used), run-length coding and 1 or 8 +/// bits per plane. `pcx` says when the rest of the header isn't one it can read. +fn is_pcx(bytes: &[u8]) -> bool { + bytes.first() == Some(&0x0A) && matches!(bytes.get(1), Some(0 | 2..=5)) && bytes.get(2) == Some(&1) && matches!(bytes.get(3), Some(1 | 8)) +} /// A classic TIFF or a BigTIFF (version 43), in either byte order. fn is_tiff(bytes: &[u8]) -> bool { @@ -640,6 +769,7 @@ fn is_image(bytes: &[u8]) -> bool { || bytes.starts_with(b"\x89PNG\r\n\x1a\n") || is_tiff(bytes) || is_gif(bytes) + || is_pcx(bytes) // JPEG 2000: a JP2 file or a raw codestream. || bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) @@ -675,6 +805,8 @@ fn embed(name: &str, bytes: &[u8], wanted: usize) -> Result, Creat Ok(vec![decoded(name, &pdfcraft_xfa::image::bmp_with_pixel_offset(bytes), image::ImageFormat::Bmp)?]) } else if is_gif(bytes) { Ok(vec![decoded(name, bytes, image::ImageFormat::Gif)?]) + } else if is_pcx(bytes) { + Ok(vec![pcx(name, bytes)?]) } else if bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) { Ok(vec![jpx(name, bytes)?]) } else { diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 9e620c19c3..bea96c887d 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -485,6 +485,82 @@ fn bigtiff_images_become_pages() { assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 1.0], "4×2 px at 144 dpi"); } +/// A PCX header for a `w` by `h` image at 72 dpi. +fn pcx_header(version: u8, bits: u8, planes: u8, stride: u16, w: u16, h: u16) -> Vec { + let mut pcx = vec![0u8; 128]; + pcx[..4].copy_from_slice(&[0x0A, version, 1, bits]); + pcx[8..10].copy_from_slice(&(w - 1).to_le_bytes()); + pcx[10..12].copy_from_slice(&(h - 1).to_le_bytes()); + pcx[12..14].copy_from_slice(&72u16.to_le_bytes()); + pcx[14..16].copy_from_slice(&72u16.to_le_bytes()); + pcx[65] = planes; + pcx[66..68].copy_from_slice(&stride.to_le_bytes()); + pcx +} + +/// A 24-bit PCX: each scanline holds a red, a green and a blue row. +#[test] +fn pcx_images_become_pages() { + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[10, 11, 20, 21, 30, 31]); + pcx.extend_from_slice(&[40, 41, 50, 51, 60, 61]); + let embedded = super::pcx("a.pcx", &pcx).unwrap(); + assert_eq!(embedded.data, [10, 20, 30, 11, 21, 31, 40, 50, 60, 41, 51, 61]); + let doc = reopen(&from_images(&[("a.pcx".into(), pcx)]).unwrap()); + assert_eq!(pages(&doc).len(), 1); + assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 2.0]); + assert_eq!(image_of(&doc, 0).name(b"ColorSpace"), Some(&b"DeviceRGB"[..])); +} + +#[test] +fn a_pcx_run_may_cross_planes_but_not_scanlines() { + // 0xC4 repeats 7 four times: the red row and half of the green one + let mut pcx = pcx_header(5, 8, 3, 2, 2, 1); + pcx.extend_from_slice(&[0xC4, 7, 8, 9]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [7, 7, 8, 7, 7, 9]); + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[0xC8, 7, 1, 2, 3, 4]); + assert!(super::pcx("a.pcx", &pcx).is_err(), "eight bytes in a six-byte scanline"); +} + +#[test] +fn an_8_bit_pcx_uses_its_palette_and_is_grey_without_one() { + let mut pcx = pcx_header(5, 8, 1, 2, 2, 1); + pcx.extend_from_slice(&[1, 2]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 1, 1, 2, 2, 2], "no palette: grey"); + pcx.push(0x0C); + let mut palette = vec![0u8; 768]; + palette[3..9].copy_from_slice(&[9, 8, 7, 6, 5, 4]); + pcx.extend_from_slice(&palette); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [9, 8, 7, 6, 5, 4]); +} + +#[test] +fn bit_plane_pcx_images_read_their_palette() { + // version 2, one plane: black and white + let mut pcx = pcx_header(2, 1, 1, 2, 2, 1); + pcx.extend_from_slice(&[0b1000_0000, 0]); + assert!(from_images(&[("a.pcx".into(), pcx.clone())]).is_ok(), "an old version is still a PCX"); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [255, 255, 255, 0, 0, 0]); + // two planes: colour 3 is at byte 16 + 9 + let mut pcx = pcx_header(5, 1, 2, 1, 1, 1); + pcx[25..28].copy_from_slice(&[1, 2, 3]); + pcx.extend_from_slice(&[0x80, 0x80]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 2, 3]); +} + +#[test] +fn damaged_pcx_headers_and_bodies_are_refused() { + let no_planes = pcx_header(5, 8, 0, 2, 2, 1); + let no_stride = pcx_header(5, 8, 3, 0, 2, 1); + let short_stride = pcx_header(5, 8, 3, 1, 2, 1); + let short_body = pcx_header(5, 8, 3, 2, 2, 1); + for bytes in [no_planes, no_stride, short_stride, short_body] { + assert!(super::pcx("a.pcx", &bytes).is_err()); + } + assert!(super::pcx("a.pcx", &[0x0A, 5, 1, 8]).is_err(), "no header"); +} + /// A synthetic ICC profile: a 128-byte header (size, device class, colour space, `acsp`) and /// some payload. Only the header is read. fn icc_profile(space: &[u8; 4]) -> Vec { From dc5cb122db4e79e795e92073340262f5c40d1344 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:23:18 +0200 Subject: [PATCH 75/85] trust certificate key type when CNG algorithm group is unreadable (#6391) * trust certificate key type when CNG algorithm group is unreadable When the algorithm group can't be read from the key store due to a provider bug, the certificate should not be rejected: its SubjectPublicKeyInfo already states the key type, and a real mismatch will surface at signing time instead of at enumeration. Change matches_key to fall back to Ok(()) when key.algorithm_group() returns an error, trusting the certificate's key type; extract the group comparison into key_group_matches(group: Option<&AlgorithmGroup>, want: StoreKey) so the fallback and mismatch cases can be unit tested without a live CNG key. Test: key_group_matches_trusts_when_group_unreadable failed before the change and passes after. * Sign: back to LF line endings, shorter test comment Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/sign/src/windows.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/sign/src/windows.rs b/crates/sign/src/windows.rs index 120fea3dab..61620ad384 100644 --- a/crates/sign/src/windows.rs +++ b/crates/sign/src/windows.rs @@ -178,7 +178,27 @@ fn entries() -> Result { /// Whether the store's key is what the certificate needs (RSA; ECDSA on its curve). fn matches_key(key: &NCryptKey, want: StoreKey) -> Result<(), String> { - let group = key.algorithm_group().map_err(|e| format!("its private key's algorithm couldn't be read: {}", describe(&e)))?; + // Some KSPs (Certisign's desktopID, for one) report NCRYPT_ALGORITHM_GROUP_PROPERTY + // as a character count instead of a byte count, so the group can't be read even though + // the key works. The certificate already names the key type, so an unreadable group is + // trusted: a working key is accepted instead of wrongly rejected. The key size, when it + // can be read, still has to agree with the certificate. + matches_key_from_group(key.algorithm_group().ok(), key, want) +} + +/// Core check, decoupled from the NCryptKey handle so the algorithm group can be injected +/// (e.g. forced to `None` to simulate a KSP that reports the property as a character count +/// instead of a byte count, as Certisign's desktopID KSP does). +fn matches_key_from_group(group: Option, key: &NCryptKey, want: StoreKey) -> Result<(), String> { + let Some(group) = group else { + return match want { + StoreKey::Rsa => Ok(()), + StoreKey::Ecdsa(bits) => match key.bits() { + Ok(actual) if actual != bits => Err(format!("its certificate holds a P-{bits} key but the store's private key has {actual} bits")), + _ => Ok(()), + }, + }; + }; let name = |g: &AlgorithmGroup| match g { AlgorithmGroup::Rsa => "an RSA".to_string(), AlgorithmGroup::Ecdsa => "an ECDSA".to_string(), @@ -272,4 +292,13 @@ mod tests { assert_eq!(describe(&CngError::WindowsError(0x57)), "Windows error 0x00000057"); assert_eq!(describe(&CngError::InvalidHashLength), "Invalid hash length"); } + + #[test] + fn key_group_matches_trusts_when_group_unreadable() { + // Certisign's desktopID KSP reports the algorithm-group property as a character + // count (3 for "RSA") instead of a byte count (8), so `algorithm_group` fails. An + // unreadable group must not make a usable key look unusable. + let key = NCryptKey::new_owned(0); + assert_eq!(matches_key(&key, StoreKey::Rsa), Ok(())); + } } From 7e8551e4bcacef6ddb8cbf76d97d0e5afee7635a Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 01:01:33 +0600 Subject: [PATCH 76/85] Add text: gray or CMYK colour, painted with g or k (#6440) page_add_text and content_update took only RGB, so text added to a CMYK print file painted 0 0 0 rg, rich black on press. They now also take gray (0 to 1) or cmyk ([c, m, y, k]), painted with g or k and kept in /PCAdded as /Gray or /CMYK beside /Color, which holds the matching look. An ink applies only while the colour still shows its look: picking another colour, also in the Format panel, drops it for good. RGB text is written byte for byte as before, and a stored ink that doesn't match its colour is ignored. --- crates/automation/README.md | 5 + crates/automation/src/content.rs | 41 +++++++- crates/automation/src/tools.rs | 10 +- crates/automation/tests/it/automation.rs | 53 ++++++++++ crates/edit/src/added.rs | 2 +- crates/edit/src/added/draw.rs | 14 ++- crates/edit/src/added/store.rs | 24 ++++- crates/edit/src/added/types.rs | 89 ++++++++++++++++- crates/edit/src/lib.rs | 4 +- crates/edit/src/tests.rs | 101 ++++++++++++++++++++ crates/engine/src/lib.rs | 3 +- crates/engine/src/tests.rs | 3 + crates/ui-egui/src/editing/tests.rs | 63 ++++++++++++ crates/ui-egui/src/panels/edit.rs | 6 +- crates/ui-egui/src/panels/format_text_ui.rs | 3 +- parity/acrobat-features.toml | 4 +- 16 files changed, 400 insertions(+), 25 deletions(-) diff --git a/crates/automation/README.md b/crates/automation/README.md index db5b988394..2fbbff8ccb 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -26,6 +26,11 @@ its undo/redo history. `page_add_text`, `content_update` and `text_edit` take per-line `aligns`. An `"inherit"` entry follows the box-level `align`; legacy `null` entries remain accepted with the same meaning. +`page_add_text` and `content_update` take one of `color` (`#RRGGBB` or a name, painted in +DeviceRGB), `gray` (0 black to 1 white, DeviceGray `g`) and `cmyk` (`[c, m, y, k]`, each 0 to 1, +DeviceCMYK `k`), so text added to a print file can be 100 K black. `content_list` reports `gray` or +`cmyk` for such text. + `Content` is `Json(Value)` or `Png { data, width, height }`. Errors are `UnknownTool`, `InvalidArgs` (the call didn't match the schema) or `Failed` (a readable message for the agent). ## Conventions diff --git a/crates/automation/src/content.rs b/crates/automation/src/content.rs index 6b25c31f61..1f26dcc26a 100644 --- a/crates/automation/src/content.rs +++ b/crates/automation/src/content.rs @@ -2,7 +2,7 @@ //! move/resize/retype/restyle it, delete it. Geometry is in points from the top-left of the //! displayed page (y down), like every other tool. -use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign}; +use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign, TextInk}; use serde_json::{Value, json}; use crate::comments::parse_color; @@ -30,6 +30,27 @@ fn rect_arg(a: &Args, key: &str) -> Result> { .transpose() } +const INK_ARGS: &str = "give one of color (#RRGGBB or a name), gray (0 black to 1 white) and cmyk ([c, m, y, k], each 0 to 1)"; + +/// `gray` or `cmyk`: the text painted in DeviceGray or DeviceCMYK rather than RGB. At most one of +/// them and `color`. +fn ink_arg(a: &Args) -> Result> { + if ["color", "gray", "cmyk"].iter().filter(|k| a.get(k).is_some()).count() > 1 { + return Err(bad(INK_ARGS)); + } + let unit = |v: Option, key: &str| v.filter(|v| (0.0..=1.0).contains(v)).ok_or_else(|| bad(format!("{key} takes numbers from 0 to 1"))); + if let Some(g) = a.get("gray") { + return Ok(Some(TextInk::Gray(unit(g.as_f64(), "gray")?))); + } + let Some(c) = a.get("cmyk") else { return Ok(None) }; + let parts = c.as_array().filter(|p| p.len() == 4).ok_or_else(|| bad("cmyk must be [c, m, y, k]"))?; + let mut cmyk = [0.0; 4]; + for (out, v) in cmyk.iter_mut().zip(parts) { + *out = unit(v.as_f64(), "cmyk")?; + } + Ok(Some(TextInk::Cmyk(cmyk))) +} + /// Apply the style arguments to `t`. fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(s) = a.opt_num("size")? { @@ -49,8 +70,12 @@ fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(i) = a.opt_bool("italic")? { t.italic = i; } - if let Some(c) = a.opt_str("color")? { - t.color = parse_color(c)?; + if let Some(ink) = ink_arg(a)? { + t.set_ink(Some(ink)); + } else if let Some(c) = a.opt_str("color")? { + // An RGB colour paints in RGB, even one that is the ink's look. + t.set_ink(None); + t.set_color(parse_color(c)?); } if let Some(al) = a.opt_str("align")? { t.align = parse_align(al)?; @@ -124,6 +149,12 @@ impl Automation { v["bold"] = json!(t.bold); v["italic"] = json!(t.italic); v["char_spacing"] = json!(t.char_spacing); + // Text painted in a print colour space says so; RGB text has neither. + match t.painted_ink() { + Some(TextInk::Gray(g)) => v["gray"] = json!(g), + Some(TextInk::Cmyk(c)) => v["cmyk"] = json!(c), + None => {} + } } AddedContent::Image(_) => v["type"] = json!("image"), } @@ -205,8 +236,8 @@ impl Automation { if a.get("text").is_some() { return Err(bad("images have no text")); } - if ["font", "size", "bold", "italic", "color", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { - return Err(bad("font, size, bold, italic, color, align, aligns and char_spacing apply to text")); + if ["font", "size", "bold", "italic", "color", "gray", "cmyk", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { + return Err(bad("font, size, bold, italic, color, gray, cmyk, align, aligns and char_spacing apply to text")); } if let Some(deg) = a.opt_int("rotate")? { if deg.rem_euclid(90) != 0 { diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index af5146377a..b0954ede5e 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -718,13 +718,13 @@ pub fn tools() -> Vec { }), &["doc"], )), - t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style.") + t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style; text painted in gray or CMYK also has gray (its level) or cmyk ([c, m, y, k]).") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t( "page_add_text", "Add text to a page", - "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name), align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", + "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name, painted in RGB) or, for print, gray (0 black to 1 white, painted in DeviceGray) or cmyk ([c, m, y, k], each 0 to 1, painted in DeviceCMYK; [0, 0, 0, 1] is 100 K black), one of the three, align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", ) .cmd("edit.text") .with(schema( @@ -735,6 +735,8 @@ pub fn tools() -> Vec { "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, @@ -751,13 +753,15 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "path": { "type": "string" }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 } }), &["doc", "page", "path"], )), - t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") + t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color or gray or cmyk as page_add_text takes them, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") .with(schema( json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "index": { "type": "integer", "minimum": 1 }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "text": { "type": "string" }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 96649c8e80..b2bf8ba25d 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -5214,6 +5214,59 @@ fn adding_content_through_tools() { assert!(matches!(a.call("content_delete", &json!({ "doc": doc, "page": 1, "index": 5 })), Err(ToolError::Failed(_)))); } +/// #6440: `page_add_text` paints text in CMYK or gray for a print file, so its black is 100 K +/// (`0 0 0 1 k`) rather than DeviceRGB; `content_update` restyles it either way. +#[test] +fn added_text_takes_a_cmyk_or_gray_ink_through_tools() { + let dir = workdir("content-ink"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "PROOF", "at": [20, 20], "cmyk": [0, 0, 0, 1] })); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "Note", "at": [20, 80], "gray": 0.5 })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], &items[1]["gray"]), (&json!([0.0, 0.0, 0.0, 1.0]), &json!(0.5))); + // The item streams of the saved file. + let saved_items = |a: &mut Automation, name: &str| -> Vec { + ok(a, "doc_save", json!({ "doc": doc, "path": name })); + let saved = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join(name)).unwrap())).unwrap(); + let page = &pdfcraft_model::pages(&saved)[0]; + let list = saved.resolve(page.dict.get(b"Contents").unwrap()).as_array().cloned().unwrap(); + list.iter() + .filter_map(|c| match &*saved.resolve(c) { + pdfcraft_cos::Object::Stream(s) if s.dict.name(b"PCMark") == Some(&b"Added"[..]) => { + Some(String::from_utf8_lossy(&s.decoded().unwrap()).into_owned()) + } + _ => None, + }) + .collect() + }; + let streams = saved_items(&mut a, "ink.pdf"); + assert!(streams[0].contains("0 0 0 1 k") && !streams[0].contains("rg"), "{}", streams[0]); + assert!(streams[1].contains("0.5 g") && !streams[1].contains("rg"), "{}", streams[1]); + // Restyled: to RGB with color, and back to CMYK. + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 2, "color": "#000000" })); + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 1, "cmyk": [1, 0, 0, 0] })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], items[1].get("gray")), (&json!([1.0, 0.0, 0.0, 0.0]), None)); + let streams = saved_items(&mut a, "restyled.pdf"); + assert!(streams[0].contains("1 0 0 0 k") && streams[1].contains("0 0 0 rg"), "{streams:?}"); + // One colour at a time, four numbers from 0 to 1. + for bad in [ + json!({ "color": "red", "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 1.5 }), + json!({ "cmyk": [0, 0, 1] }), + json!({ "cmyk": [0, 0, 0, 1, 0] }), + json!({ "cmyk": [0, "x", 0, 1] }), + json!({ "cmyk": [0, 0, 0, -0.1] }), + ] { + let mut args = json!({ "doc": doc, "page": 1, "text": "x", "at": [20, 200] }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("page_add_text", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["count"], 2, "nothing refused was added"); +} + /// Edit ▸ Copy and Paste headlessly (#2636): an added text box or image, as it is, onto its own /// page beside it, onto a point, and into another document. #[test] diff --git a/crates/edit/src/added.rs b/crates/edit/src/added.rs index a603a37060..0fc2aa7359 100644 --- a/crates/edit/src/added.rs +++ b/crates/edit/src/added.rs @@ -31,7 +31,7 @@ use layout::Piece; pub(crate) use layout::WRAP_CALLS; pub use layout::{PASTE_STEP, first_undrawable, lines, paste_place, paste_rect, text_rect, text_width}; pub use store::{add_content, delete_content, isolate_content, list_added, scan_markers, update_content}; -pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, MarkerScan}; +pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan}; #[derive(Clone, Debug, PartialEq)] pub enum Content { diff --git a/crates/edit/src/added/draw.rs b/crates/edit/src/added/draw.rs index 9404f939b1..f3f9b55859 100644 --- a/crates/edit/src/added/draw.rs +++ b/crates/edit/src/added/draw.rs @@ -1,7 +1,7 @@ use super::layout::{ Piece, arabic_error, hebrew_error, is_arabic, is_hebrew, is_thai, latin_width, norm, shaped_layout, text_rect, thai_error, wrapped, }; -use super::types::{AddedText, Align, Family}; +use super::types::{AddedText, Align, Family, Ink}; use super::{ Content, Dict, Document, EditError, GlyphError, GlyphOutline, HashMap, HashSet, ObjRef, Object, PdfString, ShapedCluster, ShapedScript, Stream, align_code, literal, matrix_number, n, shaped_glyph, win_ansi, @@ -105,8 +105,7 @@ pub(super) fn draw_laid_shaped( fonts.set(name.clone().into_bytes(), Object::Ref(font)); names.push(name); } - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT {} {} {} rg\n", n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT {}\n", t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -293,8 +292,7 @@ pub(super) fn draw(doc: &mut Document, c: &Content, view: [f64; 6], taken: &Dict } res.set(b"Font".to_vec(), Object::Dict(fonts)); let r = text_rect(t); - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT /{name} {} Tf {} {} {} rg\n", n(t.size), n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT /{name} {} Tf {}\n", n(t.size), t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -355,6 +353,12 @@ pub(super) fn params(c: &Content) -> Dict { d.set(b"Font".to_vec(), Object::name(t.family.base_font(t.bold, t.italic))); d.set(b"Size".to_vec(), Object::Real(t.size)); d.set(b"Color".to_vec(), arr(&t.color)); + // The ink beside its RGB look, which a reader that knows no ink keeps showing. + match t.painted_ink() { + Some(Ink::Gray(g)) => d.set(b"Gray".to_vec(), Object::Real(g)), + Some(Ink::Cmyk(c)) => d.set(b"CMYK".to_vec(), arr(&c)), + None => {} + } d.set(b"Align".to_vec(), Object::Int(align_code(t.align))); // Each paragraph's own alignment, or null where it follows /Align. let aligns = crate::paragraph_align::tidy(t.aligns.clone(), t.align); diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index 7dae55596f..186935b9f2 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -1,8 +1,19 @@ use super::draw::{draw, params}; use super::layout::{first_undrawable, is_thai, nums, shaped_arabic, thai_error}; -use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, MarkerScan, TAG}; +use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan, TAG}; use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Object, Stream, align_from, check, contents, page_list, place_tagged}; +/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range, have none: +/// the text paints in its RGB `/Color`. +fn ink(doc: &Document, d: &Dict) -> Option { + let ink = match (d.get(b"Gray").map(|g| doc.resolve(g)).and_then(|g| g.as_f64()), d.get(b"CMYK")) { + (Some(g), _) => Ink::Gray(g), + (None, Some(c)) => Ink::Cmyk(nums(doc, Some(c)).try_into().ok()?), + (None, None) => return None, + }; + Some(ink).filter(|i| i.valid()) +} + fn parse(doc: &Document, d: &Dict) -> Option { let r = nums(doc, d.get(b"Rect")); let rect: [f64; 4] = r.try_into().ok()?; @@ -21,7 +32,7 @@ fn parse(doc: &Document, d: &Dict) -> Option { .map(|a| a.iter().take(crate::paragraph_align::count(&text)).map(|v| v.as_int().map(|c| align_from(Some(c)))).collect()) }) .unwrap_or_default(); - Some(Content::Text(AddedText { + let mut t = AddedText { rect, text, family, @@ -29,11 +40,15 @@ fn parse(doc: &Document, d: &Dict) -> Option { italic, size: d.get(b"Size").and_then(Object::as_f64).unwrap_or(12.0), color: if c.len() == 3 { [c[0], c[1], c[2]] } else { [0.0; 3] }, + ink: ink(doc, d), align, aligns: crate::paragraph_align::tidy(aligns, align), // Items written before character spacing have none. char_spacing: d.get(b"CharSpacing").and_then(Object::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0), - })) + }; + // An ink whose look isn't the stored colour (a file edited by hand) is none. + t.drop_stale_ink(); + Some(Content::Text(t)) } b"Image" => { let crop = nums(doc, d.get(b"Crop")); @@ -138,6 +153,9 @@ fn validate(c: &Content) -> Result<(), EditError> { "the character spacing must be between -{CHAR_SPACING_LIMIT} and {CHAR_SPACING_LIMIT} points" ))); } + if t.ink.is_some_and(|i| !i.valid()) { + return Err(EditError::Invalid("a gray level and each CMYK value must be from 0 to 1".into())); + } if (r[2] - r[0]).abs() < 1.0 { return Err(EditError::Invalid("the text box is too narrow".into())); } diff --git a/crates/edit/src/added/types.rs b/crates/edit/src/added/types.rs index 0caaaa863b..43480774a3 100644 --- a/crates/edit/src/added/types.rs +++ b/crates/edit/src/added/types.rs @@ -1,5 +1,5 @@ use super::layout::norm; -use super::{Content, MarkKind, ObjRef, Std14}; +use super::{Content, MarkKind, ObjRef, Std14, n}; pub(super) const TAG: &str = "Added"; @@ -90,6 +90,50 @@ pub enum Align { Justify, } +/// A device colour added text is painted in instead of RGB, for print files: a text's black in +/// CMYK stays 100 K rather than a DeviceRGB black the printer separates into rich black. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Ink { + /// A DeviceGray level, from 0 (black) to 1 (white): painted with `g`. + Gray(f64), + /// DeviceCMYK `[c, m, y, k]`, each from 0 to 1 (`[0, 0, 0, 1]` is 100 K): painted with `k`. + Cmyk([f64; 4]), +} + +impl Ink { + /// Whether every component is a number from 0 to 1. + pub fn valid(self) -> bool { + self.components().iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) + } + + fn components(&self) -> &[f64] { + match self { + Ink::Gray(g) => std::slice::from_ref(g), + Ink::Cmyk(c) => c, + } + } + + /// How the ink looks on screen: gray is the same level in each of red, green and blue + /// (ISO 32000-2 §10.4.2.2), CMYK takes each colour plus black from 1 (§10.4.2.5). + pub fn rgb(self) -> [f64; 3] { + match self { + Ink::Gray(g) => [g.clamp(0.0, 1.0); 3], + Ink::Cmyk([c, m, y, k]) => [c, m, y].map(|v| 1.0 - (v + k).clamp(0.0, 1.0)), + } + } + + /// The operator that makes this ink the fill colour, without a line end. + pub(super) fn fill_op(self) -> String { + match self { + Ink::Gray(g) => format!("{} g", n(g.clamp(0.0, 1.0))), + Ink::Cmyk(c) => { + let [c, m, y, k] = c.map(|v| v.clamp(0.0, 1.0)); + format!("{} {} {} {} k", n(c), n(m), n(y), n(k)) + } + } + } +} + #[derive(Clone, Debug, PartialEq)] pub struct AddedText { /// The text box in display space; its width wraps the text and its top is the first line's @@ -100,7 +144,14 @@ pub struct AddedText { pub bold: bool, pub italic: bool, pub size: f64, + /// The RGB colour: what the text is painted in, unless it has an [`AddedText::ink`] this is + /// the look of. pub color: [f64; 3], + /// A gray or CMYK ink the text is painted in, with `color` its look ([`Ink::rgb`]). Set it + /// with [`AddedText::set_ink`] and the colour with [`AddedText::set_color`], which drops an + /// ink the colour no longer shows; one left behind by setting `color` directly paints nothing + /// ([`AddedText::painted_ink`]) until [`AddedText::drop_stale_ink`] forgets it. + pub ink: Option, pub align: Align, /// Extra space after every character, in points: the PDF's character spacing (`Tc`). pub char_spacing: f64, @@ -114,6 +165,41 @@ impl AddedText { pub fn paragraph_align(&self, i: usize) -> Align { crate::paragraph_align::align_of(&self.aligns, i, self.align) } + + /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. + pub fn set_ink(&mut self, ink: Option) { + if let Some(i) = ink { + self.color = i.rgb(); + } + self.ink = ink; + } + + /// Paint the text in RGB `rgb`. Its ink stays only when `rgb` is the ink's own look; once + /// gone it stays gone, so picking that look again later paints in RGB. + pub fn set_color(&mut self, rgb: [f64; 3]) { + self.color = rgb; + self.drop_stale_ink(); + } + + /// Forget an ink `color` no longer shows, so two styles that paint the same compare equal. + pub fn drop_stale_ink(&mut self) { + self.ink = self.painted_ink(); + } + + /// The ink the text is painted in, or `None` for RGB: the one it has while `color` is still + /// its look. + pub fn painted_ink(&self) -> Option { + // Within rounding: a look read back from a file, or from a colour picker, is the same. + self.ink.filter(|i| i.rgb().iter().zip(self.color).all(|(a, b)| (a - b).abs() < 1e-6)) + } + + /// The operator that sets the text's fill colour, without a line end. + pub(super) fn fill_op(&self) -> String { + match self.painted_ink() { + Some(ink) => ink.fill_op(), + None => crate::marks::rgb(self.color), + } + } } impl Default for AddedText { @@ -126,6 +212,7 @@ impl Default for AddedText { italic: false, size: 12.0, color: [0.0; 3], + ink: None, align: Align::Left, aligns: Vec::new(), char_spacing: 0.0, diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index 2a0add8d12..0cf1886afd 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -61,8 +61,8 @@ pub use text::{ }; pub mod added; pub use added::{ - Added, AddedImage, AddedText, Align, Content, Family, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, list_added, - scan_markers, update_content, + Added, AddedImage, AddedText, Align, Content, Family, Ink, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, + list_added, scan_markers, update_content, }; mod apply; diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 22fbd6e8ae..8aadcef420 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1048,6 +1048,107 @@ fn added_text_and_images_are_page_content_that_stays_editable() { assert!(marks_present(&doc).is_empty()); } +/// #6440: added text in a print file can be gray or CMYK, painted with `g` / `k`, so its black +/// stays 100 K. The ink survives save and reopen; another colour drops it for good. +#[test] +fn gray_and_cmyk_added_text_writes_g_and_k_until_it_is_recoloured() { + use crate::added::Ink; + let item_stream = |doc: &Document, i: usize| { + let obj = list_added(doc)[i].obj; + let Object::Stream(s) = &*doc.get(obj) else { panic!("an item is a stream") }; + String::from_utf8_lossy(&s.decoded().unwrap()).into_owned() + }; + let mut doc = fixture(); + let base = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + let mut black = base.clone(); + black.set_ink(Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(black.color, [0.0; 3], "100 K looks black on screen"); + let mut gray = AddedText { rect: [72.0, 400.0, 300.0, 500.0], ..base.clone() }; + gray.set_ink(Some(Ink::Gray(0.25))); + add_content(&mut doc, 0, &Content::Text(black)).unwrap(); + add_content(&mut doc, 0, &Content::Text(gray)).unwrap(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 200.0, 300.0, 300.0], ..base.clone() })).unwrap(); + + let doc = reopen(&doc); + let (k, g, rgb) = (item_stream(&doc, 0), item_stream(&doc, 1), item_stream(&doc, 2)); + assert!(k.contains("/PCFHelvetica 14 Tf 0 0 0 1 k\n") && !k.contains("rg"), "{k}"); + assert!(g.contains("/PCFHelvetica 14 Tf 0.25 g\n") && !g.contains("rg"), "{g}"); + assert!(rgb.contains("/PCFHelvetica 14 Tf 0 0 0 rg\n"), "RGB text paints as before: {rgb}"); + let inks: Vec> = list_added(&doc).iter().map(|a| if let Content::Text(t) = &a.content { t.painted_ink() } else { None }).collect(); + assert_eq!(inks, [Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), Some(Ink::Gray(0.25)), None]); + + // Another colour drops the ink, and picking the ink's look again later doesn't bring it back; + // picking that look while the ink is there keeps it. + let Content::Text(cmyk) = list_added(&doc)[0].content.clone() else { panic!() }; + let mut same = cmyk.clone(); + same.set_color([0.0; 3]); + assert_eq!(same, cmyk, "black on 100 K stays 100 K"); + let mut recoloured = cmyk.clone(); + recoloured.set_color([1.0, 0.0, 0.0]); + assert_eq!(recoloured.ink, None); + recoloured.set_color([0.0; 3]); + assert_eq!((recoloured.ink, recoloured.painted_ink()), (None, None), "the ink stays gone"); + + // Set directly, a colour paints in RGB too, and the item reads back without the ink: what a + // draft with its stale ink dropped compares equal to. + let mut doc = doc; + let red = AddedText { color: [1.0, 0.0, 0.0], ..cmyk }; + update_content(&mut doc, 0, 0, &Content::Text(red.clone())).unwrap(); + let stream = item_stream(&doc, 0); + assert!(stream.contains("1 0 0 rg") && !stream.contains(" k\n"), "{stream}"); + let mut settled = red; + settled.drop_stale_ink(); + for doc in [&doc, &reopen(&doc)] { + let Content::Text(back) = list_added(doc)[0].content.clone() else { panic!() }; + // The stored box is the text's own height. + assert_eq!(back, AddedText { rect: back.rect, ..settled.clone() }); + } + + // CMYK looks as ISO 32000-2 §10.4.2.5 maps it to RGB; an ink out of range is refused. + assert_eq!(Ink::Cmyk([0.5, 0.0, 1.0, 0.25]).rgb(), [0.25, 0.75, 0.0]); + let mut wrong = base; + wrong.ink = Some(Ink::Gray(1.5)); + assert!(add_content(&mut doc, 0, &Content::Text(wrong)).is_err()); +} + +/// #6440: a file edited by hand paints its added text in RGB unless the stored gray or CMYK is +/// in range, the right length and still the look of `/Color`; nothing in it panics. +#[test] +fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { + use crate::added::Ink; + let mut doc = fixture(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); + let obj = list_added(&doc)[0].obj; + let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); + let read = |entries: &[(&[u8], Object)]| { + let mut edited = doc.clone(); + let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; + let mut params = s.dict.get(b"PCAdded").and_then(|p| p.as_dict().cloned()).unwrap(); + for (key, value) in entries { + params.set(key.to_vec(), value.clone()); + } + s.dict.set(b"PCAdded".to_vec(), Object::Dict(params)); + edited.set(obj, Object::Stream(s)); + let Content::Text(t) = list_added(&edited)[0].content.clone() else { panic!() }; + t.ink + }; + // As PDFCraft writes it: the ink stands. + assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + for (what, entries) in [ + ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), + ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), + ("five CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0, 0.0]))]), + ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), + ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), + ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), + ("gray NaN", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::Real(f64::NAN))]), + ("CMYK infinite", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, f64::INFINITY]))]), + ] { + assert_eq!(read(&entries), None, "{what}"); + } +} + /// #4464: a page left with one added item draws that item and nothing else. #[test] fn an_isolated_item_is_all_that_is_left_on_its_page() { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 1c113d0d82..f99d8ff816 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -91,7 +91,8 @@ pub use pdfcraft_edit::added::{ }; pub use pdfcraft_edit::paragraph_align; pub use pdfcraft_edit::{ - Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, MarkKind, Watermark, + Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, Ink as TextInk, + MarkKind, Watermark, }; pub use pdfcraft_forms::{ BorderStyle, CalculatedField, CalculationIssue, CalculationOrder, CheckStyle, Field as FormField, FieldAction, FieldChange, FieldFont, diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 5b6fb01b9e..c109a0bc80 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -4182,6 +4182,7 @@ fn pasted_added_content_keeps_its_style_and_picture() { italic: true, size: 12.0, color: [0.8, 0.1, 0.2], + ink: None, align: TextAlign::Center, aligns: Vec::new(), char_spacing: 0.0, @@ -8112,6 +8113,7 @@ fn content_edit_keeps_pages_annotations_fields_and_links() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, @@ -8153,6 +8155,7 @@ fn content_refresh_lists_match_the_storages_an_edit_touches() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, diff --git a/crates/ui-egui/src/editing/tests.rs b/crates/ui-egui/src/editing/tests.rs index 76886ff54f..c0e35329c0 100644 --- a/crates/ui-egui/src/editing/tests.rs +++ b/crates/ui-egui/src/editing/tests.rs @@ -391,3 +391,66 @@ fn save_asks_where_to_keep_an_email_attachment_and_leaves_the_temporary_copy_alo assert!(app.save_active(SaveTarget::InPlace)); let _ = std::fs::remove_dir_all(&dir); } + +/// #6440: recolouring gray or CMYK text in its open box drops the ink at once. Finishing the box +/// then adds no empty Undo step (the draft equals the item the page reads back), picking the +/// ink's own look again in the same box paints RGB rather than bringing the ink back, and new +/// boxes don't take the ink. +#[test] +fn recolouring_print_ink_text_in_its_box_drops_the_ink_without_an_empty_step() { + // 100 K text, open in its box. + let open = || { + let mut app = crate::PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("a.pdf", None, one_page_pdf()).unwrap(); + let mut text = pdfcraft_engine::AddedText { rect: [20.0, 200.0, 180.0, 220.0], text: "Proof".into(), ..Default::default() }; + text.set_ink(Some(pdfcraft_engine::TextInk::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert!(app.apply_edit(pdfcraft_engine::Edit::AddText { page: 0, text })); + let stored = item(&app); + assert!(stored.painted_ink().is_some()); + app.views[0].content.draft = Some(crate::content_ui::TextDraft { + page: 0, + index: Some(0), + rect: stored.rect, + text: stored.text.clone(), + style: stored, + focus: true, + selection: None, + typing_from: None, + }); + app + }; + fn item(app: &crate::PdfCraftApp) -> pdfcraft_engine::AddedText { + match &app.session.get(app.views[0].id).unwrap().added()[0].content { + pdfcraft_engine::AddedContent::Text(t) => t.clone(), + other => panic!("{other:?}"), + } + } + // The box's style with another colour, set as the Format panel hands it over. + let recolour = |app: &mut crate::PdfCraftApp, color: [f64; 3]| { + let style = app.views[0].content.draft.as_ref().map(|d| d.style.clone()).unwrap(); + app.restyle_draft(0, pdfcraft_engine::AddedText { color, ..style }); + }; + let finish = |app: &mut crate::PdfCraftApp| { + let added = app.session.get(app.views[0].id).unwrap().added().to_vec(); + app.views[0].content.finish_draft(&added) + }; + + // Red, then Done. + let mut app = open(); + recolour(&mut app, [1.0, 0.0, 0.0]); + assert_eq!(item(&app).ink, None, "the page paints red in RGB"); + let finished = finish(&mut app); + assert!(finished.is_none(), "an empty Undo step: {finished:?}"); + assert_eq!(app.text_style.ink, None, "new boxes don't take the ink"); + + // Red, then black in the same box: RGB black, not the 100 K back. + let mut app = open(); + recolour(&mut app, [1.0, 0.0, 0.0]); + recolour(&mut app, [0.0; 3]); + let now = item(&app); + assert_eq!((now.ink, now.color), (None, [0.0; 3])); + assert_eq!(app.text_style.ink, None); + let finished = finish(&mut app); + assert!(finished.is_none(), "an empty Undo step: {finished:?}"); +} diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs index 2c2fb7a09a..883009df8e 100644 --- a/crates/ui-egui/src/panels/edit.rs +++ b/crates/ui-egui/src/panels/edit.rs @@ -406,7 +406,11 @@ impl PdfCraftApp { /// style at once and stays open. Text already on the page is updated with what has been typed, /// as one undoable change; new text waits until it is finished. Text typed after this styling /// starts from it too. - pub(crate) fn restyle_draft(&mut self, index: usize, style: pdfcraft_engine::AddedText) { + pub(crate) fn restyle_draft(&mut self, index: usize, mut style: pdfcraft_engine::AddedText) { + // A gray or CMYK ink the colour no longer shows goes now (#6440): kept, the draft would + // differ from the item the page reads back without it, and a later pick of the ink's + // look would bring it back. + style.drop_stale_ink(); let Some(view) = self.views.get_mut(index) else { return }; // Text the standard fonts can't draw stays in the box, unwritten, until it is corrected (#125). let blocked = view.content.blocked().is_some(); diff --git a/crates/ui-egui/src/panels/format_text_ui.rs b/crates/ui-egui/src/panels/format_text_ui.rs index 99ffc054e1..412563b026 100644 --- a/crates/ui-egui/src/panels/format_text_ui.rs +++ b/crates/ui-egui/src/panels/format_text_ui.rs @@ -266,7 +266,8 @@ pub(crate) fn panel( let swatch = colour_button(ui, t, look.color); egui::Popup::menu(&swatch).close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside).show(|ui| { if let Some(picked) = crate::comments::swatch_grid(ui, Some(look.color)) { - look.color = picked; + // Drops a gray or CMYK ink the new colour isn't the look of (#6440). + look.set_color(picked); ui.close(); } }); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 10b040e09f..bd7545175d 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -2047,8 +2047,8 @@ milestone = "M7" status = "shipped" commands = ["edit.text"] tools = ["page_add_text", "content_update"] -evidence = ["crates/ui-egui/tests/it/content.rs::typing_moving_styling_and_deleting_added_content", "crates/ui-egui/tests/it/content.rs::clicking_elsewhere_keeps_the_text_and_starts_a_new_box", "crates/automation/tests/it/automation.rs::adding_content_through_tools", "crates/edit/src/tests.rs::added_text_and_images_are_page_content_that_stays_editable", "crates/engine/src/tests.rs::added_text_and_images_render_and_stay_editable", "crates/edit/src/tests.rs::added_arabic_text_is_shaped_in_display_order_and_stays_searchable", "crates/automation/tests/it/automation.rs::thai_text_through_tools_saves_reopens_extracts_and_renders", "crates/ui-egui/tests/it/content.rs::typing_thai_text_keeps_marks_and_commits_to_page", "crates/automation/tests/it/automation.rs::added_arabic_text_keeps_copy_search_and_editor_order_after_retyping_and_reopen", "crates/edit/src/tests.rs::added_hebrew_text_is_drawn_right_to_left_and_stays_searchable", "crates/automation/tests/it/automation.rs::hebrew_text_through_tools_saves_reopens_extracts_and_renders"] -notes = "page content, editable later (retype, move, restyle); standard fonts, Windows-1252 text. Arabic text is shaped with the craft-fonts Arab face (Type 3 outlines with ToUnicode) and laid out in bidirectional display order; per-line ActualText keeps copy, search and editor text in typing order through retyping and save/reopen; it is drawn regular whatever the bold/italic setting, and typing in the box is still left to right (misc.rtl-ui). Hebrew (#128) takes the same path with the craft-fonts Hebr face (Noto Sans Hebrew): right-to-left display order, points and cantillation kept on their letter as one glyph, ToUnicode and per-line ActualText; regular style only, and right-to-left scripts other than Arabic and Hebrew are not covered. Thai Add Text uses the craft-fonts Thai face, with shaped clusters and ToUnicode; Thai is regular style, wrapping preserves clusters but is not dictionary-based, and forms/Fill and Sign are not covered by this change. Each click on the page keeps the box being typed and starts a new one; Done and Discard beside the box" +evidence = ["crates/ui-egui/tests/it/content.rs::typing_moving_styling_and_deleting_added_content", "crates/ui-egui/tests/it/content.rs::clicking_elsewhere_keeps_the_text_and_starts_a_new_box", "crates/automation/tests/it/automation.rs::adding_content_through_tools", "crates/edit/src/tests.rs::added_text_and_images_are_page_content_that_stays_editable", "crates/engine/src/tests.rs::added_text_and_images_render_and_stay_editable", "crates/edit/src/tests.rs::added_arabic_text_is_shaped_in_display_order_and_stays_searchable", "crates/automation/tests/it/automation.rs::thai_text_through_tools_saves_reopens_extracts_and_renders", "crates/ui-egui/tests/it/content.rs::typing_thai_text_keeps_marks_and_commits_to_page", "crates/automation/tests/it/automation.rs::added_arabic_text_keeps_copy_search_and_editor_order_after_retyping_and_reopen", "crates/edit/src/tests.rs::added_hebrew_text_is_drawn_right_to_left_and_stays_searchable", "crates/automation/tests/it/automation.rs::hebrew_text_through_tools_saves_reopens_extracts_and_renders", "crates/edit/src/tests.rs::gray_and_cmyk_added_text_writes_g_and_k_until_it_is_recoloured", "crates/automation/tests/it/automation.rs::added_text_takes_a_cmyk_or_gray_ink_through_tools", "crates/ui-egui/src/editing/tests.rs::recolouring_print_ink_text_in_its_box_drops_the_ink_without_an_empty_step"] +notes = "page content, editable later (retype, move, restyle); standard fonts, Windows-1252 text. The colour is RGB, or for print files gray or CMYK (#6440: page_add_text / content_update gray or cmyk, painted with g / k so black can be 100 K, kept through save and reopen; choosing another colour later, also from the Format panel, drops the ink for good and paints in RGB, while choosing the colour the ink already shows (black on 100 K) keeps it; content_update color always paints in RGB). Arabic text is shaped with the craft-fonts Arab face (Type 3 outlines with ToUnicode) and laid out in bidirectional display order; per-line ActualText keeps copy, search and editor text in typing order through retyping and save/reopen; it is drawn regular whatever the bold/italic setting, and typing in the box is still left to right (misc.rtl-ui). Hebrew (#128) takes the same path with the craft-fonts Hebr face (Noto Sans Hebrew): right-to-left display order, points and cantillation kept on their letter as one glyph, ToUnicode and per-line ActualText; regular style only, and right-to-left scripts other than Arabic and Hebrew are not covered. Thai Add Text uses the craft-fonts Thai face, with shaped clusters and ToUnicode; Thai is regular style, wrapping preserves clusters but is not dictionary-based, and forms/Fill and Sign are not covered by this change. Each click on the page keeps the box being typed and starts a new one; Done and Discard beside the box" [[feature]] id = "edit.find-replace" From c44896df952116be25208d0be3d98ea22889fbcd Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 02:08:46 +0600 Subject: [PATCH 77/85] Add text: an ink is read only from numbers and kept as painted (review) A stored /CMYK with a non-numeric entry was read from its numbers alone, so [0 /Bad 0 0 1] became 100 K; a /Gray that isn't a number with a /CMYK beside it fell through to the CMYK. Both now give no ink, and the text paints in its /Color. g and k are written to the thousandth, as rg is, so an ink set or read with finer digits painted other than what the item, its stored /Gray or /CMYK and content_list held. The ink is now kept as painted, with its look, when set and when read back. RGB text is unchanged. --- crates/edit/src/added/store.rs | 19 ++++++++---- crates/edit/src/added/types.rs | 15 +++++++++- crates/edit/src/tests.rs | 53 ++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index 186935b9f2..df0b697ff9 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -3,13 +3,16 @@ use super::layout::{first_undrawable, is_thai, nums, shaped_arabic, thai_error}; use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan, TAG}; use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Object, Stream, align_from, check, contents, page_list, place_tagged}; -/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range, have none: -/// the text paints in its RGB `/Color`. +/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range or not +/// made only of numbers (a `/Gray` name, a `/CMYK` entry that isn't a number), have none: the +/// text paints in its RGB `/Color`. fn ink(doc: &Document, d: &Dict) -> Option { - let ink = match (d.get(b"Gray").map(|g| doc.resolve(g)).and_then(|g| g.as_f64()), d.get(b"CMYK")) { - (Some(g), _) => Ink::Gray(g), - (None, Some(c)) => Ink::Cmyk(nums(doc, Some(c)).try_into().ok()?), - (None, None) => return None, + let ink = if let Some(g) = d.get(b"Gray") { + Ink::Gray(doc.resolve(g).as_f64()?) + } else { + let cmyk = doc.resolve(d.get(b"CMYK")?); + let parts: Vec = cmyk.as_array()?.iter().map(Object::as_f64).collect::>()?; + Ink::Cmyk(parts.try_into().ok()?) }; Some(ink).filter(|i| i.valid()) } @@ -48,6 +51,10 @@ fn parse(doc: &Document, d: &Dict) -> Option { }; // An ink whose look isn't the stored colour (a file edited by hand) is none. t.drop_stale_ink(); + // One that stands is the ink as painted, finer digits dropped, and its look. + if let Some(i) = t.ink { + t.set_ink(Some(i)); + } Some(Content::Text(t)) } b"Image" => { diff --git a/crates/edit/src/added/types.rs b/crates/edit/src/added/types.rs index 43480774a3..24101b2eea 100644 --- a/crates/edit/src/added/types.rs +++ b/crates/edit/src/added/types.rs @@ -106,6 +106,16 @@ impl Ink { self.components().iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) } + /// The ink as its operator paints it: each component to the thousandth that `g` and `k` are + /// written with (as RGB's `rg`), so the item, what it reads back as and the page agree. + pub fn painted(self) -> Ink { + let round = |v: f64| (v * 1000.0).round() / 1000.0; + match self { + Ink::Gray(g) => Ink::Gray(round(g)), + Ink::Cmyk(c) => Ink::Cmyk(c.map(round)), + } + } + fn components(&self) -> &[f64] { match self { Ink::Gray(g) => std::slice::from_ref(g), @@ -166,8 +176,11 @@ impl AddedText { crate::paragraph_align::align_of(&self.aligns, i, self.align) } - /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. + /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. An + /// ink in range is kept as painted ([`Ink::painted`]); one out of range stays as given, for + /// saving to refuse. pub fn set_ink(&mut self, ink: Option) { + let ink = ink.map(|i| if i.valid() { i.painted() } else { i }); if let Some(i) = ink { self.color = i.rgb(); } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 8aadcef420..1edcb65c25 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1120,7 +1120,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); let obj = list_added(&doc)[0].obj; let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); - let read = |entries: &[(&[u8], Object)]| { + let read_text = |entries: &[(&[u8], Object)]| { let mut edited = doc.clone(); let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; let mut params = s.dict.get(b"PCAdded").and_then(|p| p.as_dict().cloned()).unwrap(); @@ -1130,11 +1130,15 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { s.dict.set(b"PCAdded".to_vec(), Object::Dict(params)); edited.set(obj, Object::Stream(s)); let Content::Text(t) = list_added(&edited)[0].content.clone() else { panic!() }; - t.ink + t }; + let read = |entries: &[(&[u8], Object)]| read_text(entries).ink; // As PDFCraft writes it: the ink stands. assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + // Finer than it paints, it reads back as painted (`0.5 g`), its look with it. + let fine = read_text(&[(b"Color", real(&[0.50049; 3])), (b"Gray", Object::Real(0.50049))]); + assert_eq!((fine.ink, fine.color), (Some(Ink::Gray(0.5)), [0.5; 3])); for (what, entries) in [ ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), @@ -1142,6 +1146,17 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), + ( + "a CMYK entry not a number", + vec![ + (b"Color", real(&[0.0; 3])), + (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(0.0), Object::Real(1.0)])), + ], + ), + ( + "gray not a number beside a CMYK", + vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black")), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))], + ), ("gray NaN", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::Real(f64::NAN))]), ("CMYK infinite", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, f64::INFINITY]))]), ] { @@ -1149,6 +1164,40 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { } } +/// #6440: an ink finer than the thousandth `g` and `k` are written to is kept as painted, so the +/// item, its stored `/CMYK` and what it reads back as are what the page shows. +#[test] +fn an_ink_is_kept_as_it_is_painted() { + use crate::added::Ink; + let mut doc = fixture(); + let mut t = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + t.set_ink(Some(Ink::Cmyk([0.12345, 0.00049, 0.0, 0.99951]))); + let painted = Ink::Cmyk([0.123, 0.0, 0.0, 1.0]); + assert_eq!((t.ink, t.color), (Some(painted), painted.rgb())); + add_content(&mut doc, 0, &Content::Text(t)).unwrap(); + let doc = reopen(&doc); + let item = &list_added(&doc)[0]; + let Object::Stream(s) = &*doc.get(item.obj) else { panic!("an item is a stream") }; + let stream = String::from_utf8_lossy(&s.decoded().unwrap()).into_owned(); + assert!( + stream.contains( + "/PCFHelvetica 14 Tf 0.123 0 0 1 k +" + ), + "{stream}" + ); + let stored = s + .dict + .get(b"PCAdded") + .and_then(|p| p.as_dict()) + .and_then(|p| p.get(b"CMYK")) + .and_then(Object::as_array) + .map(|a| a.iter().filter_map(Object::as_f64).collect::>()); + assert_eq!(stored, Some(vec![0.123, 0.0, 0.0, 1.0])); + let Content::Text(back) = &item.content else { panic!() }; + assert_eq!(back.painted_ink(), Some(painted)); +} + /// #4464: a page left with one added item draws that item and nothing else. #[test] fn an_isolated_item_is_all_that_is_left_on_its_page() { From 2153da064438b461ded6e5d605a65752d301a4de Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 03:06:01 +0600 Subject: [PATCH 78/85] Add text: a stored ink is read without copying a long array (review) Reading an item's /CMYK resolved the value, which copied a direct array, then collected every entry before checking there were four. A file with a huge numeric /CMYK paid for that on each content listing and item lookup. Now a direct value is read where it is, a reference is followed without a copy, and the array's length is checked before any entry is read. /Gray is read the same way. A long /CMYK, direct or by reference, and a /Gray that is a long array still read as no ink. --- crates/edit/src/added/store.rs | 19 +++++++++++++++---- crates/edit/src/tests.rs | 8 ++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index df0b697ff9..3555910f04 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -8,15 +8,26 @@ use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Ob /// text paints in its RGB `/Color`. fn ink(doc: &Document, d: &Dict) -> Option { let ink = if let Some(g) = d.get(b"Gray") { - Ink::Gray(doc.resolve(g).as_f64()?) + Ink::Gray(read_value(doc, g, Object::as_f64)?) } else { - let cmyk = doc.resolve(d.get(b"CMYK")?); - let parts: Vec = cmyk.as_array()?.iter().map(Object::as_f64).collect::>()?; - Ink::Cmyk(parts.try_into().ok()?) + // Four entries or none: a longer array is never copied or read through. + read_value(doc, d.get(b"CMYK")?, |cmyk| { + let [c, m, y, k] = cmyk.as_array()?.as_slice() else { return None }; + Some(Ink::Cmyk([c.as_f64()?, m.as_f64()?, y.as_f64()?, k.as_f64()?])) + })? }; Some(ink).filter(|i| i.valid()) } +/// `f` of `o`, a reference followed. A direct value is read where it is, never copied, so a +/// large one a file puts there costs nothing to look at. +fn read_value(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> T) -> T { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + direct => f(direct), + } +} + fn parse(doc: &Document, d: &Dict) -> Option { let r = nums(doc, d.get(b"Rect")); let rect: [f64; 4] = r.try_into().ok()?; diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 1edcb65c25..8d9f5c58d2 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1120,6 +1120,10 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); let obj = list_added(&doc)[0].obj; let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); + // A `/CMYK` far longer than four numbers, as a hostile file might store it: an ink first. + let long = real(&[0.0, 0.0, 0.0, 1.0].repeat(50_000)); + let long_ref = Object::Ref(doc.add(long.clone())); + let ink_ref = Object::Ref(doc.add(real(&[0.0, 0.0, 0.0, 1.0]))); let read_text = |entries: &[(&[u8], Object)]| { let mut edited = doc.clone(); let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; @@ -1136,6 +1140,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { // As PDFCraft writes it: the ink stands. assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", ink_ref)]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), "by reference"); // Finer than it paints, it reads back as painted (`0.5 g`), its look with it. let fine = read_text(&[(b"Color", real(&[0.50049; 3])), (b"Gray", Object::Real(0.50049))]); assert_eq!((fine.ink, fine.color), (Some(Ink::Gray(0.5)), [0.5; 3])); @@ -1143,6 +1148,9 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), ("five CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0, 0.0]))]), + ("a long CMYK", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long.clone())]), + ("a long CMYK by reference", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long_ref.clone())]), + ("gray a long array", vec![(b"Color", real(&[0.0; 3])), (b"Gray", long.clone())]), ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), From a9e98e0db6fc428773711dd52c15e513fa592b9a Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:24:29 +0800 Subject: [PATCH 79/85] pdfcraft-web: window.pdfcraft, the page command API from craft-web-api, opt-in with ?control ?control attaches the UI control channel (the one --control serves) and installs window.pdfcraft: request, execute (ui.command), commands (ui.commands), docInspect (ui.state), uiInspect (ui.inspect) and screenshot. The channel keeps the accessibility tree of every frame, so it stays off unless the page asks. Opening, saving, the file name and the unsaved flag stay with the host messages. --- Cargo.lock | 15 +++- apps/pdfcraft-web/Cargo.toml | 5 +- apps/pdfcraft-web/src/api.rs | 127 ++++++++++++++++++++++++++++++++++ apps/pdfcraft-web/src/main.rs | 8 +++ packaging/web/README.md | 12 ++++ 5 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 apps/pdfcraft-web/src/api.rs diff --git a/Cargo.lock b/Cargo.lock index 8617c45522..0d315535a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,10 +1312,21 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-web-api" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +dependencies = [ + "js-sys", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "craft-web-host" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=2e2312ddaf21bc3d85cd3395d0955c7d239306f6#2e2312ddaf21bc3d85cd3395d0955c7d239306f6" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" dependencies = [ "wasm-bindgen", "web-sys", @@ -4754,11 +4765,13 @@ dependencies = [ name = "pdfcraft-web" version = "0.18.1" dependencies = [ + "craft-web-api", "craft-web-host", "eframe", "egui", "js-sys", "pdfcraft-ui-egui", + "serde_json", "wasm-bindgen-futures", "web-sys", ] diff --git a/apps/pdfcraft-web/Cargo.toml b/apps/pdfcraft-web/Cargo.toml index 56e10c0c29..ad5b62229c 100644 --- a/apps/pdfcraft-web/Cargo.toml +++ b/apps/pdfcraft-web/Cargo.toml @@ -12,9 +12,12 @@ workspace = true [dependencies] pdfcraft-ui-egui = { workspace = true } # The host messages (storytold/craft-libs): one contract and one bridge for the web builds. -craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "2e2312ddaf21bc3d85cd3395d0955c7d239306f6", version = "0.1.0" } +# The page command API (storytold/craft-libs#413): window.pdfcraft over the UI control channel. +craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } +craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } eframe = { workspace = true, features = ["default_fonts", "glow", "persistence", "accesskit"] } egui = { workspace = true, features = ["default_fonts", "monochrome_emoji_fonts"] } +serde_json = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" diff --git a/apps/pdfcraft-web/src/api.rs b/apps/pdfcraft-web/src/api.rs new file mode 100644 index 0000000000..379d092f72 --- /dev/null +++ b/apps/pdfcraft-web/src/api.rs @@ -0,0 +1,127 @@ +//! `window.pdfcraft`: the page command API (`craft-web-api`, storytold/craft-libs#413) over the app's +//! UI control channel (`pdfcraft_ui_egui::control`), opt-in with `?control`. +//! +//! The crate owns the JavaScript side (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, +//! `screenshot`, as promises with a timeout). This module is PdfCraft's part: a [`Request`] that +//! sends each call through the app's [`ControlClient`] and waits for the reply, and the table that +//! names PdfCraft's methods: its control channel drives the UI, so `commands` is `ui.commands`, +//! `execute` runs a registry command with `ui.command`, `docInspect` is `ui.state` and `uiInspect` +//! is `ui.inspect`. +//! +//! The channel keeps the accessibility tree of every frame, so it is off unless the page asks for it +//! (`?control`), as FigCraft's is. PdfCraft's control requests cannot be withdrawn, so a call the page +//! gave up on still runs when the app reaches it, as it does for a desktop client that timed out. + +use serde_json::{Value, json}; + +/// The reply of a control request as the protocol's `{ok, result}` / `{ok: false, error}` object. +fn reply_object(reply: Result) -> Value { + match reply { + Ok(result) => json!({"ok": true, "result": result}), + Err(error) => json!({"ok": false, "error": error}), + } +} + +/// The params of `ui.command`. `execute(command, params)` sends `{command, params}`; `ui.command` +/// takes `{id}` (and `params` for `ui.panel`). A request that already names an `id` is the page's own +/// `request("ui.command", {id})` and goes through unchanged. +fn command_params(params: Value) -> Value { + match params.get("command").and_then(Value::as_str) { + Some(command) if params.get("id").is_none() => json!({"id": command, "params": params.get("params").cloned().unwrap_or(Value::Null)}), + _ => params, + } +} + +#[cfg(target_arch = "wasm32")] +pub use web::install; + +#[cfg(target_arch = "wasm32")] +mod web { + use std::rc::Rc; + use std::sync::mpsc::{Receiver, TryRecvError}; + + use craft_web_api::methods::Methods; + use craft_web_api::page::{PageApi, Request}; + use pdfcraft_ui_egui::control::{ControlClient, Reply}; + use serde_json::{Value, json}; + use wasm_bindgen_futures::JsFuture; + + use super::{command_params, reply_object}; + + /// The app id in `window.`. + const APP: &str = "pdfcraft"; + + /// How often a waiting call looks for the app's reply, in milliseconds (about a frame). + const POLL_MS: i32 = 16; + + /// PdfCraft's control methods behind each function. + fn methods() -> Methods { + Methods { + commands: Some("ui.commands"), + execute: Some("ui.command"), + doc_inspect: Some("ui.state"), + ui_inspect: Some("ui.inspect"), + screenshot: Some("ui.screenshot"), + } + } + + /// Put `window.pdfcraft` on the page, sending its calls through `client`. An existing + /// `window.pdfcraft` keeps the functions it already has. + pub fn install(client: ControlClient) { + let request: Request = Rc::new(move |method, params| { + let params = if method == "ui.command" { command_params(params) } else { params }; + Box::pin(wait(client.send(&method, params))) + }); + if let Err(e) = Rc::new(PageApi::new(APP, methods(), request)).install() { + eframe::web_sys::console::warn_1(&format!("PdfCraft: couldn't install window.{APP}: {e:?}").into()); + } + } + + /// The app's reply, polled about once a frame. + async fn wait(reply: Receiver) -> Value { + loop { + match reply.try_recv() { + Ok(reply) => return reply_object(reply), + Err(TryRecvError::Empty) => sleep(POLL_MS).await, + Err(TryRecvError::Disconnected) => return json!({"ok": false, "error": "the app dropped the request"}), + } + } + } + + async fn sleep(ms: i32) { + let promise = js_sys::Promise::new(&mut |resolve, _| { + if let Some(window) = web_sys::window() { + let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms); + } + }); + let _ = JsFuture::from(promise).await; + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn execute_becomes_a_ui_command_by_id() { + assert_eq!(command_params(json!({"command": "file.word_count", "params": {"a": 1}})), json!({"id": "file.word_count", "params": {"a": 1}})); + assert_eq!(command_params(json!({"command": "file.save", "params": {}})), json!({"id": "file.save", "params": {}})); + } + + #[test] + fn a_request_that_names_an_id_goes_through_unchanged() { + let own = json!({"id": "file.save"}); + assert_eq!(command_params(own.clone()), own); + let both = json!({"id": "ui.panel", "command": "x", "params": {"reset": true}}); + assert_eq!(command_params(both.clone()), both, "an id wins over a command"); + assert_eq!(command_params(json!({})), json!({})); + } + + #[test] + fn replies_become_the_protocols_objects() { + assert_eq!(reply_object(Ok(json!({"n": 1}))), json!({"ok": true, "result": {"n": 1}})); + assert_eq!(reply_object(Err("boom".to_string())), json!({"ok": false, "error": "boom"})); + } +} diff --git a/apps/pdfcraft-web/src/main.rs b/apps/pdfcraft-web/src/main.rs index ffa5a06068..d23dd15a57 100644 --- a/apps/pdfcraft-web/src/main.rs +++ b/apps/pdfcraft-web/src/main.rs @@ -3,6 +3,9 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(any(target_arch = "wasm32", test))] +mod api; + #[cfg(target_arch = "wasm32")] fn main() { use pdfcraft_ui_egui::PdfCraftApp; @@ -68,6 +71,11 @@ fn main() { &"PdfCraft: ?host=parent needs a framing page on the same origin; saves download instead".into(), ); } + // `?control`: `window.pdfcraft`, the page's command API (`api`). Opt-in: the control + // channel keeps the accessibility tree of every frame. + if query_param("control").is_some() { + api::install(app.attach_control(&cc.egui_ctx)); + } Ok(Box::new(app)) }), ) diff --git a/packaging/web/README.md b/packaging/web/README.md index 3149dc75d5..a1fe64c952 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -136,6 +136,18 @@ The messages and both transports are the shared contract of [`craft-web-host`](h - **Files from the host are untrusted input.** An opened file goes through the same reading as one picked with File ▸ Open or dropped on the window, with the same format limits; before it is copied into the app's memory, its size is checked against the host open limit (512 MiB) and a larger file is refused, and a name is cut to 255 characters. - **The event and message names are the interface.** `pdfcraft:ready`, `pdfcraft:open`, `pdfcraft:save`, `pdfcraft:dirty` and `pdfcraft:failed`, and the fields they carry (the same as the `?host=parent` messages), stay stable; changing one is a breaking change. +### Driving it from the page (`window.pdfcraft`) + +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app still runs it when it reaches it, as it does for a desktop client that timed out. + +```js +const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog +await pdfcraft.request("ui.set", { key: "show_page_edge_shadows", value: "false" }); +const { data } = await pdfcraft.screenshot(); // the window as PNG (base64) +``` + +Opening a PDF, handing a saved or exported file to the page, the file name and the unsaved flag stay with the host messages above. + ## Renderer selection and fallback flags PdfCraft renders with wgpu. It uses **WebGPU** when the browser has it and falls back to From 9db177c1637d79348a6ca3b51ff207f0ebdf6772 Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 08:05:36 +0600 Subject: [PATCH 80/85] Add text: the ink tests reach the non-number and content_update paths (review) The hand-edited ink test's "CMYK entry not a number" case had five entries, so the length check refused it before any entry was read; it now has four. content_update gets its own refusals: two inks at once, an ink out of range beside another style change, and gray or cmyk on an image, with the item list unchanged after each. --- crates/automation/tests/it/automation.rs | 24 ++++++++++++++++++++++++ crates/edit/src/tests.rs | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index b2bf8ba25d..db47eeb37b 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -5265,6 +5265,30 @@ fn added_text_takes_a_cmyk_or_gray_ink_through_tools() { assert!(matches!(a.call("page_add_text", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); } assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["count"], 2, "nothing refused was added"); + // content_update refuses them too, an image takes no ink, and a refused update changes nothing. + let mut png = Vec::new(); + { + let mut enc = png::Encoder::new(&mut png, 40, 20); + enc.set_color(png::ColorType::Rgb); + let mut w = enc.write_header().unwrap(); + w.write_image_data(&[200u8; 2400]).unwrap(); + } + std::fs::write(dir.join("logo.png"), png).unwrap(); + ok(&mut a, "page_add_image", json!({ "doc": doc, "page": 1, "path": "logo.png", "rect": [100, 200, 180, 240] })); + let before = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + for (index, bad) in [ + (1, json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] })), + (2, json!({ "color": "red", "gray": 0.5 })), + (1, json!({ "size": 30, "cmyk": [0, 0, 0, 2] })), + (2, json!({ "bold": true, "gray": -1 })), + (3, json!({ "gray": 0.5 })), + (3, json!({ "cmyk": [0, 0, 0, 1] })), + ] { + let mut args = json!({ "doc": doc, "page": 1, "index": index }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("content_update", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["items"], before, "a refused update changed nothing"); } /// Edit ▸ Copy and Paste headlessly (#2636): an added text box or image, as it is, onto its own diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 8d9f5c58d2..49ab8abbc7 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1158,7 +1158,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { "a CMYK entry not a number", vec![ (b"Color", real(&[0.0; 3])), - (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(0.0), Object::Real(1.0)])), + (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(1.0)])), ], ), ( From 6ec23789c1a6ab072195cc6a69774d26f09667af Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:28:05 +0800 Subject: [PATCH 81/85] pdfcraft-web: say a timed-out call is not guaranteed to have been skipped --- apps/pdfcraft-web/src/api.rs | 2 +- packaging/web/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/pdfcraft-web/src/api.rs b/apps/pdfcraft-web/src/api.rs index 379d092f72..3f1d265377 100644 --- a/apps/pdfcraft-web/src/api.rs +++ b/apps/pdfcraft-web/src/api.rs @@ -10,7 +10,7 @@ //! //! The channel keeps the accessibility tree of every frame, so it is off unless the page asks for it //! (`?control`), as FigCraft's is. PdfCraft's control requests cannot be withdrawn, so a call the page -//! gave up on still runs when the app reaches it, as it does for a desktop client that timed out. +//! gave up on may still run (or be rejected) when the app reaches it, as it does for a desktop client that timed out. use serde_json::{Value, json}; diff --git a/packaging/web/README.md b/packaging/web/README.md index a1fe64c952..375f1f4ba0 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -138,7 +138,7 @@ The messages and both transports are the shared contract of [`craft-web-host`](h ### Driving it from the page (`window.pdfcraft`) -`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app still runs it when it reaches it, as it does for a desktop client that timed out. +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. ```js const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog From a9ccca5b71c70ffb20d1ea71e33e9bdea618ad49 Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:30:00 +0800 Subject: [PATCH 82/85] pdfcraft-web: say ui.open answers with an error on the web --- packaging/web/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/web/README.md b/packaging/web/README.md index 375f1f4ba0..7d305b801d 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -138,7 +138,7 @@ The messages and both transports are the shared contract of [`craft-web-host`](h ### Driving it from the page (`window.pdfcraft`) -`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`…; `ui.open` needs a file system, so on the web it answers with an error and a file is dropped on the page instead), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. ```js const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog From 5bb8bbf20be6778d4e4b9f6233510c299e4bf56e Mon Sep 17 00:00:00 2001 From: eliets2 Date: Sun, 8 Nov 2026 02:32:43 +0200 Subject: [PATCH 83/85] Add dry-run preview to doc_batch without writing outputs (#6486) * Add dry-run preview to doc_batch without writing outputs Users cannot preview a batch before it writes files, so they cannot check the planned steps and output names for a folder first. Add a dry_run option to doc_batch that reuses discovery, returns each file's ordered steps and planned output name, and skips processing and publishing. Test: doc_batch_dry_run_lists_outputs_without_writing failed before the change and passes after. * Address review for pull request 6486 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> --- crates/automation/src/batch.rs | 51 ++++++++++++++++++++++++- crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/batch.rs | 58 +++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 3 deletions(-) diff --git a/crates/automation/src/batch.rs b/crates/automation/src/batch.rs index dd5ca6aec0..f650e565b7 100644 --- a/crates/automation/src/batch.rs +++ b/crates/automation/src/batch.rs @@ -185,6 +185,7 @@ impl Automation { let recipe = Recipe::parse(self, args)?; let recursive = args.opt_bool("recursive")?.unwrap_or(false); let stop = args.opt_bool("stop_on_error")?.unwrap_or(false); + let dry_run = args.opt_bool("dry_run")?.unwrap_or(false); let paths = args.strs("paths")?; if paths.is_empty() || paths.len() > 1000 { return Err(Args::wrong("paths", "1 to 1000 files or folders")); @@ -211,11 +212,41 @@ impl Automation { let mut files = discovery.skipped; let (mut completed, mut failed_count) = (0, 0); let mut stopped = false; + let mut planned_outputs: HashSet = HashSet::new(); for (path, relative) in discovery.files { if stopped || !recipe.accepts(&path) { files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}})); continue; } + if dry_run { + let planned: Result = (|| { + let target = out.join(&relative).with_extension("pdf"); + let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?; + if !target.starts_with(&out) { + return Err(failed("output path leaves out_dir")); + } + let dir = target.parent().ok_or_else(|| failed("output has no parent"))?; + let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy().into_owned(); + for n in 1..=10_000 { + let candidate = if n == 1 { target.clone() } else { dir.join(format!("{stem} ({n}).pdf")) }; + if candidate.symlink_metadata().is_ok() || planned_outputs.contains(&candidate) { + continue; + } + planned_outputs.insert(candidate.clone()); + return Ok(candidate); + } + Err(failed("all 10000 output names are occupied")) + })(); + match planned { + Ok(target) => files.push(json!({"path":path.to_string_lossy(),"status":"planned","output":target.to_string_lossy().replace('\\', "/"),"steps":recipe.steps})), + Err(e) => { + failed_count += 1; + files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()})); + stopped = stop; + } + } + continue; + } // Recheck confinement and links before reading a discovered file. let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?; @@ -244,8 +275,8 @@ impl Automation { } } } - let skipped = files.len().saturating_sub(completed + failed_count); - Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) + let skipped = files.iter().filter(|file| file["status"] == "skipped").count(); + Ok(json!({"dry_run":dry_run,"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) } } @@ -279,4 +310,20 @@ mod tests { discovery.visited = 0; assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64")); } + + #[test] + fn doc_batch_dry_run_lists_outputs_without_writing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + let pdf = Session::new().create_from_text("Doc", "Hello").unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), &*pdf).unwrap(); + let auto = Automation::new().with_root(dir.path()).unwrap(); + let report = auto.doc_batch(&Args(&json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","dry_run":true}))).unwrap(); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["files"][0]["status"], "planned", "{report}"); + assert!(report["files"][0]["output"].as_str().unwrap().ends_with("out/a.pdf"), "{report}"); + assert!(!dir.path().join("out").exists()); + } } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 84354f51d7..674b00f329 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -148,7 +148,7 @@ pub fn tools() -> Vec { "paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000}, "out_dir": {"type":"string"}, "steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true}, - "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, + "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, "dry_run": {"type":"boolean"}, "ppi": {"type":"number","minimum":9,"maximum":1600}, "quality": {"type":"integer","minimum":1,"maximum":100}, "dpi": {"type":"number","minimum":72,"maximum":600}, diff --git a/crates/automation/tests/it/batch.rs b/crates/automation/tests/it/batch.rs index af5e0c1272..8b7650fd3f 100644 --- a/crates/automation/tests/it/batch.rs +++ b/crates/automation/tests/it/batch.rs @@ -238,3 +238,61 @@ fn mcp_batch_errors_keep_the_structured_per_file_report() { assert_eq!(report["result"]["isError"], true, "{report}"); assert_eq!(report["result"]["structuredContent"]["failed"], 1); } + +#[test] +fn dry_run_planned_files_are_not_counted_as_skipped() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + let original = pdf(); + std::fs::write(dir.path().join("input/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("input/sub/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("out/a.pdf"), b"precious").unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["optimize"],"out_dir":"out","dry_run":true})); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["skipped"], 0, "planned files are not skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 2, "{report}"); + assert!(files.iter().all(|file| file["status"] == "planned"), "{report}"); + assert!(files.iter().all(|file| file["steps"] == json!(["optimize"])), "planned files list the ordered steps: {report}"); + assert!(files[0]["output"].as_str().unwrap().ends_with("out/a (2).pdf"), "existing output gets the publish collision name: {report}"); + assert!(files[1]["output"].as_str().unwrap().ends_with("out/a (3).pdf"), "later planned files reserve their names: {report}"); + assert_eq!(std::fs::read(dir.path().join("out/a.pdf")).unwrap(), b"precious"); + assert!(!dir.path().join("out/a (2).pdf").exists(), "dry run writes nothing: {report}"); + #[cfg(unix)] + { + // An output subfolder that resolves outside out_dir is a per-file error: + // it counts as failed, honors stop_on_error, and later files are skipped. + for outside_root in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/z.pdf"), pdf()).unwrap(); + let target = if outside_root { outside.path().to_path_buf() } else { dir.path().join("outside") }; + if !outside_root { + std::fs::create_dir(&target).unwrap(); + } + std::os::unix::fs::symlink(&target, dir.path().join("out/sub")).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result( + &mut auto, + json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","recursive":true,"dry_run":true,"stop_on_error":true}), + ); + assert_eq!(report["failed"], 1, "preview target failure is per-file: {report}"); + assert_eq!(report["stopped"], true, "{report}"); + assert_eq!(report["skipped"], 1, "only the unprocessed file is skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 3, "{report}"); + assert_eq!(files[0]["status"], "planned", "{report}"); + assert_eq!(files[1]["status"], "error", "{report}"); + assert_eq!(files[2]["status"], "skipped", "stop_on_error leaves later files unplanned: {report}"); + assert!(!dir.path().join("out/a.pdf").exists(), "dry run writes nothing: {report}"); + } + } +} From f841666d5f0cf5b6bb14fb4475786fb3ebefeef7 Mon Sep 17 00:00:00 2001 From: Luis <43929895+Luis-0508@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:22:15 +0100 Subject: [PATCH 84/85] Release: refresh contributor credits for each release (#6487) (#6488) About > Contributors still showed an old snapshot, because nothing regenerated contributors/contributors.json before a release. Opt in to the shared refresh from craft-common-actions: a read-only credits job regenerates the snapshot for the release commit, and the shared release workflow installs it before every source build. The snapshot is not committed back. --- .github/workflows/release.yml | 8 ++++++++ docs/contributors.md | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9403a1e044..4b44d79b68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,14 @@ concurrency: cancel-in-progress: false jobs: + # Regenerates contributors/contributors.json for the release commit; every build bundles it. + credits: + permissions: + contents: read + pull-requests: read + uses: storytold/craft-common-actions/.github/workflows/refresh-contributors.yml@main release: + needs: credits # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: @@ -27,6 +34,7 @@ jobs: app-title: PdfCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 version: ${{ inputs.version }} + contributors-artifact: ${{ needs.credits.outputs.artifact-name }} secrets: APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} diff --git a/docs/contributors.md b/docs/contributors.md index fa7e988611..b24d1cd053 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -8,7 +8,8 @@ the AI models named in `Co-Authored-By` trailers. - Two files, both **compiled into the binary** by the UI crate's `build.rs` (nothing is read from disk or the network at run time; the web build has them too): - [`contributors/contributors.json`](../contributors/contributors.json): commit stats per GitHub - username. + username. Release builds regenerate it for the release commit + (`.github/workflows/release.yml`, `credits` job); the copy in the repository is not updated. - [`contributors/people.toml`](../contributors/people.toml): the names people chose for themselves. **Each contributor adds only their own entry**, in their own PR. - Each contributor gets one line: GitHub username (always), display name (opt-in, or their synced From adbf1dc01fefbdad070f676d16f442a76ee0c31d Mon Sep 17 00:00:00 2001 From: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:07:17 +0530 Subject: [PATCH 85/85] M9: signed appearances show their real validation state (#79) (#2808) * M9: redraw named-layer signature status layers - /N drawing one Form that holds n1 and n3: n1, n3 and n4 redrawn for the state - copies made once per original and state; originals never edited - Lucide icons; captions in Helvetica, embedded Inter, or craft-fonts outlines Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: draw signature states in the display copy, exports and print - marks follow the bytes shown; a changed verdict redraws after a metadata edit - background opens pick up the trust and captions current at commit Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: status captions in the interface language - the three captions in all 22 catalogues - Signatures panel colours from sign::status_rgb Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: signature badge in the widget's corner - a quarter of the shorter side, 48pt cap; outline only under 32pt - dropped for a drawn valid state; bottom-right where the appearance draws one - sign_list reports verdict_shown; parity and roadmap Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: keep Arabic signature captions in logical order Pass raw RTL catalogue text to the signature appearance shaper so UAX #9 is applied exactly once. This keeps the rendered word order and exported /ActualText consistent. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M2: read /ActualText over painted paths and pictures - A replacement that marks no glyph reads where it paints, cut to what is visible and along the way it is drawn. - A span the page content leaves open ends before the annotations. - OCR and scan field detection still count such a page as having no text of its own. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M8: redaction strips /ActualText around paint it clips - A path or shading written again under a clip counts as changed, so a partly redacted word drawn as outlines leaves no name in the file. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: recognise only layer forms that draw n1 and n3 - Settle on the one form holding both first, then read its content once per call. - Skip an /N that is its own layer form; copies keep the layer's /OC. - A verdict counts as drawn when any visible widget shows it. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: keep the signature badge and redraw wherever a state changes - The badge stays for every state; its mark sits bottom-right as the widget lies on the page. - An edit that changes a verdict redraws every page; captions apply before startup files open. - Print, the CLI's render and the repair log follow the display copy. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: restore the attribution usage the rebase dropped - Usage lines for the three icons, Inter and the craft-fonts input; ATTRIBUTION.md rebuilt from them. - Roadmap and localization notes for this round. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: pin signature field names on a nested form Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: bound the signature field walk - kids share their parent's name and type instead of each copying them - a widget stays listed under every field that claims it, as one shared copy - a field written in place in a shared /Kids array is walked once - caps of 20,000 fields, 100,000 entries and 4096-byte names; no states drawn past them Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: keep a document signed when the field walk stops early - list_cached adds a signed row of unknown validity for the fields the walk didn't read - Save As and actions then append, and Optimize, Sanitize and full rewrites are refused, as for any signed file Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M8: removing hidden content no longer strips the names of shadings - With no regions to clip, a shading is left alone instead of wrapped and marked clipped - clean_replacements skips scopes already marked, so deep nesting stays linear Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M2: read a picture's /ActualText over the whole picture - draw_image's transform maps pixels, not a unit square, so the text sat on one pixel in the corner; the test now draws a 2x2 picture - Painted grows in place instead of being copied through an Option Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: cap the bytes of names the signature field walk keeps - A kid's name repeats its parent's, so one long name over 20,000 kids came to 80 MB; the walk now stops at 4 MiB of names and counts the rest as unread - Cover the widget written into a shared /Kids array and the field cap among kids, which no test caught - sig_fields is private again, and the cap docs say what happens past each cap Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: skip status-layer work that changes nothing; fix tests that could not fail - show_status_layers and has_status_layers return before reading the form when nothing is signed, so an unsigned form no longer walks every field on each edit - revalidate and draw_captions leave documents with no drawn state alone, and no longer copy the signature list to update it - Scan form detection reads the page's text once; comment and badge tips share one hover rule - Tests: drop a table row that repeated another, stop taking an expectation from the code under test, stop comparing an Arc with itself, give bare asserts messages Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: test the signature field name cap at the engine boundary - A 4,096-byte name is read and blocks a full rewrite; at 4,097 bytes the list holds the unread-fields row and a full rewrite is still refused Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: test signature states beside fields the viewer draws - One display copy holds both a field's drawn appearance and the signature's state. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: draw Thai and Devanagari status captions in their own script - script_of names Thai and Devanagari, so a craft-fonts face meant for either draws the caption as outlines; without one it stays English. - Localization, parity and the craft-fonts usage line list the new scripts. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M2: paint left out by an exclusion mixes its /ActualText replacement - A replacement placed by outlines or pictures, some of them excluded, fails filtered extraction as one with excluded glyphs does. - Glyphs still place a replacement that has them. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: a long-named field that can't hold a signature leaves the walk whole - A leaf of another type passed over for its name is no signature field and has none below it, so an unsigned form stays unsigned. - A signature field, or any field with kids, still counts as unread. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: status captions use document faces only - Each script takes the face PdfCraft writes it into documents with, so Chinese stays English although the pinned craft-fonts' interface-only Chinese face is meant for Hani. - The craft-fonts usage line names Devanagari: the pinned revision has a face for it, and Hindi captions are drawn in it. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: refuse a new signature field the field walk can't read back - Signing adds a field only when the walk reads the whole form with it: past a cap the signature would be listed as unread fields, and its name could repeat an unread field's. - A field the walk already reads is signed as before. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: refuse a new signature field with a blank name - The walk lists no field without a name, so signing now also requires the walk to list the new field: one named only with spaces or NULs would be a signature nothing shows. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: an unnamed signed field or an unreadable form counts as unread - A signed field with no usable /T, or an /AcroForm or /Fields of the wrong type, cuts the field walk short: the document keeps its unread-fields row and stays signed. - Signing refuses a blank new field name before it adds the field, so the refusal still says why. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: no display copy when no signature state is drawn - display_bytes_for makes a copy only when it added an appearance or drew a state, as the app does. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: the verdict mark stays on the page - A widget past the crop box takes its mark in the part on the page, or none when too little of it is there. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: typed values stay drawn while a signature redraws every page - A form edit that changes a signature's state puts its values back in unbaked after the pages are invalidated. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: the action, Save As and shading tests check their effects - The signed action and Save As tests reopen the output for the title and the comment; the shading test checks its clip. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: values typed earlier stay drawn while a signature redraws every page - A form edit that changes a signature's state keeps every value still waiting on its page, not only its own. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: comment and content edits keep typed values drawn too - An edit to one page's comments or content that changes a signature's state moves no page, so the values still waiting on their pages stay pending, as after a form edit. - The pending-value test runs a field deletion and a comment after a fill, under a certification that allows the fill; its messages say what it checks. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: the language-change test skips without catalogs - Like the other language tests, it returns early when the build has no craft-i18n catalogs. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> * M9: signed fields behind kids without a name count - A kid with no /T but kids of its own is walked as a field node, so a signed field under it is listed by its full name. - An unnamed kid claiming a signature its field doesn't have cuts the walk short, so the document stays signed and keeps its unread-fields row. Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> --------- Signed-off-by: Chiranjeevi U <244287281+chiranjeevi-max@users.noreply.github.com> --- ATTRIBUTION.md | 10 +- ATTRIBUTION.toml | 10 +- Cargo.lock | 2 + ROADMAP.md | 6 + apps/pdfcraft-cli/src/commands.rs | 5 +- crates/automation/src/signing.rs | 1 + crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/automation.rs | 39 + crates/engine/src/document.rs | 115 +- crates/engine/src/export.rs | 11 +- crates/engine/src/js.rs | 8 +- crates/engine/src/lib.rs | 4 +- crates/engine/src/ocr.rs | 46 +- crates/engine/src/open.rs | 32 +- crates/engine/src/session.rs | 12 +- crates/engine/src/session/editing.rs | 34 +- crates/engine/src/session/files.rs | 5 + crates/engine/src/session/save.rs | 58 +- crates/engine/src/tests.rs | 636 +++++++- crates/engine/src/word_count.rs | 1 + crates/fonts/src/lib.rs | 2 +- crates/fonts/src/shaping.rs | 10 + crates/redact/src/interp.rs | 35 +- crates/redact/src/tests.rs | 62 + crates/render/src/text.rs | 3 + crates/render/src/text/device.rs | 207 ++- crates/render/src/text/tests.rs | 201 +++ crates/sign/Cargo.toml | 4 + crates/sign/README.md | 4 + crates/sign/src/lib.rs | 2 + crates/sign/src/pdf.rs | 13 +- crates/sign/src/pdf/listing.rs | 278 +++- crates/sign/src/pdf/signing.rs | 14 +- crates/sign/src/pdf/types.rs | 4 +- crates/sign/src/pdf_fields_tests.rs | 448 ++++++ crates/sign/src/status_layers.rs | 1645 ++++++++++++++++++++ crates/ui-egui/src/app.rs | 2 + crates/ui-egui/src/app/files.rs | 3 + crates/ui-egui/src/canvas.rs | 57 + crates/ui-egui/src/canvas/area.rs | 3 + crates/ui-egui/src/canvas/page_input.rs | 43 +- crates/ui-egui/src/canvas/tests.rs | 143 ++ crates/ui-egui/src/editing.rs | 35 +- crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/tools/comments.rs | 2 +- crates/ui-egui/src/tools/comments/input.rs | 15 +- crates/ui-egui/src/tools/sign_ui.rs | 66 +- crates/ui-egui/src/tools/sign_ui/dialog.rs | 15 +- crates/ui-egui/src/tools/sign_ui/panel.rs | 13 +- crates/ui-egui/tests/it/main.rs | 1 + crates/ui-egui/tests/it/signing.rs | 53 + crates/ui-egui/tests/it/status_layers.rs | 434 ++++++ docs/localization.md | 2 +- docs/roadmap.md | 5 +- docs/word-count.md | 5 +- parity/acrobat-features.toml | 16 +- 56 files changed, 4687 insertions(+), 197 deletions(-) create mode 100644 crates/sign/src/pdf_fields_tests.rs create mode 100644 crates/sign/src/status_layers.rs create mode 100644 crates/ui-egui/tests/it/status_layers.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 38c28a80ec..a8de85e07a 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -45,7 +45,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/chevron-up.svg` | Lucide icon "chevron-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevron-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-left.svg` | Lucide icon "chevrons-left" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-left.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-right.svg` | Lucide icon "chevrons-right" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-right.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/circle-dashed.svg` | Lucide icon "circle-dashed" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dashed.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-dot.svg` | Lucide icon "circle-dot" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dot.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-help.svg` | Lucide icon "circle-help" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-help.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -174,7 +174,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/text-select.svg` | Lucide icon "text-select" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/text-select.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash-2.svg` | Lucide icon "trash-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash.svg` | Lucide icon "trash" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/type.svg` | Lucide icon "type" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/type.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/underline.svg` | Lucide icon "underline" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/underline.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/undo-2.svg` | Lucide icon "undo-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/undo-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -184,7 +184,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/x.svg` | Lucide icon "x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-in.svg` | Lucide icon "zoom-in" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-in.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-out.svg` | Lucide icon "zoom-out" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-out.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths | +| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs | | `assets/fonts/Inter-Medium.ttf` | Inter Medium | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/Inter-SemiBold.ttf` | Inter SemiBold | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/JetBrainsMono-Regular.ttf` | JetBrains Mono Regular | The JetBrains Mono Project Authors | OFL-1.1 | https://github.com/JetBrains/JetBrainsMono | UI monospace font; demo PDF code | @@ -249,7 +249,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/thumbs-up.svg` | Lucide icon "thumbs-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/thumbs-down.svg` | Lucide icon "thumbs-down" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-down.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-plus.svg` | Lucide icon "message-square-plus" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-plus.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `crates/ui-egui/tests/data/form.pdf` | One-page AcroForm test fixture (text, check box, radio group, combo box) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: hand-written PDF objects, no third-party content | Form-filling UI tests (crates/ui-egui/tests/it/forms.rs, commands.rs) | | `assets/icons/pentagon.svg` | Lucide icon "pentagon" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pentagon.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-quote.svg` | Lucide icon "message-square-quote" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-quote.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -314,4 +314,4 @@ Not in this repository and never downloaded by it: compiled in only when the bui | Input | Option | Title | Author | Licence | Source | Attribution | Used for | |---|---|---|---|---|---|---|---| | `craft-i18n` | `CRAFT_I18N_DIR` | Interface translations: the craft-i18n catalogs (locales//common.tsv and pdfcraft.tsv) of the 34 languages PdfCraft offers besides English | The craft-i18n translators, including every translator of PdfCraft's former in-repo catalogs (crates/ui-egui/src/i18n/*.tsv, migrated at pdfcraft@ecdd1e6) | MIT OR Apache-2.0 | https://github.com/storytold/craft-i18n (pinned by commit in craft-i18n.rev, which CI, releases and the flatpak manifests read) | https://github.com/storytold/craft-i18n/blob/main/ATTRIBUTION.md | Interface text in languages other than English, embedded by crates/ui-egui/build.rs and looked up through craft-libs' craft-i18n crate; a build without CRAFT_I18N_DIR is English only. Clean-room translations from PdfCraft's English interface labels | -| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | +| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 99073babb1..608bb601a5 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -425,7 +425,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "201e790f3a05c65b2c4b0f952c9b9e1f35cd1c5b21698b3f4be131883fe90bc8" [[asset]] @@ -1844,7 +1844,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "55aa37e0b73e7f8cd906952dbdeb96283c6595ca64aba240a17f3455cb989114" [[asset]] @@ -1954,7 +1954,7 @@ source = "https://github.com/rsms/inter" licence = "OFL-1.1" licence_file = "assets/fonts/OFL-Inter.txt" kind = "font" -usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths" +usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs" sha256 = "40d692fce188e4471e2b3cba937be967878f631ad3ebbbdcd587687c7ebe0c82" [[asset]] @@ -3004,7 +3004,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "ef1fec17d88794ba6c34c5cf63bdadbd234b3d4dffbcc3d55468b47e3f398472" [[asset]] @@ -3142,7 +3142,7 @@ source = "https://github.com/storytold/craft-fonts (pinned by commit in .github/ licence = "OFL-1.1" attribution = "https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md" kind = "font" -usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" +usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" [[asset]] path = "crates/sign/tests/data/x509-rsa-sha1.pdf" diff --git a/Cargo.lock b/Cargo.lock index 0d315535a4..6710dddeea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4687,9 +4687,11 @@ dependencies = [ "p521", "pbkdf2", "pdfcraft-annot", + "pdfcraft-content", "pdfcraft-cos", "pdfcraft-fonts", "pdfcraft-model", + "pdfcraft-testkit", "rc2", "ripemd", "rsa", diff --git a/ROADMAP.md b/ROADMAP.md index 820ceb0179..1de96a7cda 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -315,6 +315,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-22 (M14, macOS association recovery candidate):** Successful desktop open/save/sign can queue recovery of an existing per-file binding to the running PDFCraft bundle through NSWorkspace. Metadata-only watcher changes retain undo and avoid false overwrite prompts; real same-length/same-mtime edits remain protected. Rebased on 0.12.0 and reuses #3042's shared atomic metadata preservation; the bounded recovery parser does not narrow save limits. Prior native Preview TEST-signature and Finder workflows passed on macOS 27.0.1; other versions remain unqualified. Estimates unchanged. - **2026-10-13 (M7, free image rotation):** A selected page image has a rotate knob (Shift: 15 degree steps) and `image_edit` a `rotate_by` action; the angle is written as one undoable `cm`. Snapping to page edges while moving is in #1740; resize snapping is not done. Estimates unchanged. +- **2026-10-20 (M9 #79 review, M2, M8):** text extraction reads a marked-content /ActualText that marks no glyph where it paints (paths and images, cut to what is visible, along the direction they are drawn), so words drawn as outlines can be found, copied and exported; OCR and scan field detection still count such a page as having no text of its own. Redaction strips /ActualText, /Alt, /E and /T from inline marked content around paint it clips, as it already did around paint it removed, so a partly redacted outlined word leaves no name in the file. Named-layer recognition also needs the layer form to draw n1 and n3. Estimates unchanged. - **2026-10-20 (M4, initial view):** Window and interface options read indirect boolean ViewerPreferences values. Synthetic regressions cover true/false, direct/indirect preference dictionaries and save/reopen; malformed entries retain false defaults. Estimates unchanged. - **2026-10-20 (M3, #3535):** Escape with a menu open closes the menu and keeps the text selection; the next Escape clears it, as before. One keypress used to do both. A synthetic UI regression checks it. Estimates unchanged. - **2026-10-16 (M3, close prompt keyboard):** The "Save changes?" prompt opens with Save chosen (Enter saves, as before) and the arrow keys and Tab / Shift+Tab move the choice to Cancel or Don't save, shown with a focus ring; Enter or Space press the chosen button and Escape still cancels. Before, Enter always saved whatever was Tab-focused. Estimates unchanged. @@ -405,6 +406,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-16 (M10, #1201 partial):** Lossless image export applies each component's Decode pair. Indexed values round and clip before palette lookup. Color-key masks keep raw samples. The existing 16-bit precision limit remains. JPEG and mask Decode, ICC transforms, and calibrated colors remain outside this fix. Estimates unchanged. - **2026-10-16 (M3, #2739):** Wrapped revision buttons stay above Binding in Document Properties. A synthetic regression checks control bounds and opens a saved revision at two viewport widths. Estimates unchanged. - **2026-10-19 (M7, tracked text):** Editable paragraphs and object labels distinguish repeated glyph tracking from word gaps in TJ arrays and separately positioned glyphs. Synthetic regressions cover character spacing, horizontal scaling, rotated baselines, larger word gaps, and rows of one-glyph words or cells (inferred tracking is capped at half an em, so `a b c` and digit or Y/N columns keep their spaces). Estimates unchanged. +- **2026-10-16 (M9, #79):** a signature appearance that follows the named-layer convention (its /N draws one Form that holds and draws n1 and n3) shows the signature's real state: n1, n3 and n4 are redrawn in the display copy with the panel's icons and a caption in the interface language (Helvetica; else shaped in embedded Inter, or in a craft-fonts face for Arabic, Japanese and Telugu; else English, as Chinese always is). Exports, both print modes and the CLI's render show the redrawn state; saves never do. Every signature keeps its badge, since a document can clip or cover its own layers. Estimates unchanged. +- **2026-10-16 (M9):** the page validation overlay moves to the widget's bottom-right corner as it lies on the page (so it turns with the page and the view) and takes a quarter of the widget's shorter side, at most 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field too small for a legible mark keeps its outline, a rectangle the screen cannot place is not drawn at all, and the whole widget answers the pointer. Estimates unchanged. - **2026-10-16 (M5, #2669, item 5):** The Comments panel groups replies by name and counts all page roots once per frame. Estimates unchanged. - **2026-10-16 (M7, #2140):** Header & footer (and Bates numbering, which uses the same dialog) can shrink the page content so the header and footer don't cover it: the original content is scaled evenly into the band they leave free, centred across, also on rotated pages; Remove and Update take the scaling away, and margins that leave no room are refused. Dialog checkbox with a preview of the band, `shrink` on `doc_header_footer`, all catalogs. Annotations and added text and images are not moved yet. Estimates unchanged. - **2026-10-20 (M14, Spanish catalog gaps):** `es.tsv` gains the 143 `(context, English)` keys the other catalogs carried and it was missing, so Spanish covers the whole bundled key union like Lithuanian and Vietnamese: calendar months and weekdays, the `certificate` context, X.509 key usages, signature-revision and digital-ID lines, password validation, eight plural messages, the combine-from-folder strings, the rendering and stale-file notices and the shortcut-spelled tooltips. Counts in `localization-parity.md` re-measured against the current union (`vi` restated against it for the same reason). Status stays `partial`: no translated help, and OS details stay outside the catalog. Estimates unchanged. @@ -793,6 +796,7 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-11-01 | patch | M7 #5657: header/footer, watermark and background operations commit all selected pages together; later-page errors preserve marks, content, shared/inherited resources and pending edits; estimates unchanged. | | 2026-10-28 | patch | M2, PR #4172: integrate main 5af10276 and retain both roadmap histories; viewer-only CJK and print/output font policies and estimates unchanged. | | 2026-10-27 | minor | M10: OCR stages SHA-256-verified ONNX models after the upstream RTEN URL brownout; discovery prefers a complete ONNX pair per directory, falls back to a complete RTEN pair, and never mixes formats or directories; language/web limits and estimates unchanged. | +| 2026-10-20 | minor | M9 #79 review, M2, M8: /ActualText that marks no glyph extracts where it paints, as far as it is visible; redaction strips /ActualText around paint it clips; named-layer recognition needs the layer form to draw n1 and n3. Estimates unchanged. | | 2026-10-23 | minor | M8, #3205 follow-up: phone-pattern redaction includes the +39 country code on labelled Italian geographic numbers with a slash; dates, protocol identifiers and whole-run guards remain. Estimates unchanged. | | 2026-10-23 | minor | M11: image downsampling measures physical resolution after page-local UserUnit scaling, including shared nested forms; synthetic measurement and optimization regressions. Estimates unchanged. | | 2026-10-23 | minor | M10: Word table cells use the grid columns they span, including cells after a merged cell; synthetic export regression and CLI reproduction. Estimates unchanged. | @@ -807,6 +811,8 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-10-18 | minor | M1, #1143: a live reference may match the xref generation or a same-number header at its indexed offset; neither-match and free slots remain null. Bounded repair diagnostics and synthetic cache, free-slot, edit and classic/packed save regressions; estimates unchanged. | | 2026-10-18 | minor | M5, PR #1971: nested source `/Page` and `/Pages` dictionaries export as null without traversing page content or the page tree. Synthetic saved/reopened three-page regression retains unrelated metadata and aliases. FDF import semantics, copied-graph limits and estimates unchanged. | | 2026-10-15 | minor | M5: budget retained FDF entries before cloning; discovery keeps lazy array owners and annotation positions, with alias/cache-eviction stress fixtures. Limits cover the copied graph; source parsing/page discovery retain existing COS/parser limits, with no total source-memory bound. FDF import semantics and estimates unchanged. | +| 2026-10-16 | minor | M9 #79: named-layer signature appearances show the signature's real state in the display copy, exports and both print modes, never in saves; captions follow the interface language. Estimates unchanged. | +| 2026-10-16 | minor | M9: the page validation overlay moves to the widget's bottom-right corner as it lies on the page and stops growing past 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field under 32 screen points keeps its outline. Estimates unchanged. | | 2026-10-15 | minor | M7: the Edit text caret is as tall as the row it is on in a region with several sizes; estimates unchanged. | | 2026-10-18 | minor | M4, PR #1919 review: repeated page indices apply each rotation to the latest page-local value, preserving indirect/inherited angles. Synthetic adjacent/interleaved duplicates retain siblings and page contents through incremental and full saves. Estimates unchanged. | | 2026-10-18 | minor | M3, #3149: page replacement cancels render demand collected for the previous geometry; all viewer raster upload paths reject empty or oversized textures before cache eviction. Synthetic regressions reproduce the reported 1880×2350 upload at a 2048 limit, retain cached images and unsaved edits, and resume bounded backdrops and tiles. Estimates unchanged. | diff --git a/apps/pdfcraft-cli/src/commands.rs b/apps/pdfcraft-cli/src/commands.rs index 9d0b2057f0..3118c901b4 100644 --- a/apps/pdfcraft-cli/src/commands.rs +++ b/apps/pdfcraft-cli/src/commands.rs @@ -366,8 +366,9 @@ pub(crate) fn render(args: &[String]) -> Result<(), CliError> { Some("pam") => None, _ => return Err(format!("render: --out {out}: use a .png, .jpg, .tif or .pam name").into()), }; - // The app renders a display copy that draws comments missing their /AP streams; the CLI - // must agree with it (and with independent viewers), while keeping the file it reads intact. + // The app renders a display copy that draws comments missing their /AP streams and the state + // of named-layer signatures (here with no trusted certificates); the CLI must agree with it + // (and with independent viewers), while keeping the file it reads intact. let bytes = read(path)?; let password = flag(args, "--password"); // The renderer takes only the user password of R2–R4 files; `Session::open` and the display diff --git a/crates/automation/src/signing.rs b/crates/automation/src/signing.rs index 743a36ff56..df0fd643fb 100644 --- a/crates/automation/src/signing.rs +++ b/crates/automation/src/signing.rs @@ -119,6 +119,7 @@ impl Automation { "page": s.page.map(|p| p + 1), "rect": rect, "visible": s.visible, + "verdict_shown": s.verdict_shown, "revision": s.revision, "sub_filter": s.sub_filter, "algorithm": s.algorithm, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 674b00f329..99f27bba61 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -896,7 +896,7 @@ pub fn tools() -> Vec { comment_ref(json!({ "status": { "type": "string", "enum": ["none", "accepted", "rejected", "cancelled", "completed"] }, "author": { "type": "string" } })), &["doc", "status"], )), - t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") + t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, whether PdfCraft draws that state into the signature's own appearance as displayed (verdict_shown; never written to the file), the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t("sign_id_create", "Create a digital ID", "Create a self-signed digital ID and save it as a password-protected .p12 file (Acrobat: Configure a new digital ID ▸ Create a new digital ID ▸ Save to file). Default key: 2048-bit RSA, valid 5 years.").with(schema( diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index dc5dec0521..1b5ad777a1 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -6899,6 +6899,43 @@ fn a_dynamic_xfa_forms_signature_box_is_signed_under_its_som_path() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn named_layer_verdicts_are_reported_and_rendered_through_tools() { + use pdfcraft_cos::{Dict, Document, Lexer, ObjRef, Object, SaveOptions, Stream}; + let dir = workdir("named-layer-verdict"); + let mut cos = Document::open(std::sync::Arc::new(fixture(1))).unwrap(); + let dict = |text: &str| Lexer::new(text.as_bytes(), 0).object().unwrap().as_dict().unwrap().clone(); + let form = |resources: Dict, ops: &str| { + let mut d = dict("<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] >>"); + d.set(b"Resources".to_vec(), resources); + Stream::from_raw(d, ops.as_bytes().to_vec()) + }; + let blank = cos.add(form(Dict::new(), "")); + let layers = dict(&format!("<< /XObject << /n1 {blank} /n3 {blank} >> >>")); + let layer_form = cos.add(form(layers, "/n1 Do /n3 Do")); + let normal = cos.add(form(dict(&format!("<< /XObject << /FRM {layer_form} >> >>")), "/FRM Do")); + // Deliberately malformed CMS: an invalid signature still gets the redrawn invalid state. + let value = cos.add(dict("<< /Type /Sig /SubFilter /ETSI.CAdES.detached /ByteRange [0 0 0 0] /Contents <00> >>")); + let widget = cos + .add(dict(&format!("<< /FT /Sig /T (Signature1) /Subtype /Widget /F 4 /Rect [50 50 150 150] /P 4 0 R /V {value} /AP << /N {normal} >> >>"))); + cos.update_dict(ObjRef::new(4, 0), |p| p.set(b"Annots".to_vec(), vec![Object::Ref(widget)])).unwrap(); + cos.update_dict(cos.root().unwrap(), |root| root.set(b"AcroForm".to_vec(), dict(&format!("<< /Fields [{widget}] >>")))).unwrap(); + let original = pdfcraft_cos::write_full(&cos, &SaveOptions::default()).unwrap(); + std::fs::write(dir.join("named.pdf"), &original).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({"path": "named.pdf"}))["doc"].as_u64().unwrap(); + let list = ok(&mut a, "sign_list", json!({"doc": doc})); + assert_eq!(list["signatures"][0]["status"], "invalid"); + assert_eq!(list["signatures"][0]["verdict_shown"], true); + let rendered = a.call("page_render", &json!({"doc": doc, "page": 1, "dpi": 72})).unwrap(); + let Content::Png { data, .. } = &rendered[0] else { panic!("expected a PNG") }; + let pixels = image::load_from_memory_with_format(data, image::ImageFormat::Png).unwrap().to_rgba8(); + let red = pixels.pixels().filter(|p| p.0.iter().zip([0xD7, 0x37, 0x3F]).all(|(a, b)| a.abs_diff(b) < 30)).count(); + assert!(red > 100, "the invalid state is drawn: {red} red pixels"); + ok(&mut a, "doc_save", json!({"doc": doc, "path": "saved.pdf", "full": false})); + assert_eq!(std::fs::read(dir.join("saved.pdf")).unwrap(), original, "the drawn state never enters saves"); +} + #[test] fn digital_ids_signing_and_validation_through_tools() { let dir = workdir("signing"); @@ -6957,6 +6994,8 @@ fn sign_document_and_verify_trust(a: &mut Automation, dir: &Path) -> u64 { assert_eq!(t["trusted"].as_array().unwrap().len(), 1); let list = ok(a, "sign_list", json!({ "doc": doc })); assert_eq!((list["all_valid"].as_bool(), list["signatures"][0]["status"].as_str()), (Some(true), Some("valid"))); + // PdfCraft's own appearance has no status layers to draw into: only the page badge shows it. + assert_eq!(list["signatures"][0]["verdict_shown"], false); ok(a, "comment_add", json!({ "doc": doc, "page": 1, "type": "note", "at": [20, 20], "contents": "ok" })); let s = &ok(a, "sign_list", json!({ "doc": doc }))["signatures"][0]; assert_eq!((s["status"].as_str(), s["modification"].as_str()), (Some("valid"), Some("allowed"))); diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index 03c4f7ace8..5405467edd 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -8,7 +8,8 @@ use pdfcraft_render::{DocInfo, RenderConfig, RenderPool}; use crate::history::Editor; use crate::{ CalculationOrder, CheckStyle, DocId, Edit, EditError, EditableObject, FormFieldKind, Hidden, InitialView, Label, MarkKind, SecuritySummary, - SignatureInfo, TabOrder, TrustStore, XfaLayout, a11y, form_appearance, guard, js, measure, optimize, renderer_password, unrestricted, + SignatureInfo, StatusCaptions, TabOrder, TrustStore, XfaLayout, a11y, form_appearance, guard, js, measure, optimize, renderer_password, + signatures_of, unrestricted, }; #[cfg(test)] use crate::{record_link_scan, record_marker_scan}; @@ -19,8 +20,9 @@ pub struct Document { pub path: Option, /// The working file: what Save writes. pub bytes: Arc>, - /// What is displayed (the page view, `page_render`, page images): `bytes`, plus appearances - /// drawn for display only for comments that have none ([`display_bytes`]). Never saved. + /// What is displayed (the page view, `page_render`, page images): `bytes`, plus what is drawn + /// for display only ([`display_bytes`]): appearances for comments that have none, and the + /// states of named-layer signatures. Never saved. pub display: Arc>, pub info: DocInfo, pub renderer: RenderPool, @@ -41,6 +43,9 @@ pub struct Document { /// Signature fields, validated against the session's trust store (Signatures panel). pub signatures: Arc>, pub(crate) trust: Arc, + /// The words signature status layers are drawn with (the session's, see + /// [`Session::set_signature_captions`](crate::Session::set_signature_captions)). + pub(crate) captions: Arc, pub(crate) sig_cache: Arc, pub(crate) editor: Option, pub(crate) config: RenderConfig, @@ -649,25 +654,84 @@ pub(crate) fn is_placeholder_title(title: &str) -> bool { address || t == "untitled" } -/// The working file plus appearances for comments that have none, and for form fields the file -/// leaves to the viewer (#3470). Signature artwork is preserved; validation badges belong to the -/// viewer and never enter these bytes, which image/text exports and Print as Image also consume. -pub(crate) fn display_bytes(editor: Option<&mut Editor>, bytes: &Arc>) -> Arc> { +/// What is displayed: the working file plus, in memory only and never saved, the appearances +/// PdfCraft draws for comments that have none ([`pdfcraft_annot::with_missing_appearances`]) and +/// for form fields the file leaves to the viewer (#3470), and the signature states drawn into +/// named-layer signature appearances ([`pdfcraft_sign::show_status_layers`]). The working file is +/// unchanged, and so is everything that reads it (saving, signatures, …). Image and text exports +/// and Print as Image read these bytes, so they show the page as displayed; validation badges +/// belong to the viewer and never enter them. +/// +/// `signatures` is input and output: each one whose state gets drawn comes back marked +/// (`verdict_shown`). A mark describes the bytes returned, so every mark is cleared first and a +/// copy that cannot be written takes its marks with it. +pub(crate) fn display_bytes( + editor: Option<&mut Editor>, + bytes: &Arc>, + signatures: &mut [SignatureInfo], + captions: &StatusCaptions, +) -> Arc> { + for sig in signatures.iter_mut() { + sig.verdict_shown = false; + } let Some(e) = editor else { return bytes.clone() }; let comments = pdfcraft_annot::with_missing_appearances_cached(&e.cos, &mut e.appearances); let fields = pdfcraft_forms::with_field_appearances_cached(comments.as_ref().unwrap_or(&e.cos), &mut e.field_appearances); - let Some(display) = fields.or(comments) else { + if let Some(mut display) = fields.or(comments) { + // Written whether or not a signature changed: the copy exists for the comments and fields. + let _ = pdfcraft_sign::show_status_layers(&mut display, signatures, captions); + return write_display_copy(e, bytes, &display, signatures); + } + // No comment or field needed an appearance, so there is no copy yet: build one only when a + // signature's state would change it. + if !pdfcraft_sign::has_status_layers(&e.cos, signatures) { return bytes.clone(); - }; - match write_incremental(&display, &SaveOptions::default()) { + } + let mut display = e.cos.clone(); + if !pdfcraft_sign::show_status_layers(&mut display, signatures, captions) { + return bytes.clone(); + } + write_display_copy(e, bytes, &display, signatures) +} + +/// Write the display copy, or keep the working file when it cannot be written. The page then +/// shows the file's own artwork, so the marks are cleared. Losing the copy is a repair, so it is +/// logged, once however many refreshes lose it again. +pub(crate) fn write_display_copy( + e: &mut Editor, + bytes: &Arc>, + display: &pdfcraft_cos::Document, + signatures: &mut [SignatureInfo], +) -> Arc> { + match write_incremental(display, &SaveOptions::default()) { Ok(written) => Arc::new(written), Err(err) => { - e.cos.note_repair(format!("missing comment or field appearances could not be drawn ({err})")); + for sig in signatures.iter_mut() { + sig.verdict_shown = false; + } + let line = format!("the display copy could not be written, so pages show the file as stored ({err})"); + if !e.cos.repair_log().contains(&line) { + e.cos.note_repair(line); + } bytes.clone() } } } +/// Carry the marks across a revalidation that leaves the display copy alone, when it may: only +/// when every verdict is the one the copy was drawn with. Returns whether it did; otherwise the +/// copy has to be drawn again. +pub(crate) fn carry_verdict_marks(previous: &[SignatureInfo], next: &mut [SignatureInfo]) -> bool { + let unchanged = previous.len() == next.len() + && previous.iter().zip(next.iter()).all(|(p, n)| p.field == n.field && p.revision == n.revision && p.status == n.status); + if unchanged { + for (p, n) in previous.iter().zip(next.iter_mut()) { + n.verdict_shown = p.verdict_shown; + } + } + unchanged +} + /// `comments` (the display copy with comment appearances, if it needed any; else the document /// itself) with its form fields drawn from their values where the file leaves that to the viewer. fn with_field_appearances(comments: Option, doc: &pdfcraft_cos::Document) -> Option { @@ -694,17 +758,28 @@ pub(crate) fn info_fields<'a>(info: &'a mut DocInfo, cos: &pdfcraft_cos::Documen ] } -/// The copy PDFCraft displays when opening `bytes`: with appearances drawn, in memory only, for -/// comments that have none and form fields the file leaves to the viewer. Render paths outside the app, such as the CLI, use this so their page -/// images agree with the app and independent viewers. It never changes the source document, and an -/// unparseable file, a write failure or a parser panic falls back to the original bytes. +/// The copy PDFCraft displays when opening `bytes` with no trusted certificates and English +/// captions ([`display_bytes`]): appearances drawn, in memory only, for comments that have none +/// and form fields the file leaves to the viewer, and the state of each named-layer signature. +/// Render paths outside the app, such as the CLI, use this so their page images agree with the app +/// and independent viewers. It never changes the source document: a file with nothing to draw, an +/// unparseable file, a write failure or a parser panic gives back the original bytes. pub fn display_bytes_for(bytes: Arc>, password: Option<&str>) -> Arc> { guard(|| { - pdfcraft_cos::Document::open_with_password(bytes.clone(), password) - .ok() - .and_then(|cos| with_field_appearances(pdfcraft_annot::with_missing_appearances(&cos), &cos)) - .and_then(|display| write_incremental(&display, &SaveOptions::default()).ok()) - .map(Arc::new) + let cos = pdfcraft_cos::Document::open_with_password(bytes.clone(), password).ok()?; + let mut signatures = signatures_of(&cos, &bytes, &TrustStore::default(), &pdfcraft_sign::DigestCache::default()); + let layered = pdfcraft_sign::has_status_layers(&cos, &signatures); + let (mut display, added) = match with_field_appearances(pdfcraft_annot::with_missing_appearances(&cos), &cos) { + Some(display) => (display, true), + None if layered => (cos, false), + None => return None, + }; + // As in the app, a layer recognised but not drawn after all makes no copy. + let drawn = layered && pdfcraft_sign::show_status_layers(&mut display, &mut signatures, &StatusCaptions::default()); + if !(added || drawn) { + return None; + } + write_incremental(&display, &SaveOptions::default()).ok().map(Arc::new) }) .ok() .flatten() diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index 1330faf272..4512bcef43 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -1,8 +1,8 @@ //! Export a PDF ▸ Image and Text (execution plan M10.3, the first formats). //! //! [`Exporter`] renders pages to PNG, JPEG or TIFF at a resolution and extracts reading-order -//! text, from a -//! document's working file (so unsaved edits and hidden layers are respected, as on screen). +//! text, from the copy a document displays ([`Document::export_source`]): its working file, so +//! unsaved edits and hidden layers are respected, with what is drawn for display only. use std::collections::BTreeMap; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -67,9 +67,10 @@ pub struct ExportSource { impl Document { /// What an export reads, for exporting on another thread. These are the displayed - /// bytes, so they may carry missing comment appearances PdfCraft drew. Signature - /// artwork stays original; viewer validation badges are not document content and - /// do not enter image/text exports or Print as Image. Saving uses the working file. + /// bytes, so they may carry appearances PdfCraft drew that the file itself does not hold: + /// missing comment appearances, and the signature states of named-layer signature + /// appearances. Exports and Print as Image follow the page as shown; the viewer's badges + /// are not document content and never enter them. Saving uses the working file. pub fn export_source(&self) -> ExportSource { ExportSource { bytes: self.display.clone(), diff --git a/crates/engine/src/js.rs b/crates/engine/src/js.rs index cc78785815..f581504204 100644 --- a/crates/engine/src/js.rs +++ b/crates/engine/src/js.rs @@ -429,12 +429,16 @@ impl Session { let mut out = Vec::new(); for page in list { let Some(info) = doc.info.pages.get(page) else { continue }; - let res = r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); let shapes = pdfcraft_forms::detect::page_shapes(cos, page); let existing: Vec<[f64; 4]> = doc.form.iter().flat_map(|f| f.widgets.iter()).filter(|w| w.page == Some(page)).map(|w| w.rect).collect(); let (found, turn) = if shapes.boxes.is_empty() && shapes.rules.is_empty() && crate::scan_fields::is_scan(doc, page, info) { - self.detect_on_scan(id, &mut r, page, res.text.as_deref(), &existing, &taken) + // Labels come from the page's own text. Words that only /ActualText names (a stamp + // drawn as outlines) are no labels, and leave the scan to be read. + let own = crate::ocr::own_text(&mut r, page).ok(); + self.detect_on_scan(id, &mut r, page, own.as_ref(), &existing, &taken) } else { + let res = + r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); let words = res.text.map(|t| detection_words(&t, info)).unwrap_or_default(); (pdfcraft_forms::detect::detect(&words, &shapes, &existing, &taken), 0) }; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 378f9483b2..4fb8d8a472 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -130,7 +130,7 @@ pub use pdfcraft_redact::patterns::{ }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; pub use pdfcraft_sign as sign; -pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; +pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, StatusCaptions, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; pub use pdfcraft_xfdf::{Outline as CommentOutline, comment_count, comment_outlines, comments_bounds, is_xfdf_comments}; @@ -141,7 +141,7 @@ pub use comments::{SummarySort, comment_kind, comment_summary, comment_type_name #[cfg(test)] use document::page_runs; pub use document::{Document, EditNotice, TextNotice, display_bytes_for, render_password_for}; -use document::{display_bytes, info_fields, recovered_render_password, same_pages, shown_under_redactions, undrawn_notices}; +use document::{carry_verdict_marks, display_bytes, info_fields, recovered_render_password, same_pages, shown_under_redactions, undrawn_notices}; pub use edit::{BATCH_LABELS, EDIT_LABELS, Edit}; pub use error::{EditError, SourceProblem}; pub use fill_sign::{displayed_offset, text_box_at, typed_signature_graphic, typed_signature_shape}; diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index 18ea331d41..c4854d7e9f 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -11,6 +11,7 @@ use std::sync::{Arc, Mutex}; #[cfg(not(target_arch = "wasm32"))] use std::time::{Duration, Instant}; +use pdfcraft_render::text::TextFilter; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; pub use pdfcraft_ocr::{ @@ -161,6 +162,18 @@ const ALREADY_ON_PAGE: f32 = 0.5; /// is a resolution on paper this size or smaller. const SHEET: f32 = 842.0; +/// The text of `page`'s own: its glyphs, without the words drawn as outlines or pictures that only +/// `/ActualText` names. +pub(crate) fn own_text(r: &mut PageRenderer, page: usize) -> Result { + r.filtered_text(page, TextFilter { painted: true, ..Default::default() }) +} + +/// Whether `page` has text of its own ([`own_text`]). A stamp drawn as outlines on a scan still +/// leaves the scan to read; its words are not read twice, since they count as already on the page. +fn has_own_text(r: &mut PageRenderer, page: usize) -> bool { + own_text(r, page).is_ok_and(|t| t.glyphs.iter().any(|g| !g.text.trim().is_empty())) +} + /// How much of `word` the `text` boxes cover, from 0 to 1. Boxes are `[x0, y0, x1, y1]` in page /// view space. Overlaps between the text boxes are counted twice (glyphs rarely overlap), so the /// sum is capped at the whole word. @@ -702,13 +715,14 @@ impl OcrJob { } let Some(info) = self.infos.get(page) else { continue }; let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; - // The boxes of the page's own text (an earlier OCR layer's included), in view space. + // The boxes of all text on the page, in view space: an earlier OCR layer's, and words + // only /ActualText names, so neither is read again. let existing: Vec<[f32; 4]> = r .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) .text .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) .unwrap_or_default(); - if self.settings.skip_text_pages && !self.settings.force_ocr && !existing.is_empty() { + if self.settings.skip_text_pages && !self.settings.force_ocr && !existing.is_empty() && has_own_text(&mut r, page) { out.push(skip("the page already contains text")); continue; } @@ -1118,6 +1132,34 @@ mod tests { use super::*; use crate::export; + /// One 300 × 300 page drawing `content`, with Helvetica as `/F1`. + fn page(content: &str) -> Arc> { + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + content.len() + ); + Arc::new(pdf.into_bytes()) + } + + #[test] + fn a_stamp_drawn_as_outlines_leaves_a_scan_to_read() { + let stamp = page("/Span <> BDC 100 40 60 20 re f EMC"); + let mut r = PageRenderer::new(stamp, RenderConfig::default()); + assert!(!has_own_text(&mut r, 0), "words only /ActualText names are no text of the page's own"); + // They still count as on the page, so reading the scan doesn't add them twice. + let text = r.render(RenderRequest { page: 0, kind: RequestKind::Text, scale: 1.0, ..Default::default() }).text.unwrap(); + assert_eq!(text.plain_text(), "APPROVED"); + } + + #[test] + fn printed_glyphs_count_as_own_text() { + let mut r = PageRenderer::new(page("BT /F1 12 Tf 20 150 Td (Total) Tj ET"), RenderConfig::default()); + assert!(has_own_text(&mut r, 0)); + } + /// #1028: a recognised word over the page's own text is that text again; one in an empty /// part of the page (a picture) is new. #[test] diff --git a/crates/engine/src/open.rs b/crates/engine/src/open.rs index 32db07abee..80a5ad30c7 100644 --- a/crates/engine/src/open.rs +++ b/crates/engine/src/open.rs @@ -7,8 +7,8 @@ use pdfcraft_render::{DocInfo, OpenError, RenderConfig, RenderPool, inspect}; use crate::history::{Editor, Keys, TextBlocksCache}; use crate::{ - AppearanceCache, DocId, Document, Session, TrustStore, XFA_SCRIPTS_OFF, XfaLayout, display_bytes, display_config, guard, info_fields, js, - note_warnings, recovered_render_password, render_threads, same_pages, signatures_of, xfa, xfa_layout, xfa_values_from_datasets, + AppearanceCache, DocId, Document, Session, StatusCaptions, TrustStore, XFA_SCRIPTS_OFF, XfaLayout, display_bytes, display_config, guard, + info_fields, js, note_warnings, recovered_render_password, render_threads, same_pages, signatures_of, xfa, xfa_layout, xfa_values_from_datasets, }; /// Immutable session state needed while a document is prepared away from its owning thread. @@ -17,6 +17,7 @@ pub struct OpenContext { now: Option, js_off: bool, trust: Arc, + captions: Arc, xfa_time_scale: u32, } @@ -54,7 +55,13 @@ impl OpenContext { impl Session { /// Snapshot the session state that affects open-time scripts, dates and signature trust. pub fn open_context(&self) -> OpenContext { - OpenContext { now: self.now(), js_off: self.js_off, trust: self.trust.clone(), xfa_time_scale: self.xfa_time_scale.unwrap_or(1) } + OpenContext { + now: self.now(), + js_off: self.js_off, + trust: self.trust.clone(), + captions: self.captions.clone(), + xfa_time_scale: self.xfa_time_scale.unwrap_or(1), + } } /// Prepare a document without adding it to this session. @@ -74,6 +81,14 @@ impl Session { let id = DocId(self.next_id); let mut document = prepared.document; document.id = id; + // Trust or the interface language may have changed while it was prepared on another + // thread. + if !Arc::ptr_eq(&document.trust, &self.trust) { + document.captions = self.captions.clone(); + Self::revalidate(&mut document, self.trust.clone()); + } else if *document.captions != *self.captions { + Self::draw_captions(&mut document, self.captions.clone()); + } self.docs.push(document); id } @@ -253,7 +268,7 @@ impl Session { // XFA notes survive the document being re-read after edits. let xfa_warnings: Vec = if info.xfa.is_some() { info.warnings.clone() } else { Vec::new() }; let mut document = - Self::build_document(&context.trust, DocumentOpenData { name, path, bytes, info, structure: cos, render_password, password, xfa }); + Self::build_document(context, DocumentOpenData { name, path, bytes, info, structure: cos, render_password, password, xfa }); // A script that ran away on open turns the form's scripts off for this document. document.xfa_template = if ran_away { None } else { xfa_template }; document.js_output.append(script_output); @@ -266,7 +281,7 @@ impl Session { } /// The last step of opening: build the document record and register it. - fn build_document(trust: &Arc, data: DocumentOpenData<'_>) -> Document { + fn build_document(context: &OpenContext, data: DocumentOpenData<'_>) -> Document { let DocumentOpenData { name, path, bytes, mut info, structure: cos, render_password, password, xfa } = data; let config = RenderConfig { password: render_password.as_deref().map(Arc::from), ..Default::default() }; let (mut editor, read_only_reason) = match cos { @@ -293,7 +308,7 @@ impl Session { Err(_) => (None, Some("the document structure could not be read for editing".into())), }; let sig_cache = Arc::new(pdfcraft_sign::DigestCache::default()); - let signatures = editor.as_ref().map(|e| signatures_of(&e.cos, &bytes, trust, &sig_cache)).unwrap_or_default(); + let mut signatures = editor.as_ref().map(|e| signatures_of(&e.cos, &bytes, &context.trust, &sig_cache)).unwrap_or_default(); // The inspection reads only `/Info`; a file stating its title or author only in XMP still // shows them (#2373). if let Some(e) = &editor { @@ -303,7 +318,7 @@ impl Session { } } } - let display = display_bytes(editor.as_mut(), &bytes); + let display = display_bytes(editor.as_mut(), &bytes, &mut signatures, &context.captions); let renderer = RenderPool::new(display.clone(), render_threads(), display_config(&config)); let (form, adopted) = editor .as_ref() @@ -341,7 +356,8 @@ impl Session { marker_scan: OnceLock::new(), links: OnceLock::new(), signatures: Arc::new(signatures), - trust: trust.clone(), + trust: context.trust.clone(), + captions: context.captions.clone(), sig_cache, editor, config, diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index 9a4cb04dac..e5029ccbd6 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -11,11 +11,11 @@ use crate::edit::{EditCtx, check_permission, run_edit}; use crate::history::{Keys, MAX_UNDO, Scope, keys_after, scope_of, uses_scripts}; use crate::{ BookmarkPage, BookmarkRow, CombineSource, Combined, DataFormat, DocId, Document, Edit, EditError, Hidden, ImageResolution, - LOW_RESOLUTION_PRINT_DPI, Label, ObjectKind, OptimizeReport, RecoverySnapshot, SignOptions, SourceKind, SplitPart, SummarySort, TrustStore, - XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, apply_layer_state, comment_list, comment_summary, decode_text, display_bytes, display_config, guard, - info_fields, js, local_utc_offset, note_warnings, open_source, open_source_with, optimize, outline_rows, parse_ymd, print, print_raster, - render_threads, scan_colours, set_default_layer_state, shown_under_redactions, signatures_of, source_kind, sync_xfa_state, undrawn_notices, - use_layer_choices, writable, xfa, xfa_sync_datasets, + LOW_RESOLUTION_PRINT_DPI, Label, ObjectKind, OptimizeReport, RecoverySnapshot, SignOptions, SourceKind, SplitPart, StatusCaptions, SummarySort, + TrustStore, XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, apply_layer_state, carry_verdict_marks, comment_list, comment_summary, decode_text, + display_bytes, display_config, guard, info_fields, js, local_utc_offset, note_warnings, open_source, open_source_with, optimize, outline_rows, + parse_ymd, print, print_raster, render_threads, scan_colours, set_default_layer_state, shown_under_redactions, signatures_of, source_kind, + sync_xfa_state, undrawn_notices, use_layer_choices, writable, xfa, xfa_sync_datasets, }; mod editing; @@ -36,6 +36,8 @@ pub struct Session { utc_offset: Option i64>, /// Certificates trusted for signing (Acrobat: Trusted Certificates). pub(crate) trust: Arc, + /// The words signature status layers are drawn with, in the interface language. + pub(crate) captions: Arc, /// Preferences ▸ JavaScript ▸ Enable Acrobat JavaScript, inverted (on by default). pub(crate) js_off: bool, /// Preferences ▸ Date format, when not the default (see [`dates`]). diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index be7fd5aca5..8ab3ca22fa 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -2,9 +2,9 @@ use super::check_refresh_panic; use super::{ Arc, DocId, Document, Edit, EditCtx, EditError, Label, MAX_UNDO, ObjectKind, OnceLock, RenderPool, SaveOptions, Scope, Session, - XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, check_permission, comment_list, display_bytes, display_config, guard, info_fields, inspect, js, keys_after, - note_warnings, render_threads, run_edit, scan_colours, scope_of, shown_under_redactions, signatures_of, sync_xfa_state, undrawn_notices, - uses_scripts, write_incremental, xfa, xfa_sync_datasets, + XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, carry_verdict_marks, check_permission, comment_list, display_bytes, display_config, guard, info_fields, + inspect, js, keys_after, note_warnings, render_threads, run_edit, scan_colours, scope_of, shown_under_redactions, signatures_of, sync_xfa_state, + undrawn_notices, uses_scripts, write_incremental, xfa, xfa_sync_datasets, }; use crate::{EditNotice, MarkKind}; @@ -342,7 +342,14 @@ impl Session { #[cfg(test)] check_refresh_panic(); if !doc.signatures.is_empty() { - let signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + let mut signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + // The display copy is kept for a metadata edit, and so are its marks, unless the + // edit changed a verdict (a certification can forbid it): the page then draws the + // new one. + if !carry_verdict_marks(&doc.signatures, &mut signatures) { + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } doc.signatures = Arc::new(signatures); } doc.bytes = bytes; @@ -386,8 +393,8 @@ impl Session { } doc.info.file_size = bytes.len(); doc.form = form; - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; @@ -444,8 +451,8 @@ impl Session { doc.marker_scan = OnceLock::new(); } doc.info.file_size = bytes.len(); - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; @@ -467,6 +474,13 @@ impl Session { doc.renderer = RenderPool::new_inline(Arc::new(Vec::new()), doc.config.clone()); doc.display = original.clone(); doc.bytes = original; + // The page shows the original's own artwork again, so nothing counts as drawn, even if + // the refresh that follows fails. + if doc.signatures.iter().any(|s| s.verdict_shown) { + for sig in Arc::make_mut(&mut doc.signatures) { + sig.verdict_shown = false; + } + } #[cfg(test)] check_refresh_panic(); } @@ -528,8 +542,8 @@ impl Session { if invalidate_links { doc.links = OnceLock::new(); } - let signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index 44bed85cde..f4e25bc2f6 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -170,6 +170,11 @@ impl Session { if !doc.config.layers.is_empty() { set_default_layer_state(&mut cos, &doc.info.layers); } + // Signature states as the screen shows them, so not when its display copy could not be + // written. This copy is only printed, never saved. + if doc.signatures.iter().any(|s| s.verdict_shown) { + let _ = pdfcraft_sign::show_status_layers(&mut cos, &mut doc.signatures.to_vec(), &doc.captions); + } pdfcraft_print::impose(&cos, settings).map_err(|e| EditError::Print(e.to_string())) } diff --git a/crates/engine/src/session/save.rs b/crates/engine/src/session/save.rs index 1e6918f6a6..c67cd391b9 100644 --- a/crates/engine/src/session/save.rs +++ b/crates/engine/src/session/save.rs @@ -1,6 +1,6 @@ use super::{ - Arc, DocId, EditError, Keys, OpenError, RecoverySnapshot, RenderPool, SaveOptions, Session, SignOptions, TrustStore, display_bytes, - display_config, guard, render_threads, signatures_of, writable, write_full, write_incremental, xfa_sync_datasets, + Arc, DocId, Document, EditError, Keys, OpenError, RecoverySnapshot, RenderPool, SaveOptions, Session, SignOptions, StatusCaptions, TrustStore, + display_bytes, display_config, guard, render_threads, signatures_of, writable, write_full, write_incremental, xfa_sync_datasets, }; impl Session { @@ -242,17 +242,55 @@ impl Session { pub fn set_trusted_certificates(&mut self, certs: Vec) { self.trust = Arc::new(TrustStore { certs }); for doc in &mut self.docs { - doc.trust = self.trust.clone(); - let Some(editor) = doc.editor.as_mut() else { continue }; - // Keep verdict refresh and renderer refresh together. Viewer caches are - // invalidated by the UI trust path; trust alone changes no document artwork. - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &doc.bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &doc.bytes); - doc.signatures = Arc::new(signatures); - doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + Self::revalidate(doc, self.trust.clone()); } } + /// Validate `doc`'s signatures against `trust`, and rebuild its display copy and renderer + /// with them: the verdicts choose what the copy draws. An unsigned document has nothing to + /// validate and keeps both. Viewer caches are invalidated by the UI trust path. + pub(crate) fn revalidate(doc: &mut Document, trust: Arc) { + doc.trust = trust; + if doc.signatures.is_empty() { + return; + } + let Some(editor) = doc.editor.as_mut() else { return }; + let mut signatures = signatures_of(&editor.cos, &doc.bytes, &doc.trust, &doc.sig_cache); + doc.display = display_bytes(Some(editor), &doc.bytes, &mut signatures, &doc.captions); + doc.signatures = Arc::new(signatures); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } + + /// The words the caption of a named-layer signature appearance is drawn with + /// ([`pdfcraft_sign::show_status_layers`]), in the interface language. When they change, every + /// open document's display copy and renderer are rebuilt; like trust, this is not an edit. + /// Returns whether they changed. + pub fn set_signature_captions(&mut self, captions: StatusCaptions) -> bool { + if *self.captions == captions { + return false; + } + self.captions = Arc::new(captions); + for doc in &mut self.docs { + Self::draw_captions(doc, self.captions.clone()); + } + true + } + + /// Draw `doc`'s signature status layers with `captions`: the display copy and renderer are + /// rebuilt when it draws a signature's state (working file, generation and dirty unchanged). + /// Whether a state is drawn does not depend on the words: a caption that can't be drawn in + /// them is drawn in English. + pub(crate) fn draw_captions(doc: &mut Document, captions: Arc) { + doc.captions = captions; + if !doc.signatures.iter().any(|s| s.verdict_shown) { + return; + } + let Some(editor) = doc.editor.as_mut() else { return }; + let signatures = Arc::make_mut(&mut doc.signatures); + doc.display = display_bytes(Some(editor), &doc.bytes, signatures, &doc.captions); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } + /// The signing time as a PDF date in local time with its offset (`D:…+02'00'`). pub fn signing_date(&self) -> String { let offset = self.local_offset(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c109a0bc80..d230d098de 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use super::document::is_placeholder_title; +use super::document::{is_placeholder_title, write_display_copy}; use super::edit::{EditCtx, check_permission, run_edit}; use super::*; use pdfcraft_render::{Layer, LayerOp, OpenError, ReadingColors}; @@ -3116,6 +3116,26 @@ fn a_word_in_a_font_set_by_an_extgstate_is_redacted() { ); } +/// A word drawn as outlines reads through its /ActualText. A mark over part of it applies (the +/// outlines are clipped), and the name that would still read the whole word is gone from the page +/// and from the saved file. +#[test] +fn a_word_drawn_as_outlines_is_redacted_with_its_name() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page("", "/Span <> BDC 20 150 m 140 150 l 140 170 l 20 170 l h f EMC"); + let id = s.open("outlines.pdf", None, Arc::new(page), None).unwrap(); + assert_eq!(page_texts(&s, id), ["SECRET42"]); + let shape = + Shape::Redact { quads: vec![pdfcraft_annot::rect_quad([10.0, 140.0, 90.0, 180.0])], overlay: String::new(), look: Default::default() }; + let mark = + Edit::AddAnnotation(NewAnnotation { page: 0, style: Style::default_for(&shape), shape, contents: String::new(), author: "Ada".into() }); + s.apply(id, mark).unwrap(); + s.apply(id, Edit::ApplyRedactions { pages: None }).unwrap(); + assert_eq!(page_texts(&s, id), [""]); + let saved = s.save_bytes(id).unwrap(); + assert!(!saved.windows(b"SECRET42".len()).any(|w| w == b"SECRET42"), "the name is not left in the file"); +} + #[test] fn redaction_marks_apply_for_good_and_undo() { let (mut s, id) = session_with(2); @@ -4855,6 +4875,17 @@ fn bytes_has(haystack: &[u8], needle: &[u8]) -> bool { /// ranges and embed a detached CMS over them. The signed bytes keep the /// prepared appearance, so they validate exactly like a product signer's file. fn sign_prepared(doc: &mut pdfcraft_cos::Document, id: &sign::pkcs12::DigitalId, date: &str) -> Vec { + sign_prepared_with(doc, id, date, |_| {}) +} + +/// [`sign_prepared`], with `value` adding to the signature dictionary first (a certification's +/// `/Reference`, say). +fn sign_prepared_with( + doc: &mut pdfcraft_cos::Document, + id: &sign::pkcs12::DigitalId, + date: &str, + value: impl FnOnce(&mut pdfcraft_cos::Dict), +) -> Vec { use pdfcraft_cos::{Dict, Object, PdfString}; let reserve = 8192 + id.certificate.raw.len() + id.chain.iter().map(|c| c.raw.len()).sum::(); let mut v = Dict::new(); @@ -4864,6 +4895,7 @@ fn sign_prepared(doc: &mut pdfcraft_cos::Document, id: &sign::pkcs12::DigitalId, v.set(b"ByteRange".to_vec(), Object::Array([0].iter().chain(LAYERED_BR_MARK.iter()).map(|n| Object::Int(*n)).collect())); v.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0; reserve], hex: true })); v.set(b"M".to_vec(), PdfString::literal(date.as_bytes().to_vec())); + value(&mut v); let sig = doc.add(Object::Dict(v)); // The field by name, never by number: a full save (encryption, repairs) is // free to renumber objects, and a hardcoded reference would stamp the value @@ -5104,6 +5136,442 @@ fn real_layered_signatures_round_trip_behind_a_user_password() { assert_eq!(t.get(tid).unwrap().signatures[0].status, SignatureStatus::Valid); } +/// A one-page file whose unsigned signature field carries a named-layer appearance, in the +/// geometry of a real one scaled by 4 (every word and drawing synthetic): /N draws /FRM, which +/// draws n0–n4; n1 a yellow question mark and n4 "Signature Not Verified". Object numbers: +/// 1 catalog, 2 pages, 3 page (400×300), 4 field, 5 AcroForm, 6 /N, 7 /FRM, 8–12 n0–n4, +/// 13 Helvetica, 14 page content. `sign_prepared` adds the value. +fn named_layer_signable_fixture() -> Vec { + named_layer_signable_fixture_with("[0 0 100 100]") +} + +/// [`named_layer_signable_fixture`] with `n1_bbox` as its n1 layer's `/BBox`. +fn named_layer_signable_fixture_with(n1_bbox: &str) -> Vec { + fn form(bbox: &str, resources: &str, ops: &str) -> String { + format!("<< /Type /XObject /Subtype /Form /BBox {bbox} /Resources {resources} /Length {} >>\nstream\n{ops}\nendstream", ops.len()) + } + let helv = "<< /Font << /F1 13 0 R >> >>"; + let glyph = "1.08 0 0 1.08 46 6 cm"; + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 5 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 300] /Contents 14 0 R /Resources << /Font << /F1 13 0 R >> >> /Annots [4 0 R] >>" + .to_string(), + "<< /FT /Sig /T (Signature1) /Type /Annot /Subtype /Widget /F 132 /Rect [100 100 300 220] /P 3 0 R /AP << /N 6 0 R >> >>".to_string(), + "<< /Fields [4 0 R] /SigFlags 3 >>".to_string(), + form("[0 0 200 120]", "<< /XObject << /FRM 7 0 R >> >>", "q 1 0 0 1 0 0 cm /FRM Do Q"), + form( + "[0 0 200 120]", + "<< /XObject << /n0 8 0 R /n1 9 0 R /n2 10 0 R /n3 11 0 R /n4 12 0 R >> >>", + &format!("q /n0 Do Q q {glyph} /n1 Do Q q /n2 Do Q q {glyph} /n3 Do Q q /n4 Do Q"), + ), + form("[0 0 100 100]", "<< >>", "% DSBlank"), + form( + n1_bbox, + "<< >>", + "q 1 0.84 0.2 rg 0.55 0.4 0 RG 3 w 30 72 m 30 88 42 98 56 98 c 72 98 84 88 84 74 c 66 50 l 58 28 l 44 28 l 44 48 l h B 44 6 14 14 re B Q", + ), + form("[0 0 200 120]", helv, "BT /F1 13.6 Tf 1 0 0 1 8 62.4 Tm (Digitally signed by) Tj 1 0 0 1 8 48.8 Tm (Test Signer) Tj ET"), + form("[0 0 100 100]", "<< >>", "% DSBlank"), + form("[0 84 200 120]", helv, "BT /F1 18.8 Tf 1 0 0 1 8 93.2 Tm (Signature Not Verified) Tj ET"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_string(), + "<< /Length 44 >>\nstream\nBT /F1 14 Tf 20 270 Td (Contract text) Tj ET\nendstream".to_string(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (k, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", k + 1).as_bytes()); + } + let start = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in &offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{start}\n%%EOF", objs.len() + 1).as_bytes()); + out +} + +/// The status colours as the regenerated icons draw them. +const VALID_GREEN: [u8; 3] = [0x2D, 0x9D, 0x78]; +const UNKNOWN_AMBER: [u8; 3] = [0xE6, 0x86, 0x19]; +const INVALID_RED: [u8; 3] = [0xD7, 0x37, 0x3F]; + +/// How many pixels of `rgba` are within 40 of `rgb` on every channel. A regenerated icon's +/// stroke covers thousands at 1 px/pt; the fixture holds none of these colours otherwise. +fn pixels_near(rgba: &[u8], rgb: [u8; 3]) -> usize { + rgba.as_chunks::<4>().0.iter().filter(|p| p.iter().zip(rgb).all(|(a, b)| a.abs_diff(b) <= 40)).count() +} + +/// The displayed page at 1 px/pt. +fn displayed_pixels(s: &Session, id: DocId) -> Vec { + let doc = s.get(id).unwrap(); + let mut renderer = pdfcraft_render::PageRenderer::new(doc.display.clone(), doc.config.clone()); + let r = renderer.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(r.error.is_none(), "{:?}", r.error); + r.rgba.to_vec() +} + +fn exported_text(s: &Session, id: DocId) -> String { + export::Exporter::new(s.get(id).unwrap()).text_of(&[0]).unwrap() +} + +/// The test identity and the certificate that makes it trusted. +fn test_identity() -> (sign::pkcs12::DigitalId, sign::x509::Certificate) { + let p12 = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/../sign/tests/data/ec-p256.p12")).unwrap(); + let id = sign::pkcs12::open(&p12, "test").unwrap(); + let anchor = id.chain.first().cloned().unwrap_or_else(|| id.certificate.clone()); + (id, anchor) +} + +/// A named-layer signature through `Session`: the unknown state drawn on opening, the valid one +/// once trusted, the signed bytes saved unchanged and still valid on reopening. +fn check_named_layer_states(signed: Arc>, password: Option<&str>, anchor: &sign::x509::Certificate) { + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("named.pdf", None, signed.clone(), password).unwrap(); + let sig = &s.get(id).unwrap().signatures[0]; + assert_eq!(sig.status, SignatureStatus::Unknown, "untrusted at first"); + assert!(sig.verdict_shown, "the appearance shows the unknown state"); + let text = exported_text(&s, id); + assert!(text.contains("Validity unknown") && !text.contains("Not Verified"), "{text:?}"); + let amber = pixels_near(&displayed_pixels(&s, id), UNKNOWN_AMBER); + assert!(amber > 500, "{amber} amber pixels"); + s.set_trusted_certificates(vec![anchor.clone()]); + let doc = s.get(id).unwrap(); + assert_eq!(doc.signatures[0].status, SignatureStatus::Valid); + assert!(doc.signatures[0].verdict_shown); + assert!(!doc.dirty, "trust is not an edit"); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); + let page = displayed_pixels(&s, id); + assert!(pixels_near(&page, VALID_GREEN) > 500 && pixels_near(&page, UNKNOWN_AMBER) < 50, "the page redraws for the new state"); + // Saving writes the working file, which never held the drawn state. + let saved = s.save_bytes(id).unwrap(); + assert!(saved == signed, "an unchanged save is the signed file exactly"); + let mut t = Session::new().with_clock(|| 1_800_000_000); + t.set_trusted_certificates(vec![anchor.clone()]); + let tid = t.open("named-saved.pdf", None, saved, password).unwrap(); + let sig = &t.get(tid).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Valid && sig.verdict_shown); + let exported = exported_text(&t, tid); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn named_layer_signatures_show_their_state_through_trust_save_and_reopen() { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + check_named_layer_states(signed, None, &anchor); +} + +#[test] +fn named_layer_signatures_round_trip_behind_a_user_password() { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes128, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let encrypted = Arc::new(pdfcraft_cos::write_full(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap()); + let mut doc = pdfcraft_cos::Document::open_with_password(encrypted, Some("user")).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + check_named_layer_states(signed, Some("user"), &anchor); +} + +#[test] +fn a_tampered_named_layer_signature_shows_the_invalid_state() { + let (id, _) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let mut signed = sign_prepared(&mut doc, &id, "D:20261002120000+01'00'"); + let at = signed.windows(15).position(|w| w == b"(Contract text)").unwrap(); + signed[at + 10] = b'X'; + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("tampered.pdf", None, Arc::new(signed), None).unwrap(); + let sig = &s.get(id).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Invalid && sig.verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature invalid"), "{exported}"); + let red = pixels_near(&displayed_pixels(&s, id), INVALID_RED); + assert!(red > 500, "{red} red pixels"); +} + +/// A trusted named-layer signature. +fn trusted_named_layer_session() -> (Session, DocId) { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let vid = s.open("named.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + assert_eq!(s.get(vid).unwrap().signatures[0].status, SignatureStatus::Valid); + (s, vid) +} + +#[test] +fn named_layer_states_reach_exports_and_both_print_modes() { + // What is drawn on the page is what leaves it. + let (s, id) = trusted_named_layer_session(); + let doc = s.get(id).unwrap(); + let source = doc.export_source(); + let mut file = export::Exporter::from_source(export::ExportSource { bytes: doc.bytes.clone(), ..source.clone() }); + let mut shown = export::Exporter::from_source(source); + assert_ne!(shown.png(0, 72.0).unwrap(), file.png(0, 72.0).unwrap(), "image export follows the display copy"); + let text = shown.text_of(&[0]).unwrap(); + assert!(text.contains("Signature valid") && !text.contains("Not Verified"), "{text:?}"); + let mut settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; + let (_, _, vector) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + let (_, _, raster) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + for (mode, pixels) in [("vector", &vector), ("raster", &raster)] { + assert!(pixels_near(pixels, VALID_GREEN) > 500, "{mode} print draws the valid state"); + assert!(pixels_near(pixels, [255, 214, 51]) < 50, "{mode} print drops the stored question mark"); + } +} + +#[test] +fn named_layers_show_beside_comments_that_need_appearances() { + // A comment without an appearance makes the display copy for its own sake; the signature's + // state is drawn into that same copy. + use pdfcraft_cos::{ObjRef, Object}; + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let square = cos.add(pdfcraft_cos::Lexer::new(b"<< /Type /Annot /Subtype /Square /Rect [10 10 60 60] /C [1 0 0] >>", 0).object().unwrap()); + cos.update_dict(ObjRef::new(3, 0), |page| page.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(square)]))) + .unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("commented.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let shown = pdfcraft_cos::Document::open(s.get(id).unwrap().display.clone()).unwrap(); + assert!(shown.get(square).as_dict().is_some_and(|d| d.contains(b"AP")), "the comment's appearance is in the same copy"); + assert!(s.get(id).unwrap().signatures[0].verdict_shown); + let green = pixels_near(&displayed_pixels(&s, id), VALID_GREEN); + assert!(green > 500, "{green} green pixels"); +} + +#[test] +fn named_layers_show_beside_fields_the_viewer_draws() { + // A text field without an appearance makes the display copy for its own sake; the + // signature's state is drawn into that same copy. + use pdfcraft_cos::{Lexer, ObjRef, Object}; + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let field = cos.add( + Lexer::new( + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /V (Test Signer) /DA (/F1 12 Tf 0 g) /Rect [20 20 380 40] /F 4 /P 3 0 R >>", + 0, + ) + .object() + .unwrap(), + ); + let both = Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(field)]); + cos.update_dict(ObjRef::new(3, 0), |page| page.set(b"Annots".to_vec(), both.clone())).unwrap(); + let resources = Lexer::new(b"<< /Font << /F1 13 0 R >> >>", 0).object().unwrap(); + cos.update_dict(ObjRef::new(5, 0), |form| { + form.set(b"Fields".to_vec(), both); + form.set(b"DR".to_vec(), resources); + }) + .unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("filled.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let shown = pdfcraft_cos::Document::open(s.get(id).unwrap().display.clone()).unwrap(); + assert!(shown.get(field).as_dict().is_some_and(|d| d.contains(b"AP")), "the field's appearance is in the same copy"); + assert!(s.get(id).unwrap().signatures[0].verdict_shown); + let green = pixels_near(&displayed_pixels(&s, id), VALID_GREEN); + assert!(green > 500, "{green} green pixels"); +} + +#[test] +fn a_metadata_edit_keeps_the_regenerated_state() { + let (mut s, id) = trusted_named_layer_session(); + s.apply(id, Edit::SetInfo { key: "Title".into(), value: "Contract".into() }).unwrap(); + let doc = s.get(id).unwrap(); + assert!(doc.signatures[0].verdict_shown, "the page still draws the state"); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn a_metadata_edit_that_changes_a_verdict_redraws_it() { + // A certification that allows no changes: setting the title invalidates the signature, and + // the page has to say so rather than keep the valid state it drew. + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = sign_prepared_with(&mut cos, &signer, "D:20261002120000+01'00'", |v| { + let reference = + pdfcraft_cos::Lexer::new(b"[<< /Type /SigRef /TransformMethod /DocMDP /TransformParams << /Type /TransformParams /P 1 /V /1.2 >> >>]", 0) + .object() + .unwrap(); + v.set(b"Reference".to_vec(), reference); + }); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("certified.pdf", None, Arc::new(signed), None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + assert_eq!(s.get(id).unwrap().signatures[0].status, SignatureStatus::Valid); + s.apply(id, Edit::SetInfo { key: "Title".into(), value: "Changed".into() }).unwrap(); + let doc = s.get(id).unwrap(); + assert_eq!(doc.signatures[0].status, SignatureStatus::Invalid); + assert!(doc.signatures[0].verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature invalid"), "{exported}"); + let red = pixels_near(&displayed_pixels(&s, id), INVALID_RED); + assert!(red > 500, "{red} red pixels"); +} + +#[test] +fn verdict_marks_carry_over_only_while_every_verdict_is_unchanged() { + let (s, id) = trusted_named_layer_session(); + let previous = s.get(id).unwrap().signatures.to_vec(); + assert!(previous[0].verdict_shown); + let mut same = previous.clone(); + same[0].verdict_shown = false; + assert!(carry_verdict_marks(&previous, &mut same) && same[0].verdict_shown); + let mut changed = previous.clone(); + (changed[0].verdict_shown, changed[0].status) = (false, SignatureStatus::Invalid); + assert!(!carry_verdict_marks(&previous, &mut changed) && !changed[0].verdict_shown); + let mut later = previous.clone(); + (later[0].verdict_shown, later[0].revision) = (false, previous[0].revision + 1); + assert!(!carry_verdict_marks(&previous, &mut later) && !later[0].verdict_shown); +} + +#[test] +fn releasing_the_working_file_drops_the_marks() { + // The page shows the original's own artwork until the refresh finishes. + let (mut s, id) = trusted_named_layer_session(); + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + assert!(doc.signatures[0].verdict_shown); + Session::release_working(doc); + assert!(!doc.signatures[0].verdict_shown); +} + +#[test] +fn background_open_uses_the_current_trust_at_commit() { + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let prepared = s.open_context().prepare_open("background.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let id = s.commit_open(prepared); + let sig = &s.get(id).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Valid && sig.verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn background_open_uses_the_current_trust_and_language_at_commit() { + // Both changed while the document was prepared: the state and its words are both the new ones. + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let prepared = s.open_context().prepare_open("background.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + s.set_signature_captions(StatusCaptions { + valid: "Signatur gültig".into(), + unknown: "Gültigkeit unbekannt".into(), + invalid: "Signatur ungültig".into(), + }); + let id = s.commit_open(prepared); + assert_eq!(s.get(id).unwrap().signatures[0].status, SignatureStatus::Valid); + let exported = exported_text(&s, id); + assert!(exported.contains("Signatur gültig"), "{exported}"); +} + +#[test] +fn printing_draws_signature_states_only_where_the_screen_does() { + let (mut s, id) = trusted_named_layer_session(); + let settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; + // The screen falls back to the file's artwork when its display copy cannot be written, and + // takes the marks with it; print follows. + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + let mut cleared = doc.signatures.to_vec(); + cleared[0].verdict_shown = false; + doc.signatures = Arc::new(cleared); + let (_, _, printed) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + let green = pixels_near(&printed, VALID_GREEN); + assert!(green < 50, "{green} green pixels"); +} + +#[test] +fn a_failed_display_write_drops_the_marks() { + let (mut s, id) = trusted_named_layer_session(); + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + let mut signatures = doc.signatures.to_vec(); + assert!(signatures[0].verdict_shown); + // A stream-limited document is read-only: the writer refuses it. + let unwritable = pdfcraft_cos::Document::open_with_stream_limit(doc.bytes.clone(), None, 1).unwrap(); + let editor = doc.editor.as_mut().unwrap(); + let shown = write_display_copy(editor, &doc.bytes, &unwritable, &mut signatures); + assert!(Arc::ptr_eq(&shown, &doc.bytes), "the page falls back to the file's own artwork"); + assert!(!signatures[0].verdict_shown, "so nothing counts as drawn"); + // Logged once, however many refreshes lose the copy again. + write_display_copy(editor, &doc.bytes, &unwritable, &mut signatures); + let logged = editor.cos.repair_log().iter().filter(|l| l.contains("the display copy could not be written")).count(); + assert_eq!(logged, 1); +} + +#[test] +fn background_open_uses_the_current_caption_language_at_commit() { + let (mut s, id) = trusted_named_layer_session(); + let bytes = s.get(id).unwrap().bytes.clone(); + let prepared = s.open_context().prepare_open("background.pdf", None, bytes, None).unwrap(); + s.set_signature_captions(StatusCaptions { + valid: "Signatur gültig".into(), + unknown: "Gültigkeit unbekannt".into(), + invalid: "Signatur ungültig".into(), + }); + let later = s.commit_open(prepared); + let exported = exported_text(&s, later); + assert!(exported.contains("Signatur gültig"), "{exported}"); + assert!(s.get(later).unwrap().signatures[0].verdict_shown); + assert!(!s.get(later).unwrap().dirty); +} + +#[test] +fn caption_language_follows_the_session_captions() { + let (mut s, id) = trusted_named_layer_session(); + let generation = s.get(id).unwrap().generation; + let german = StatusCaptions { valid: "Signatur gültig".into(), unknown: "Gültigkeit unbekannt".into(), invalid: "Signatur ungültig".into() }; + s.set_signature_captions(german); + let doc = s.get(id).unwrap(); + assert!(!doc.dirty && doc.generation == generation, "captions are not an edit"); + assert!(doc.signatures[0].verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signatur gültig"), "{exported}"); + // Documents opened later draw with them too. + let (signer, _) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let later = s.open("later.pdf", None, Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")), None).unwrap(); + let exported = exported_text(&s, later); + assert!(exported.contains("Signatur gültig"), "{exported}"); +} + +#[test] +fn non_win_ansi_captions_embed_inter_in_the_display_copy_only() { + // Words WinAnsi cannot encode are drawn in Inter, embedded into the display copy only. + let (mut s, id) = trusted_named_layer_session(); + let russian = StatusCaptions { + valid: "Подпись действительна".into(), + unknown: "Действительность неизвестна".into(), + invalid: "Подпись недействительна".into(), + }; + s.set_signature_captions(russian); + let exported = exported_text(&s, id); + assert!(exported.contains("Подпись действительна"), "{exported}"); + let doc = s.get(id).unwrap(); + assert!(bytes_has(&doc.display, b"/BaseFont /Inter") && !bytes_has(&doc.bytes, b"/BaseFont /Inter"), "Inter goes into the display copy only"); +} + #[test] #[ignore = "timing probe"] fn probe_edit_latency_on_a_large_signed_document() { @@ -7654,6 +8122,37 @@ fn free_text_without_appearance() -> Arc> { Arc::new(bytes) } +/// The CLI renders what the app shows with no trusted certificates: a named-layer signature in +/// the state it has then (validity unknown), not the "not verified" its file stores. The source +/// is not changed. +#[test] +fn cli_display_draws_signature_states_as_the_app_does_without_trust() { + let (id, _) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let display = display_bytes_for(signed.clone(), None); + assert!(!Arc::ptr_eq(&display, &signed), "the state is drawn into a copy"); + let mut renderer = pdfcraft_render::PageRenderer::new(display, pdfcraft_render::RenderConfig::default()); + let page = renderer.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(pixels_near(&page.rgba, UNKNOWN_AMBER) > 500, "the unknown state is drawn"); + assert!(pixels_near(&page.rgba, [255, 214, 51]) < 50, "the stored question mark is not"); +} + +/// A layer recognised but too small to draw an icon in (a side so small the icon's scale rounds to +/// zero) changes nothing, so the CLI renders the source, as the app does. +#[test] +fn cli_display_makes_no_copy_when_no_state_could_be_drawn() { + let (id, _) = test_identity(); + let n1 = format!("[0 0 0.{}6 100]", "0".repeat(322)); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture_with(&n1))).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let cos = pdfcraft_cos::Document::open(signed.clone()).unwrap(); + let trust = pdfcraft_sign::TrustStore::default(); + let signatures = crate::doc_state::signatures_of(&cos, &signed, &trust, &pdfcraft_sign::DigestCache::default()); + assert!(pdfcraft_sign::has_status_layers(&cos, &signatures), "the layers are recognised"); + assert!(Arc::ptr_eq(&display_bytes_for(signed.clone(), None), &signed), "nothing was drawn, so there is no copy"); +} + /// #2371: a comment without an appearance stream is displayed by the app, so the CLI's page image /// must show it too. The helper's copy is for rendering only; documents that need no copy, and /// files it can't display, keep exactly the original source bytes. @@ -8514,6 +9013,141 @@ fn a_signed_document_rebuilt_from_object_headers_refuses_edits() { assert!(!s.get(id).unwrap().dirty); } +/// A one-page document whose only signature field is signed and named with `len` bytes. The +/// signature field walk reads names of up to 4096 bytes and passes longer ones over. +fn signed_field_named(len: usize) -> Vec { + pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Sig /T ({}) /V 5 0 R /Rect [20 20 180 60] /P 3 0 R >>", "a".repeat(len)), + "<< /Type /Sig /Filter /Adobe.PPKLite /SubFilter /adbe.pkcs7.detached /ByteRange [0 1 2 1] /Contents <00> >>".into(), + ]) +} + +/// A PDF of `objs`, numbered from 1, with a cross-reference table. +fn pdf_of(objs: &[String]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn open_signed_field_named(len: usize) -> (Session, DocId) { + let mut s = Session::new(); + let id = s.open("form.pdf", None, Arc::new(signed_field_named(len)), None).expect("opens"); + (s, id) +} + +/// The cap's boundary as the engine meets it: a 4096-byte name is read, and its signature blocks a +/// full rewrite. +#[test] +fn a_signature_named_at_the_cap_is_listed_and_blocks_a_full_rewrite() { + let (s, id) = open_signed_field_named(4096); + let doc = s.get(id).unwrap(); + let listed: Vec<_> = doc.signatures.iter().map(|r| (r.field.len(), r.signed)).collect(); + assert_eq!(listed, [(4096, true)], "the field is read, and no row stands for unread ones"); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +/// One byte past the cap the field goes unread: the list says so, and a full rewrite is still +/// refused. +#[test] +fn a_signature_named_one_byte_past_the_cap_still_blocks_a_full_rewrite() { + let (s, id) = open_signed_field_named(4097); + let doc = s.get(id).unwrap(); + let unread = doc.signatures.iter().find(|r| r.field == "UnreadFields"); + let said = unread.map(|r| (r.signed, r.status, r.details.first().is_some_and(|d| d.message.starts_with("This form has too many fields")))); + assert_eq!(said, Some((true, SignatureStatus::Unknown, true)), "{:?}", doc.signatures); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +/// A signed field behind a kid with no /T: one merged with its widget under a named field that +/// isn't a signature field (the walk can't list it, so its row is the unread one), and one under a +/// node with no /T (listed by its full name). Either way the document is signed, and neither a full +/// save nor Optimize may rewrite it. +#[test] +fn signed_fields_behind_kids_without_a_name_block_a_full_rewrite() { + let page = |annot: u32| format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Annots [{annot} 0 R] >>"); + let head = |annot: u32| { + vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + page(annot), + ] + }; + let sig = "<< /Type /Sig /SubFilter /ETSI.CAdES.detached /ByteRange [0 1 2 1] /Contents <00> >>".to_string(); + let mut merged = head(5); + merged.extend([ + "<< /T (parent) /Kids [5 0 R] >>".to_string(), + "<< /Type /Annot /Subtype /Widget /FT /Sig /V 6 0 R /Parent 4 0 R /Rect [20 20 180 60] /P 3 0 R >>".into(), + sig.clone(), + ]); + let mut nested = head(6); + nested.extend([ + "<< /T (p) /Kids [5 0 R] >>".to_string(), + "<< /Kids [6 0 R] /Parent 4 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Sig /T (s) /V 7 0 R /Parent 5 0 R /Rect [20 20 180 60] /P 3 0 R >>".into(), + sig, + ]); + for (shape, objs, row) in [("merged", merged, "UnreadFields"), ("nested", nested, "p.s")] { + let mut s = Session::new(); + let id = s.open("form.pdf", None, Arc::new(pdf_of(&objs)), None).expect("opens"); + let doc = s.get(id).unwrap(); + assert!(doc.is_signed() && doc.signatures.iter().any(|r| r.field == row), "{shape}: {:?}", doc.signatures); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed, "{shape}"); + assert!(matches!(s.optimize_job(id, &optimize::Settings::default(), &[]), Err(EditError::Signed)), "{shape}"); + } +} + +/// The walk stops short of a signed field, so a signature may go unlisted; the document still +/// counts as signed, and each guard against rewriting a signed document holds. +#[test] +fn a_signature_the_field_walk_passes_over_still_counts_as_signed() { + let (s, id) = open_signed_field_named(64 << 10); + let doc = s.get(id).unwrap(); + assert!(doc.is_signed(), "{:?}", doc.signatures); +} + +#[test] +fn a_full_rewrite_is_refused_when_the_field_walk_passes_a_signature_over() { + let (s, id) = open_signed_field_named(64 << 10); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +#[test] +fn optimizing_is_refused_when_the_field_walk_passes_a_signature_over() { + let (s, id) = open_signed_field_named(64 << 10); + assert!(matches!(s.optimize_job(id, &optimize::Settings::default(), &[]), Err(EditError::Signed))); +} + +#[test] +fn a_rewriting_edit_is_refused_when_the_field_walk_passes_a_signature_over() { + let (mut s, id) = open_signed_field_named(64 << 10); + assert!(matches!(s.apply(id, Edit::Sanitize), Err(EditError::SignedRewrite(_)))); +} + +#[test] +fn an_action_appends_to_a_file_whose_field_walk_passes_a_signature_over() { + let bytes = Arc::new(signed_field_named(64 << 10)); + let steps = vec![actions::Step::from_id("set_title", "Checked").unwrap()]; + let title = actions::Action { name: "Title".into(), description: String::new(), steps, builtin: false }; + let r = actions::run_on(&title, "form.pdf", bytes.clone(), |_, _| {}).unwrap(); + assert!(r.bytes.starts_with(&bytes[..]), "the action rewrote the file instead of appending to it"); + let mut s = Session::new(); + let id = s.open("out.pdf", None, r.bytes, None).unwrap(); + assert_eq!(s.get(id).unwrap().info.title.as_deref(), Some("Checked"), "the title is in what was appended"); +} + /// #1324: marking a figure decorative rewrites that figure only. The header, added text and an /// unknown entry on the original stream stay attached to the page. This lives in the engine tests /// because edit and a11y are the same layer and cannot depend on each other. diff --git a/crates/engine/src/word_count.rs b/crates/engine/src/word_count.rs index feeb0a066f..c20dc8325f 100644 --- a/crates/engine/src/word_count.rs +++ b/crates/engine/src/word_count.rs @@ -78,6 +78,7 @@ fn count_inner(source: ExportSource, pages: &[usize], options: Options, mut prog tables: options.exclude_tables, code: options.exclude_code, images: options.exclude_images, + painted: false, mcids: excluded.remove(&page).unwrap_or_default(), }; let text = renderer.filtered_text(page, filter).map_err(|e| format!("page {}: {e}", page.saturating_add(1)))?.plain_text(); diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index 45cb843ab0..4ebe181a75 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -31,7 +31,7 @@ pub use script::{ script_missing, script_outline, }; pub use shaping::{ - ShapedCluster, ShapedScript, arabic_glyph, arabic_has, hebrew_has, shape_arabic, shape_hebrew, shape_thai, shaped_glyph, thai_has, + ShapedCluster, ShapedScript, arabic_glyph, arabic_has, face_glyph, hebrew_has, shape_arabic, shape_hebrew, shape_thai, shaped_glyph, thai_has, }; pub use std14::Std14; diff --git a/crates/fonts/src/shaping.rs b/crates/fonts/src/shaping.rs index f0bd118c15..54a401875f 100644 --- a/crates/fonts/src/shaping.rs +++ b/crates/fonts/src/shaping.rs @@ -147,6 +147,16 @@ pub fn arabic_glyph(id: u32) -> Result { /// A bounded outline from the same face that produced the shaped glyph id. pub fn shaped_glyph(script: ShapedScript, id: u32) -> Result { let (font, _) = face(script).ok_or(GlyphError::NoFont)?; + glyph(font, id) +} + +/// Glyph `id` of the face in `bytes`, bounded like [`shaped_glyph`]: for text shaped with +/// [`crate::source_shaping::shape`] in a face no [`ShapedScript`] names. +pub fn face_glyph(bytes: &[u8], id: u32) -> Result { + glyph(&FontRef::new(bytes).map_err(|_| GlyphError::NoFont)?, id) +} + +fn glyph(font: &FontRef, id: u32) -> Result { if font.maxp().map_or(true, |m| id >= u32::from(m.num_glyphs())) { return Err(GlyphError::Missing); } diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b0a7257cb3..bd42efce39 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -361,7 +361,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], processed.push((done.ops, done.changed)); } if scope.mode == Mode::Apply { - clean_replacements(doc, resources, &original, &mut processed, scope.report); + clean_replacements(doc, resources, &original, &walk.clipped, &mut processed, scope.report); } walk.out.streams = processed.into_iter().zip(comment_edges).map(|((ops, changed), edges)| changed.then(|| piece_bytes(&ops, edges))).collect(); walk.out @@ -424,6 +424,9 @@ struct Walk { path: Vec, path_box: Option<[f64; 4]>, clip: bool, + /// Painting operators written again under a clip that cuts the regions out: unchanged as + /// operators, but what they paint is no longer all there. + clipped: HashSet, out: Output, } @@ -442,6 +445,7 @@ impl Walk { path: Vec::new(), path_box: None, clip: false, + clipped: HashSet::new(), out: Output::default(), } } @@ -474,7 +478,7 @@ impl Walk { } b"Tj" | b"TJ" | b"'" | b"\"" => self.show_text(scope, op, piece), b"BI" => Self::inline_image(&self.gs, scope, &mut self.out, op, piece), - b"sh" if scope.mode == Mode::Apply => Self::shading(&self.gs, scope, op, piece), + b"sh" if scope.mode == Mode::Apply => Self::shading(&self.gs, scope, &mut self.clipped, op, piece), b"Do" => self.draw_xobject(doc, scope, resources, op, piece), // `/Name scn` selects a pattern as the fill colour; numbers select a plain colour // and clear it. `cs` with a space other than /Pattern clears it too. @@ -717,10 +721,12 @@ impl Walk { None } - /// A shading fills the current clip: the regions are clipped out. - fn shading(gs: &Gs, scope: &Scope<'_>, op: Op, piece: &mut Piece) -> Option { - let Some(c) = clip_out(scope.rects, &gs.ctm, [-1e6, -1e6, 1e6, 1e6]) else { return Some(op) }; + /// A shading fills the current clip: the regions are clipped out. Removing hidden content has + /// no regions, so the shading and its name stay as they are. + fn shading(gs: &Gs, scope: &Scope<'_>, clipped: &mut HashSet, op: Op, piece: &mut Piece) -> Option { + let Some(c) = clip_out(scope.rects, &gs.ctm, [-1e6, -1e6, 1e6, 1e6]).filter(|c| !c.is_empty()) else { return Some(op) }; piece.changed = true; + clipped.insert(op.span.start); piece.ops.push(Op::new("q", vec![])); piece.ops.extend(c); piece.ops.push(op); @@ -739,7 +745,16 @@ impl Walk { } } -fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], processed: &mut [(Vec, bool)], report: &mut Report) { +/// Strip `/ActualText` (and `/Alt`, `/E`, `/T`) from the marked content around anything redaction +/// removed, changed or `clipped`: the words they name may be among what is gone. +fn clean_replacements( + doc: &Document, + resources: &Dict, + original: &[Op], + clipped: &HashSet, + processed: &mut [(Vec, bool)], + report: &mut Report, +) { let originals: HashMap = original.iter().map(|op| (op.span.start, op)).collect(); let unchanged: HashSet = processed.iter().flat_map(|(ops, _)| ops).filter(|op| originals.get(&op.span.start) == Some(op)).map(|op| op.span.start).collect(); @@ -751,7 +766,13 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process b"EMC" => { stack.pop(); } - _ if !unchanged.contains(&op.span.start) => affected.extend(stack.iter().copied()), + // A scope joins `affected` only with every scope around it, so once the innermost one + // is in, the rest are too: deep nesting over many changed ops stays linear. + _ if (!unchanged.contains(&op.span.start) || clipped.contains(&op.span.start)) + && stack.last().is_some_and(|top| !affected.contains(top)) => + { + affected.extend(stack.iter().copied()) + } _ => {} } } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 8978e0e7d6..70c9baacd0 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -61,6 +61,68 @@ fn reopen(doc: &Document) -> Document { Document::open(Arc::new(write_incremental(doc, &SaveOptions::default()).unwrap())).unwrap() } +/// A word drawn as outlines names itself in /ActualText. A mark over part of it removes the +/// outlines (a shading painting them is clipped instead), and the name, which would still read the +/// whole word, goes with them. A word the mark misses keeps its own. +#[test] +fn redacted_paint_loses_the_actual_text_that_names_it() { + let mut doc = one_page( + b"/Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h f EMC /Span <> BDC 100 200 60 20 re f EMC", + "", + vec![], + ); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_clipped, r.paths_removed), (0, 1)); + let c = content(&doc, 0); + assert!(!c.contains("SECRET") && c.contains("KEPT"), "{c}"); + + let shading = + "<< /ShadingType 2 /ColorSpace /DeviceGray /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [0] /C1 [1] /N 1 >> >>"; + let mut doc = one_page(b"/Span <> BDC /Sh0 sh EMC", "/Shading << /Sh0 7 0 R >>", vec![shading.as_bytes().to_vec()]); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + apply(&mut doc, None).unwrap(); + let c = content(&doc, 0); + // Drawn after a clip that cuts the region out. + assert!(!c.contains("SHADED") && c.find("W*").zip(c.find(" sh")).is_some_and(|(w, s)| w < s), "{c}"); + + // A path that clips as it paints. + let mut doc = one_page(b"q /Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h W f EMC Q", "", vec![]); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("CLIPS"), "{c}"); +} + +/// Nested names around redacted paint all go, the innermost too, whichever scope was cut first. +#[test] +fn redacted_paint_loses_every_name_around_it() { + let mut doc = one_page( + b"/Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h f /Span <> BDC 100 130 m 160 130 l 160 150 l 100 150 l h f EMC EMC", + "", + vec![], + ); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 155.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 2); + let c = content(&doc, 0); + assert!(!c.contains("OUTER") && !c.contains("INNER"), "{c}"); +} + +/// Removing hidden text clips nothing, so a shading on the same page keeps the words that name it. +#[test] +fn removing_hidden_text_keeps_the_alt_text_of_a_shading() { + let shading = + "<< /ShadingType 2 /ColorSpace /DeviceGray /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [0] /C1 [1] /N 1 >> >>"; + let mut doc = one_page( + b"BT /F1 10 Tf 3 Tr 10 200 Td (SECRET) Tj ET /Figure <> BDC /Sh0 sh EMC", + "/Shading << /Sh0 7 0 R >>", + vec![shading.as_bytes().to_vec()], + ); + sanitize::remove_hidden(&mut doc, &[sanitize::Hidden::HiddenText]).unwrap(); + let c = content(&doc, 0); + assert!(!c.contains("SECRET") && c.contains("(chart)"), "{c}"); +} + #[test] fn glyphs_under_a_mark_go_and_the_rest_stays_put() { // "AB1234CD" from x = 10 at 10 pt: each glyph 5 pt wide; 1234 spans x 20–40. diff --git a/crates/render/src/text.rs b/crates/render/src/text.rs index 12a1e3aecd..aa66a4b683 100644 --- a/crates/render/src/text.rs +++ b/crates/render/src/text.rs @@ -23,6 +23,9 @@ pub struct TextFilter { pub code: bool, /// Exclude Figure-marked text and invisible text layers, including OCR. pub images: bool, + /// Exclude `/ActualText` that marks no glyph, whose words are drawn as outlines or pictures: + /// a page's own text, for deciding whether it still needs reading from its pixels. + pub painted: bool, /// Page-local MCIDs whose structure ancestors match the selected exclusions. pub mcids: HashSet, } diff --git a/crates/render/src/text/device.rs b/crates/render/src/text/device.rs index 7c4881e8be..6c14325400 100644 --- a/crates/render/src/text/device.rs +++ b/crates/render/src/text/device.rs @@ -25,6 +25,16 @@ pub(super) struct TextDevice { /// The outermost open `/ActualText` replacement: its depth, text and the glyphs drawn inside /// it (with no text of their own yet). actual: Option<(usize, String, Vec)>, + /// What the open `/ActualText` replacement paints besides glyphs (paths, images), as far as + /// it can be seen: where its text reads when it marks no glyph (words drawn as outlines). + painted: Option, + /// Whether the open `/ActualText` replacement paints anything an exclusion leaves out. + paint_filtered: bool, + /// Whether the open `/ActualText` replacement paints anything wholly outside the crop box. + paint_cropped: bool, + /// The bounding boxes of the clip paths in force (view space), innermost last, each cut to the + /// one before it: an outer bound on what can show. + clips: Vec, /// Every `/ActualText` replacement made on the page. pub(super) replaced: Vec, /// Visible page bounds in view space (`intersected_crop_box` after `initial_transform`). @@ -105,12 +115,109 @@ fn merge(drawn: &[Drawn], text: String) -> Option { Some((quarter, u, TextGlyph { text, rect, quad, direction: first.2.direction })) } +/// The widest or tallest box (view-space points) a glyph, or the paint standing in for one, may +/// have and still be placed: anything bigger is damage, not text. +const MAX_GLYPH_EXTENT: f64 = 10_000.0; + +/// The quarter turn of direction `(dx, dy)` (view space, y down): 0 right, 1 down, 2 left, 3 up. +fn quarter_turn(dx: f64, dy: f64) -> usize { + if dx.abs() >= dy.abs() { + if dx >= 0.0 { 0 } else { 2 } + } else if dy > 0.0 { + 1 + } else { + 3 + } +} + +/// What an `/ActualText` replacement paints besides glyphs, measured as [`merge`] measures +/// glyphs: along the writing direction of the first of it (its transform's x axis, `u`) and +/// across it. +struct Painted { + quarter: usize, + u: [f64; 2], + along: [f64; 2], + across: [f64; 2], +} + +impl Painted { + fn new(transform: Affine) -> Self { + let [a, b, ..] = transform.as_coeffs(); + let length = a.hypot(b); + let u = if length.is_finite() && length > 0.0 { [a / length, b / length] } else { [1.0, 0.0] }; + let empty = [f64::INFINITY, f64::NEG_INFINITY]; + Painted { quarter: quarter_turn(u[0], u[1]), u, along: empty, across: empty } + } + + /// Grow the measure by the view-space box `b`. + fn add(&mut self, b: Rect) { + let n = [-self.u[1], self.u[0]]; + for (x, y) in [(b.x0, b.y0), (b.x1, b.y0), (b.x1, b.y1), (b.x0, b.y1)] { + let (s, t) = (x * self.u[0] + y * self.u[1], x * n[0] + y * n[1]); + self.along = [self.along[0].min(s), self.along[1].max(s)]; + self.across = [self.across[0].min(t), self.across[1].max(t)]; + } + } + + /// A stand-in glyph with no text of its own, boxed around what was painted and read along its + /// direction; `None` when that is too big to be the place of words. + fn glyph(self) -> Option { + let ([s0, s1], [t0, t1]) = (self.along, self.across); + // Too big to be the place of words, or NaN. + if !(s1 - s0 <= MAX_GLYPH_EXTENT && t1 - t0 <= MAX_GLYPH_EXTENT) { + return None; + } + let (u, n) = (self.u, [-self.u[1], self.u[0]]); + let point = |s: f64, t: f64| [(s * u[0] + t * n[0]) as f32, (s * u[1] + t * n[1]) as f32]; + // The edge towards the next line starts the quad, as a glyph's em box does. + let quad = [point(s0, t1), point(s1, t1), point(s1, t0), point(s0, t0)]; + let rect = quad.iter().fold([f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], |r, p| { + [r[0].min(p[0]), r[1].min(p[1]), r[2].max(p[0]), r[3].max(p[1])] + }); + rect.iter() + .all(|v| v.is_finite()) + .then(|| (self.quarter, u, TextGlyph { text: String::new(), rect, quad, direction: [u[0] as f32, u[1] as f32] })) + } +} + +/// The part of `b` inside `c`, or `None` when none of it is. A NaN edge of `c` (a damaged clip) +/// leaves that edge uncut. +fn cut(b: Rect, c: Rect) -> Option { + let (x0, y0, x1, y1) = (b.x0.max(c.x0), b.y0.max(c.y0), b.x1.min(c.x1), b.y1.min(c.y1)); + (x0 <= x1 && y0 <= y1).then(|| Rect::new(x0, y0, x1, y1)) +} + +/// The view-space box of `path` drawn with `transform`, from its control points (as redaction +/// measures a path), or `None` when it has no finite point. +fn control_box(path: &BezPath, transform: Affine) -> Option { + let mut b: Option = None; + let mut add = |p: kurbo::Point| { + let q = transform * p; + if q.is_finite() { + b = Some(b.map_or_else(|| Rect::from_points(q, q), |r| r.union_pt(q))); + } + }; + for el in path.elements() { + match *el { + kurbo::PathEl::MoveTo(p) | kurbo::PathEl::LineTo(p) => add(p), + kurbo::PathEl::QuadTo(p1, p2) => [p1, p2].into_iter().for_each(&mut add), + kurbo::PathEl::CurveTo(p1, p2, p3) => [p1, p2, p3].into_iter().for_each(&mut add), + kurbo::PathEl::ClosePath => {} + } + } + b +} + impl TextDevice { pub(super) fn new() -> Self { TextDevice { glyphs: Vec::new(), depth: 0, actual: None, + painted: None, + paint_filtered: false, + paint_cropped: false, + clips: Vec::new(), replaced: Vec::new(), page: Rect::new(f64::NEG_INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::INFINITY), filter: TextFilter::default(), @@ -128,9 +235,14 @@ impl TextDevice { /// words can't be found. /// /// A replacement spanning several lines or writing directions stays per glyph (each glyph's own - /// Unicode). An empty one (a soft hyphen at a line break, say) stands for no text at all. + /// Unicode). An empty one (a soft hyphen at a line break, say) stands for no text at all. One + /// that marks no glyph reads where it paints, as far as that can be seen: words drawn as + /// outlines or pictures. fn end_actual_text(&mut self) { - let Some((_, text, drawn)) = self.actual.take() else { return }; + let Some((_, text, mut drawn)) = self.actual.take() else { return }; + let painted = self.painted.take(); + let paint_filtered = std::mem::take(&mut self.paint_filtered); + let paint_cropped = std::mem::take(&mut self.paint_cropped); let cropped = std::mem::take(&mut self.actual_cropped); if std::mem::take(&mut self.actual_filtered) { // ActualText has no mapping from replacement words to individual glyphs. A mixed @@ -138,10 +250,27 @@ impl TextDevice { self.partial_replacement |= !drawn.is_empty(); return; } + // Before the paint: a span whose glyphs the crop box took has none left, yet it marks + // glyphs, so its paint doesn't place it. if cropped { self.glyphs.extend(drawn.into_iter().filter(|d| !d.2.text.chars().all(|c| c.is_control()))); return; } + if drawn.is_empty() + && !self.filter.painted + && let Some(stand_in) = painted.and_then(Painted::glyph) + { + // The paint places it, so paint left out mixes it as excluded glyphs would. + if paint_filtered { + self.partial_replacement = true; + return; + } + // Its words may be among what the crop box hides, as with a glyph cropped away. + if paint_cropped { + return; + } + drawn.push(stand_in); + } let Some(glyph) = merge(&drawn, text.clone()) else { self.glyphs.extend(drawn.into_iter().filter(|d| !d.2.text.chars().all(char::is_control))); return; @@ -156,6 +285,25 @@ impl TextDevice { self.replaced.push(Replaced { at, text, originals }); } + /// Record view-space box `b`, drawn with `transform`, as painted by the open `/ActualText` + /// replacement, cut to the visible area (the crop box, and the clip in force); paint wholly + /// outside the crop box, or that an exclusion leaves out (`excluded`), only marks the + /// replacement. + fn paint(&mut self, b: Rect, transform: Affine, excluded: bool) { + // Wholly outside the crop box, as a glyph dropped there is: the replacement can't be read. + let Some(on_page) = cut(b, self.page) else { + self.paint_cropped = true; + return; + }; + // Clipped away entirely: nothing of it shows. + let Some(seen) = self.clips.last().map_or(Some(on_page), |c| cut(on_page, *c)) else { return }; + if excluded { + self.paint_filtered = true; + } else { + self.painted.get_or_insert_with(|| Painted::new(transform)).add(seen); + } + } + /// Ends the page. Some producers draw text twice, the copy for an outline, shadow or /// glow, and mark only one drawing with `/ActualText` (Canva). The unmarked redraw of a /// replacement's glyphs (the same text at the same places, in content order) takes the same @@ -259,8 +407,23 @@ fn text_debug() -> bool { impl<'a> Device<'a> for TextDevice { fn set_soft_mask(&mut self, _: Option>) {} fn set_blend_mode(&mut self, _: BlendMode) {} - fn draw_path(&mut self, _: &BezPath, _: Affine, _: &Paint<'a>, _: &PathDrawMode) {} - fn push_clip_path(&mut self, _: &ClipPath) {} + fn draw_path(&mut self, path: &BezPath, transform: Affine, _: &Paint<'a>, _: &PathDrawMode) { + // A path is no text, but it places an `/ActualText` replacement that marks no glyph. + if self.actual.is_some() + && let Some(b) = control_box(path, transform) + { + self.paint(b, transform, self.excluded_depth.is_some()); + } + } + fn push_clip_path(&mut self, clip: &ClipPath) { + // Clip paths arrive in view space. NaN edges (a damaged clip) leave that edge unclipped. + let b = clip.path.bounding_box(); + let visible = match self.clips.last() { + Some(c) => Rect::new(c.x0.max(b.x0), c.y0.max(b.y0), c.x1.min(b.x1), c.y1.min(b.y1)), + None => b, + }; + self.clips.push(visible); + } fn push_transparency_group(&mut self, _: f32, _: Option>, _: BlendMode) {} fn draw_glyph(&mut self, glyph: &Glyph<'a>, transform: Affine, glyph_transform: Affine, _: &Paint<'a>, mode: &GlyphDrawMode) { if self.excluded_depth.is_some() || (self.filter.images && matches!(mode, GlyphDrawMode::Invisible)) { @@ -309,13 +472,7 @@ impl<'a> Device<'a> for TextDevice { } else { (coefficients[0], coefficients[1]) }; - let dir = if dx.abs() >= dy.abs() { - if dx >= 0.0 { 0 } else { 2 } - } else if dy > 0.0 { - 1 - } else { - 3 - }; + let dir = quarter_turn(dx, dy); let em = Rect::new(0.0, -200.0, advance, 800.0); let corners = if matches!(glyph, Glyph::Outline(g) if g.is_vertical()) { [(em.x0, em.y1), (em.x0, em.y0), (em.x1, em.y0), (em.x1, em.y1)] @@ -328,8 +485,8 @@ impl<'a> Device<'a> for TextDevice { }); let outside_crop = b.x0.max(self.page.x0) >= b.x1.min(self.page.x1) || b.y0.max(self.page.y0) >= b.y1.min(self.page.y1); if !(b.x0.is_finite() && b.y0.is_finite() && b.x1.is_finite() && b.y1.is_finite()) - || b.width() > 10_000.0 - || b.height() > 10_000.0 + || b.width() > MAX_GLYPH_EXTENT + || b.height() > MAX_GLYPH_EXTENT || outside_crop { self.actual_cropped |= replaced && outside_crop; @@ -350,8 +507,23 @@ impl<'a> Device<'a> for TextDevice { None => self.glyphs.push((dir, direction, glyph)), } } - fn draw_image(&mut self, _: Image<'a, '_>, _: Affine) {} - fn pop_clip_path(&mut self) {} + fn draw_image(&mut self, image: Image<'a, '_>, transform: Affine) { + // A picture of words, like an outline, places a replacement that marks no glyph; with + // images excluded, it is not counted. + if self.actual.is_none() { + return; + } + // The transform places the picture's pixels, not a unit square. + let (w, h) = (f64::from(image.width()), f64::from(image.height())); + let corners = [(0.0, 0.0), (w, 0.0), (w, h), (0.0, h)].map(|(x, y)| transform * kurbo::Point::new(x, y)); + let [a, b, c, d] = corners; + if corners.iter().all(|p| p.is_finite()) { + self.paint(Rect::from_points(a, c).union_pt(b).union_pt(d), transform, self.excluded_depth.is_some() || self.filter.images); + } + } + fn pop_clip_path(&mut self) { + self.clips.pop(); + } fn pop_transparency_group(&mut self) {} fn begin_marked_content(&mut self, tag: &[u8], mcid: Option) { self.depth = self.depth.saturating_add(1); @@ -374,6 +546,11 @@ impl<'a> Device<'a> for TextDevice { } self.depth = self.depth.saturating_sub(1); } + fn begin_annotations(&mut self) { + // A replacement the page content never closed ends with it: the annotations drawn next + // are not part of it. + self.end_actual_text(); + } } /// Extract the text of one page. Panics inside the interpreter are the caller's to catch. diff --git a/crates/render/src/text/tests.rs b/crates/render/src/text/tests.rs index 8776dfeb72..f0bd9d864b 100644 --- a/crates/render/src/text/tests.rs +++ b/crates/render/src/text/tests.rs @@ -279,6 +279,207 @@ fn actual_text_follows_the_pdf_rules_for_marked_content() { assert_eq!(plain, ["ofice", "AB", "co op", "x\ny", "afi", "abc"]); } +#[test] +fn actual_text_over_painted_paths_reads_where_they_are_painted() { + // Words drawn as outlines (a face that can't be embedded) carry their text in /ActualText + // (ISO 32000-2 §14.9.4): it reads at the box of the paths it marks, in view space (y down). + let pdf = marked_pages(&[ + "/Span <> BDC 100 40 m 160 40 l 160 60 l 100 60 l f EMC", + // A curve is measured by its control points, as redaction measures it. + "/Span <> BDC 100 40 m 100 100 160 100 160 40 c f EMC", + "BT /F1 20 Tf 1 0 0 1 20 150 Tm (a) Tj ET /Span <> BDC 100 140 60 20 re f EMC BT /F1 20 Tf 1 0 0 1 200 150 Tm (c) Tj ET", + // Glyphs inside the span keep their own box; the paths beside them don't widen it. + "/Span <> BDC BT /F1 20 Tf 1 0 0 1 100 150 Tm (y) Tj ET 0 0 300 300 re f EMC", + // Paths are not text without a replacement, and an empty one stands for nothing. + "100 140 60 20 re f", + "/Span <> BDC 100 140 60 20 re f EMC", + ]); + let painted = marked_text(&pdf, 0); + assert_eq!(painted.plain_text(), "valid"); + assert_eq!(painted.glyphs[0].rect, [100.0, 240.0, 160.0, 260.0]); + assert_eq!(marked_text(&pdf, 1).glyphs[0].rect, [100.0, 200.0, 160.0, 260.0]); + let between = marked_text(&pdf, 2); + assert_eq!(between.glyphs.iter().map(|g| g.text.as_str()).collect::>(), ["a", "b", "c"]); + let mixed = marked_text(&pdf, 3); + assert_eq!(mixed.plain_text(), "x"); + assert!(mixed.glyphs[0].rect[2] - mixed.glyphs[0].rect[0] < 20.0, "{:?}", mixed.glyphs[0].rect); + for page in 4..6 { + assert_eq!(marked_text(&pdf, page).plain_text(), "", "page {page}"); + } +} + +#[test] +fn the_painted_filter_leaves_out_replacements_that_mark_no_glyph() { + // A page's own text, without words that only /ActualText names. + let pdf = marked_pages(&[ + "/Span <> BDC 100 40 m 160 40 l 160 60 l 100 60 l f EMC", + "BT /F1 20 Tf 1 0 0 1 20 150 Tm (a) Tj ET /Span <> BDC 100 140 60 20 re f EMC BT /F1 20 Tf 1 0 0 1 200 150 Tm (c) Tj ET", + ]); + let own = |page| extract_page_filtered(&pdf, page, &InterpreterSettings::default(), TextFilter { painted: true, ..Default::default() }); + assert_eq!(own(0).unwrap().plain_text(), ""); + let between = own(1).unwrap().plain_text(); + assert!(between.contains('a') && between.contains('c') && !between.contains('b'), "the glyphs stay: {between:?}"); +} + +#[test] +fn actual_text_over_paint_reads_only_where_it_shows() { + let pdf = marked_pages(&[ + // A page-sized fill clipped to a word's box reads at that box. + "q 100 40 30 20 re W n /Span <> BDC 0 0 300 300 re f EMC Q", + // Wholly clipped away, it reads nowhere. + "q 0 0 10 10 re W n /Span <> BDC 100 40 60 20 re f EMC Q", + // A clip that ended before the paint no longer cuts it. + "q 0 0 10 10 re W n Q /Span <> BDC 100 40 60 20 re f EMC", + ]); + assert_eq!(marked_text(&pdf, 0).glyphs[0].rect, [100.0, 240.0, 130.0, 260.0]); + assert_eq!(marked_text(&pdf, 1).plain_text(), ""); + assert_eq!(marked_text(&pdf, 2).glyphs[0].rect, [100.0, 240.0, 160.0, 260.0]); +} + +#[test] +fn actual_text_over_paint_reads_only_inside_the_crop_box() { + use lopdf::{Document, Object, Stream, dictionary}; + let (page, crop) = ([0, 0, 300, 300], [50, 60, 250, 240]); + let pages = [ + // Wholly outside the crop box, as a glyph there is: it reads nowhere. + ("/Span <> BDC 100 10 60 20 re f EMC", page, crop), + // Across its left edge: it reads from that edge, as a glyph across it still shows. + ("/Span <> BDC 20 100 60 20 re f EMC", page, crop), + // Partly painted outside it: its words may be among what the crop box hides. + ("/Span <> BDC 100 100 60 20 re f 100 10 60 20 re f EMC", page, crop), + // A glyph the crop takes leaves the replacement no glyph, and the paint can't place it. + ("/Span <> BDC BT /F1 12 Tf 1 0 0 1 100 10 Tm (x) Tj ET 100 100 60 20 re f EMC", page, crop), + // On a page wide enough to hold them, paint too far apart to be the place of words. + ("/Span <> BDC 100 40 10 10 re f 12100 40 10 10 re f EMC", [0, 0, 14400, 300], [0, 0, 14400, 300]), + ]; + let boxed = |b: [i64; 4]| b.map(Object::Integer).to_vec(); + let mut doc = Document::with_version("1.7"); + let font = doc.add_object(dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" }); + let parent = doc.new_object_id(); + let mut kids = Vec::new(); + for (ops, media, crop) in pages { + let content = doc.add_object(Stream::new(dictionary! {}, ops.as_bytes().to_vec())); + kids.push(Object::Reference(doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => parent, + "MediaBox" => boxed(media), + "CropBox" => boxed(crop), + "Resources" => dictionary! { "Font" => dictionary! { "F1" => font } }, + "Contents" => content, + }))); + } + let count = kids.len() as i64; + doc.objects.insert(parent, dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => count }.into()); + let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => parent }); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).unwrap(); + let pdf = Pdf::new(std::sync::Arc::new(bytes)).unwrap(); + assert_eq!(marked_text(&pdf, 0).plain_text(), ""); + // View space starts at the crop box's top-left corner. + assert_eq!(marked_text(&pdf, 1).glyphs[0].rect, [0.0, 120.0, 30.0, 140.0]); + for page in 2..5 { + assert_eq!(marked_text(&pdf, page).plain_text(), "", "page {page}"); + } +} + +#[test] +fn actual_text_over_paint_reads_along_the_way_it_is_drawn() { + // The same quarter turn for a path and for text: the replacement runs the way the text does. + let pdf = marked_pages(&[ + "q 0 1 -1 0 150 150 cm /Span <> BDC 0 0 40 10 re f EMC Q", + "q 0 1 -1 0 150 150 cm BT /F1 10 Tf 0 0 Td (t) Tj ET Q", + ]); + let (painted, text) = (marked_text(&pdf, 0), marked_text(&pdf, 1)); + assert_eq!(painted.plain_text(), "turned"); + assert_eq!(painted.glyphs[0].direction, text.glyphs[0].direction); + let [x0, y0, x1, y1] = painted.glyphs[0].rect; + assert_eq!((x1 - x0, y1 - y0), (10.0, 40.0), "40 along the turned line, 10 across it"); +} + +#[test] +fn actual_text_over_a_picture_reads_where_it_is_drawn() { + // Two pixels square, so a box placed in pixels rather than across the picture shows. + let pdf = marked_pages(&["/Span <> BDC q 60 0 0 20 100 40 cm BI /W 2 /H 2 /CS /G /BPC 8 ID AAAA EI Q EMC"]); + let text = marked_text(&pdf, 0); + assert_eq!(text.plain_text(), "E=mc2"); + assert_eq!(text.glyphs[0].rect, [100.0, 240.0, 160.0, 260.0]); + // Counting without images leaves it out. + let filter = TextFilter { images: true, ..Default::default() }; + assert_eq!(extract_page_filtered(&pdf, 0, &InterpreterSettings::default(), filter).unwrap().plain_text(), ""); +} + +#[test] +fn excluded_glyphs_beside_paint_leave_the_whole_replacement_out() { + // The marked glyphs are a figure's, so the replacement goes with them; the paint beside + // them neither keeps it nor turns it into a mixed span that can't be counted. + let pdf = marked_pages(&[ + "/Span <> BDC /Figure BMC BT /F1 20 Tf 1 0 0 1 100 150 Tm (x) Tj ET EMC 100 40 60 20 re f EMC", + // Paint inside an excluded region is excluded with it. + "/Span <> BDC /Figure BMC 100 40 60 20 re f EMC EMC", + "/Span <> BDC /Table BMC q 60 0 0 20 100 40 cm BI /W 1 /H 1 /CS /G /BPC 8 ID A EI Q EMC EMC", + ]); + let filtered = |page, filter| extract_page_filtered(&pdf, page, &InterpreterSettings::default(), filter).unwrap().plain_text(); + let (figures, tables) = (TextFilter { images: true, ..Default::default() }, TextFilter { tables: true, ..Default::default() }); + assert_eq!(filtered(0, figures.clone()), ""); + assert_eq!((filtered(1, TextFilter::default()), filtered(1, figures)), ("z".to_string(), String::new())); + assert_eq!((filtered(2, TextFilter::default()), filtered(2, tables)), ("t".to_string(), String::new())); +} + +#[test] +fn paint_split_by_an_exclusion_is_a_mixed_span_that_cant_be_counted() { + // Paint places a replacement that marks no glyph, so paint left out mixes it as excluded + // glyphs do: the words left out can't be told from the rest. + let picture = "q 60 0 0 20 100 40 cm BI /W 1 /H 1 /CS /G /BPC 8 ID A EI Q"; + let (table, beside) = ( + format!("/Span <> BDC /Table BMC {picture} EMC 200 40 60 20 re f EMC"), + format!("/Span <> BDC {picture} 200 40 60 20 re f EMC"), + ); + let pdf = marked_pages(&[ + "/Span <> BDC /Figure BMC 100 40 60 20 re f EMC 200 40 60 20 re f EMC", + &table, + &beside, + // Glyphs place this one, and the paint beside them is not its words. + "/Span <> BDC BT /F1 20 Tf 1 0 0 1 100 150 Tm (y) Tj ET /Figure BMC 100 40 60 20 re f EMC EMC", + ]); + let filtered = |page, filter| extract_page_filtered(&pdf, page, &InterpreterSettings::default(), filter).map(|t| t.plain_text()); + let (figures, tables) = (TextFilter { images: true, ..Default::default() }, TextFilter { tables: true, ..Default::default() }); + for (page, text, filter) in [(0, "w", figures.clone()), (1, "t", tables), (2, "p", figures.clone())] { + assert_eq!(filtered(page, TextFilter::default()), Ok(text.to_string())); + assert!(filtered(page, filter).is_err(), "page {page}"); + } + assert_eq!(filtered(3, figures), Ok("g".to_string())); +} + +#[test] +fn annotations_never_join_a_replacement_the_page_left_open() { + use lopdf::{Document, Object, Stream, dictionary}; + let mut doc = Document::with_version("1.7"); + let pages = doc.new_object_id(); + let content = doc.add_object(Stream::new(dictionary! {}, b"/Span <> BDC 100 40 60 20 re f".to_vec())); + let ap = doc.add_object(Stream::new( + dictionary! { "Type" => "XObject", "Subtype" => "Form", "BBox" => vec![0.into(), 0.into(), 50.into(), 50.into()] }, + b"0 0 50 50 re f".to_vec(), + )); + let annot = doc.add_object(dictionary! { + "Type" => "Annot", "Subtype" => "Square", "F" => 4, + "Rect" => vec![200.into(), 200.into(), 250.into(), 250.into()], "AP" => dictionary! { "N" => ap }, + }); + let page = doc.add_object(dictionary! { + "Type" => "Page", "Parent" => pages, "Contents" => content, "Annots" => vec![Object::Reference(annot)], + "MediaBox" => vec![Object::Integer(0), Object::Integer(0), Object::Integer(300), Object::Integer(300)], + }); + doc.objects.insert(pages, dictionary! {"Type" => "Pages", "Kids" => vec![Object::Reference(page)], "Count" => 1}.into()); + let catalog = doc.add_object(dictionary! {"Type" => "Catalog", "Pages" => pages}); + doc.trailer.set("Root", catalog); + let mut bytes = Vec::new(); + doc.save_to(&mut bytes).unwrap(); + let pdf = Pdf::new(std::sync::Arc::new(bytes)).unwrap(); + let text = marked_text(&pdf, 0); + assert_eq!(text.plain_text(), "open"); + assert_eq!(text.glyphs[0].rect, [100.0, 240.0, 160.0, 260.0], "the square comment is not part of it"); +} + #[test] fn an_unmarked_redraw_of_actual_text_reads_as_the_same_text() { // Canva draws text twice (an outline or shadow) and marks only one drawing. The copy, diff --git a/crates/sign/Cargo.toml b/crates/sign/Cargo.toml index 3c0f207a98..bfb04a26cb 100644 --- a/crates/sign/Cargo.toml +++ b/crates/sign/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] pdfcraft-cos = { workspace = true } pdfcraft-annot = { workspace = true } +pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } pdfcraft-model = { workspace = true } thiserror.workspace = true @@ -41,6 +42,9 @@ sha3 = "0.12.0" ripemd = "0.2.0" ed25519-dalek = "3.0.0" +[dev-dependencies] +pdfcraft-testkit = { workspace = true } + [target.'cfg(not(target_arch = "wasm32"))'.dependencies] aws-lc-rs = { version = "1.18", default-features = false, features = ["aws-lc-sys", "prebuilt-nasm"] } getrandom = "0.4" diff --git a/crates/sign/README.md b/crates/sign/README.md index 6844b1a6ec..126e938786 100644 --- a/crates/sign/README.md +++ b/crates/sign/README.md @@ -50,6 +50,10 @@ for s in signatures(&doc, &bytes, &trust) { // li RFC 6960 OCSP responses (responder identity, OCSP-signing EKU for delegated responders, CertID hash matching, validity windows); validation checks embedded evidence against the signer's chain and a verified revocation invalidates the signature. +- **Status layers:** `show_status_layers` draws each signature's validated state (icon, and a + caption in the caller's words) into named-layer appearances, whose `/N` draws one layer form + that holds and draws `n1` and `n3`; `has_status_layers` says whether a document has any. It + works on a display copy and never edits an original object, so saving keeps the signed bytes. Not yet: revocation fetching (AIA/CRLDP extraction and a fetcher), timestamp-server configuration, FieldMDP locks, certificate security, direct PKCS #11 access to tokens. diff --git a/crates/sign/src/lib.rs b/crates/sign/src/lib.rs index 2954de288d..a3a35540a0 100644 --- a/crates/sign/src/lib.rs +++ b/crates/sign/src/lib.rs @@ -19,6 +19,7 @@ pub mod pdf; pub mod pkcs12; pub mod revocation; mod rsa_pad; +pub mod status_layers; pub mod timestamp; #[cfg(target_os = "windows")] pub mod windows; @@ -32,6 +33,7 @@ pub use pdf::{ in_english, kind, list as signatures, sign, sign_with_timestamp, timestamp_document, validate, }; pub use pkcs12::DigitalId; +pub use status_layers::{StatusCaptions, has_status_layers, show_status_layers, status_rgb}; pub use timestamp::{TimestampAuthority, TimestampQuery, TimestampToken}; pub use x509::{Certificate, Name}; diff --git a/crates/sign/src/pdf.rs b/crates/sign/src/pdf.rs index 8fafff7f01..77678f2c83 100644 --- a/crates/sign/src/pdf.rs +++ b/crates/sign/src/pdf.rs @@ -32,8 +32,11 @@ mod validate; #[cfg(test)] use changes::only_grew; #[cfg(test)] -use listing::{DISCOVERY_CACHE_LIMIT, is_document_timestamp}; +pub(crate) use listing::MAX_FIELD_NAME_BYTES; +#[cfg(test)] +use listing::{DISCOVERY_CACHE_LIMIT, Field, MAX_ENTRIES, MAX_FIELDS, MAX_NAME_BYTES, is_document_timestamp, sig_fields}; pub use listing::{DigestCache, list, list_cached}; +pub(crate) use listing::{walk_sig_fields, widget_shown}; pub(crate) use signing::fmt; pub use signing::{Appearance, AppearanceLabels, SignOptions, display_date, sign, sign_with_timestamp, timestamp_document}; pub use types::{Detail, Fill, Modification, Status, TrustStore, WORDINGS, Words, in_english, kind}; @@ -46,6 +49,9 @@ use validate::{DssEvidence, MAX_DSS_BYTES, MAX_DSS_ENTRIES}; #[path = "pdf_discovery_tests.rs"] mod discovery_tests; #[cfg(test)] +#[path = "pdf_fields_tests.rs"] +mod fields_tests; +#[cfg(test)] #[path = "pdf/revision_classification_tests.rs"] mod revision_classification_tests; @@ -89,6 +95,11 @@ pub struct SignatureInfo { pub timestamp_time: Option