From ce3857c2e5ec61dd6a5cf8565c7e699acc7618a7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 20:55:45 +0000 Subject: [PATCH 01/14] sign: classify changes by signed object role Judge each object changed after signing by the role it held in the signed revision, keyed by object number, and compare a slot's accepted generations across revisions. Adds `Document::accepted_generations` to cos. Co-Authored-By: Claude Opus 5.5 --- crates/cos/README.md | 4 +- crates/cos/src/document/objects.rs | 24 ++ crates/cos/tests/it/generations.rs | 35 +++ crates/sign/src/pdf.rs | 3 + crates/sign/src/pdf/changes.rs | 168 ++++++++------ .../src/pdf/revision_classification_tests.rs | 207 ++++++++++++++++++ crates/sign/tests/it/main.rs | 1 + .../sign/tests/it/revision_classification.rs | 158 +++++++++++++ 8 files changed, 536 insertions(+), 64 deletions(-) create mode 100644 crates/sign/src/pdf/revision_classification_tests.rs create mode 100644 crates/sign/tests/it/revision_classification.rs diff --git a/crates/cos/README.md b/crates/cos/README.md index 774f2c4367..9f5fb56725 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -62,7 +62,9 @@ another. Edits never touch the original bytes. not expand compact xref ranges; the winning xref entry remains authoritative for free slots. Compressed objects require generation zero. Explicit `set` overlays accept only the supplied generation; `set_existing` and `update_dict` retain both accepted generations of a - damaged live slot, including through snapshots, and require full rewrite. Headers beyond + damaged live slot, including through snapshots, and require full rewrite. + `accepted_generations(num)` lists the generations a slot accepts, so code comparing two + revisions can tell whether a reference still reaches the same slot. Headers beyond this bounded probe are not additionally recovered. Materialized primary and hybrid rows are diagnosed before folding, so a newer row does not hide a retained disagreement. - **Tolerant, never fatal.** The lexer works on a byte slice and a position; it allocates nothing diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 8d9917d749..458840de3a 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -175,6 +175,30 @@ impl Document { } } + /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), + /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged + /// file slot (or a repair-aware edit of one) also the generation its object header carries. + /// Code comparing revisions (signature validation) uses it to tell whether the references + /// that reached an object in one revision still reach the same slot in another. + pub fn accepted_generations(&self, num: u32) -> Vec { + // Mirrors `reference_matches`, which stays allocation-free for every lookup. + let mut generations = match self.overlay.get(&num) { + Some(Slot::Set(generation, _)) => vec![*generation], + Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], + Some(Slot::Freed(_)) => Vec::new(), + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() + } + Some(XrefEntry::InStream { .. }) => vec![0], + None | Some(XrefEntry::Free { .. }) => Vec::new(), + }, + }; + generations.sort_unstable(); + generations.dedup(); + generations + } + pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { if !self.reference_matches(r) { return Ok(Arc::new(Object::Null)); diff --git a/crates/cos/tests/it/generations.rs b/crates/cos/tests/it/generations.rs index 3d56628c93..bbe2ff0b33 100644 --- a/crates/cos/tests/it/generations.rs +++ b/crates/cos/tests/it/generations.rs @@ -473,3 +473,38 @@ fn restoring_explicit_sets_removes_old_generation_aliases() { } } } + +/// `accepted_generations` lists exactly the generations `get` resolves, for every kind of slot: +/// a clean file slot, a damaged one (row and header disagree), a repair-aware edit, a plain +/// edit, a freed slot, a free row, an object stream member and an unknown number. +#[test] +fn accepted_generations_are_the_generations_get_resolves() { + let agrees = |doc: &Document, num: u32, expected: &[u16]| { + assert_eq!(doc.accepted_generations(num), expected, "object {num}"); + for generation in 0..8 { + let live = !matches!(*doc.get(ObjRef::new(num, generation)), Object::Null); + assert_eq!(live, expected.contains(&generation), "object {num} generation {generation}"); + } + }; + let doc = Document::open(contradictory_entry(false)).unwrap(); + agrees(&doc, 1, &[0]); + agrees(&doc, 3, &[0, 1]); + agrees(&doc, 9, &[]); + let mut repaired = doc.clone(); + repaired.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + agrees(&repaired, 3, &[0, 1]); + let mut narrowed = doc.clone(); + narrowed.set(ObjRef::new(3, 2), Object::Int(99)); + agrees(&narrowed, 3, &[2]); + let mut freed = doc.clone(); + freed.free(ObjRef::new(3, 0)); + agrees(&freed, 3, &[]); + agrees(&Document::open(contradictory_entry(true)).unwrap(), 3, &[]); + let mut doc = + Document::open(object_file(&[(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")])).unwrap(); + let member = doc.add(Object::Dict(Default::default())); + doc.update_dict(ObjRef::new(1, 0), |c| c.set(b"Extra".to_vec(), Object::Ref(member))).unwrap(); + let packed = Document::open(Arc::new(write_full(&doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap(); + assert!(matches!(packed.xref_entry(member.num), Some(XrefEntry::InStream { .. })), "the member is in an object stream"); + agrees(&packed, member.num, &[0]); +} diff --git a/crates/sign/src/pdf.rs b/crates/sign/src/pdf.rs index 75b42f9905..8fafff7f01 100644 --- a/crates/sign/src/pdf.rs +++ b/crates/sign/src/pdf.rs @@ -45,6 +45,9 @@ use validate::{DssEvidence, MAX_DSS_BYTES, MAX_DSS_ENTRIES}; #[cfg(test)] #[path = "pdf_discovery_tests.rs"] mod discovery_tests; +#[cfg(test)] +#[path = "pdf/revision_classification_tests.rs"] +mod revision_classification_tests; /// One signature field. #[derive(Clone, Debug)] diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..81467542e6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -7,7 +7,11 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< let Some(old) = old else { return Modification::Disallowed(vec![kind::UNREADABLE]); }; - let old_content: HashSet = pdfcraft_annot::page_refs(&old) + // Every role below is keyed by object number, not by the reference as written: a damaged + // file's slot accepts two generations (#1143), and a page signed as `3 5 R` and edited as + // `3 0 R` is still that page. The loop over changed objects refuses any slot whose accepted + // generations changed, so a number names the same slot in both revisions. + let old_content: HashSet = pdfcraft_annot::page_refs(&old) .unwrap_or_default() .into_iter() .flat_map(|pg| { @@ -21,24 +25,34 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< _ => Vec::new(), } }) + .map(|r| r.num) .collect(); // The signed revision's document-level XMP stream. - let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))); + let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))).map(|r| r.num); // The signed revision's XFA data packets: filling the form rewrites them. let old_xfa_data = xfa_data_packets(&old); // What each object was in the signed revision: its pages, and its trailer /Info. A changed // object is judged by the role it held when signed, never by the labels or keys its new, // attacker-written version carries (relabelling a signed page /Type /Sig, or dropping its // /Type and adding /ModDate, must not reclassify it away from page content). - let signed_pages: HashSet = pdfcraft_annot::page_refs(&old).unwrap_or_default().into_iter().collect(); + let signed_page_refs = pdfcraft_annot::page_refs(&old).unwrap_or_default(); + let signed_pages: HashSet = signed_page_refs.iter().map(|r| r.num).collect(); // The signed pages' /Annots arrays that are objects of their own. Adding an annotation updates // such an array in place (the page dictionary does not change), so it is judged like a // page's inline /Annots: by the annotations it gained. - let signed_annots: HashSet = - signed_pages.iter().filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))).collect(); + let signed_annots: HashSet = signed_page_refs + .iter() + .filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))) + .map(|r| r.num) + .collect(); + // The annotations the signed pages listed. One of them listed again elsewhere was moved or + // copied there, not added. + let placed_annots: HashSet = + signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); // What each signed page draws from, by the role it held when signed (#1787). - let signed_resources = signed_page_resources(&old, &signed_pages); - let old_info = old.trailer().get(b"Info").and_then(Object::as_ref); + let signed_resources = signed_page_resources(&old, &signed_page_refs); + let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. @@ -61,8 +75,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // while every comparison here finds the signed object untouched. No writer has a reason to // add such an entry after signing: it is never a permitted change. for (_, num) in doc.xref_oddities().iter().filter(|o| !old.xref_oddities().contains(o)) { - let r = ObjRef { num: *num, generation: doc.generation(*num) }; - let what = if old_content.contains(&r) || signed_pages.contains(&r) { kind::PAGE_CONTENT } else { kind::OTHER }; + let what = if old_content.contains(num) || signed_pages.contains(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); } @@ -80,6 +93,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + placed_annots: &placed_annots, signed_resources: &signed_resources, old_info, dss: &dss, @@ -88,6 +102,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< if same_place(num) { continue; } + // The signed revision's references reached this slot by the generations it accepted. If + // it accepts others now, those references may reach nothing here while another reader + // follows them back through /Prev to an intermediate revision, or to another alias, and + // shows what that holds: the object compared below is not provably what is displayed. + // No writer changes a signed object's generation. + let signed_generations = old.accepted_generations(num); + if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; + if !disallowed.contains(&what) { + disallowed.push(what); + } + continue; + } let r = ObjRef { num, generation: doc.generation(num) }; let new = doc.get(r); let before = old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)); @@ -124,9 +151,8 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // rewritten by every save and is not a change. let current: HashSet = doc.object_numbers().into_iter().collect(); for num in old.object_numbers().into_iter().filter(|n| !current.contains(n)) { - let r = ObjRef { num, generation: old.generation(num) }; - let was = old.get(r); - let signed_role = old_content.contains(&r) || signed_pages.contains(&r) || signed_resources.contains_key(&r); + let was = old.get(ObjRef { num, generation: old.generation(num) }); + let signed_role = signed_role(num); // A signed page's content or resource is judged by that role, whatever /Type its stream carried. if !signed_role && matches!(&*was, Object::Stream(s) if matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))) { continue; @@ -142,13 +168,14 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, - old_content: &'a HashSet, - old_metadata: Option, - old_xfa_data: &'a HashSet, - signed_pages: &'a HashSet, - signed_annots: &'a HashSet, - signed_resources: &'a HashMap, - old_info: Option, + old_content: &'a HashSet, + old_metadata: Option, + old_xfa_data: &'a HashSet, + signed_pages: &'a HashSet, + signed_annots: &'a HashSet, + placed_annots: &'a HashSet, + signed_resources: &'a HashMap, + old_info: Option, dss: &'a HashSet, } @@ -170,7 +197,7 @@ impl ChangedObjectClassifier<'_> { } fn resource_kind(&self, r: ObjRef, new: &Object, before: Option<&Object>) -> Option<&'static str> { - match (self.signed_resources.get(&r), new) { + match (self.signed_resources.get(&r.num), new) { (Some(false), _) => Some(kind::PAGE_CONTENT), (Some(true), Object::Dict(dict)) if self.keeps_previous_entries(dict, before) => None, (Some(true), _) => Some(kind::PAGE_CONTENT), @@ -183,32 +210,40 @@ impl ChangedObjectClassifier<'_> { } fn is_signed_annotation_array(&self, r: ObjRef, before: Option<&Object>) -> bool { - self.signed_annots.contains(&r) && before.is_some_and(|object| object.as_array().is_some()) + self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: HashSet = - before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - let kept: HashSet = now.iter().filter_map(Object::as_ref).collect(); - let removed: Vec = was.iter().filter(|reference| !kept.contains(reference)).copied().collect(); - if removed.is_empty() { - let added: Vec = now.iter().filter_map(Object::as_ref).filter(|reference| !was.contains(reference)).collect(); - added_annots(self.doc, &added) - } else if removed.iter().any(|reference| self.old.get(*reference).as_dict().is_some_and(|dict| dict.name(b"Subtype") == Some(b"Widget"))) { - // A form field or signature widget taken off the page: no permission covers it. + let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); + self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + } + + /// A signed page's annotations going from `was` to `now`, compared by object number. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + let had: HashSet = was.iter().map(|r| r.num).collect(); + let kept: HashSet = now.iter().map(|r| r.num).collect(); + let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); + let added: Vec = now.iter().filter(|r| !had.contains(&r.num)).copied().collect(); + let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); + // A signed annotation listed on another page was moved or copied there. + let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + if removed.iter().chain(&moved).any(|num| was_widget(*num)) { + // A form field or signature widget taken off its page, or onto another: no permission covers it. kind::OTHER - } else { - // Deleting comments is a comment change (permitted at P 3 only). + } else if !removed.is_empty() || !moved.is_empty() { + // Deleting or moving comments is a comment change (permitted at P 3 only). kind::COMMENTS + } else { + added_annots(self.doc, &added) } } fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { let previous = before.and_then(Object::as_dict); - if self.signed_pages.contains(&r) { + if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. - page_change_kind(self.doc, self.old, dict, previous) - } else if previous.is_some() && Some(r) == self.old_info { + self.page_change_kind(dict, previous) + } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. kind::METADATA } else { @@ -216,17 +251,29 @@ impl ChangedObjectClassifier<'_> { } } + /// A changed object that was a page in the signed revision, judged by what the page shows + /// rather than by any `/Type` its new version carries. Only its annotations may change (see + /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped + /// `/Type`) is a page content change. + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + match before { + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, + Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), + None => kind::PAGE_CONTENT, + } + } + fn stream_change(&self, r: ObjRef, stream: &pdfcraft_cos::Stream, before: Option<&Object>) -> Option<&'static str> { // File plumbing is the stream's role when signed; a later /Type label cannot change it. let was_plumbing = matches!(before, Some(Object::Stream(previous)) if matches!(previous.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))); - if self.old_content.contains(&r) { + if self.old_content.contains(&r.num) { Some(kind::PAGE_CONTENT) } else if before.is_none() || was_plumbing { None - } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r) == self.old_metadata { + } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r.num) == self.old_metadata { // Only the signed catalog's XMP stream is metadata. Some(kind::METADATA) - } else if self.old_xfa_data.contains(&r) { + } else if self.old_xfa_data.contains(&r.num) { // Signed XFA data and state packets are changed by a form fill. Some(kind::FORM_FILL) } else { @@ -238,7 +285,7 @@ impl ChangedObjectClassifier<'_> { /// The streams of `doc`'s XFA form that hold its data and state: the `datasets` and `form` /// packets of the AcroForm's `/XFA` array (name, stream, name, stream, …). A form given as one /// XFA stream holds its template too, and is not listed. -fn xfa_data_packets(doc: &Document) -> HashSet { +fn xfa_data_packets(doc: &Document) -> HashSet { let Some(form) = acroform(doc) else { return HashSet::new() }; let Some(xfa) = form.get(b"XFA").map(|x| doc.resolve(x)) else { return HashSet::new() }; let Object::Array(xfa) = &*xfa else { return HashSet::new() }; @@ -247,6 +294,7 @@ fn xfa_data_packets(doc: &Document) -> HashSet { .iter() .filter(|[name, _]| doc.resolve(name).as_string().is_some_and(|s| matches!(s.to_text().as_str(), "datasets" | "form"))) .filter_map(|[_, packet]| packet.as_ref()) + .map(|r| r.num) .collect() } @@ -442,23 +490,24 @@ fn is_signature_dict(d: &Dict) -> bool { /// properties), where any change changes what the page shows. Resources are the page's own or /// inherited. Bounded by depth and by the number of objects visited; past the bound the rest is /// judged as before. -fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap { +fn signed_page_resources(old: &Document, pages: &[ObjRef]) -> HashMap { const CATEGORIES: [&[u8]; 7] = [b"Font", b"XObject", b"ExtGState", b"ColorSpace", b"Pattern", b"Shading", b"Properties"]; const MAX_VISITED: usize = 50_000; const MAX_DEPTH: u8 = 32; const MAX_INHERITANCE: usize = 64; /// Everything reachable from `o` is content the page draws (a leaf role, overriding a - /// container one reached by another path). A signed page or the catalog reached this way - /// (a `/P` or `/Parent` back-reference) keeps the role of its own: the walk stops there. - fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { + /// container one reached by another path, under any generation alias). A signed page or the + /// catalog reached this way (a `/P` or `/Parent` back-reference) keeps the role of its own: + /// the walk stops there. + fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { if roles.len() >= MAX_VISITED || depth > MAX_DEPTH { return; } let resolved; let o = match o { Object::Ref(r) => { - if stop.contains(r) || roles.insert(*r, false).is_some() { + if stop.contains(&r.num) || roles.insert(r.num, false).is_some() { return; } resolved = old.get(*r); @@ -490,14 +539,14 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap = pages.iter().map(|r| r.num).collect(); + stop.extend(old.root().map(|r| r.num)); for page in pages { let Some(res) = resources(old, *page) else { continue }; - let container = |o: &Object, roles: &mut HashMap| -> Arc { + let container = |o: &Object, roles: &mut HashMap| -> Arc { match o { Object::Ref(r) => { - roles.entry(*r).or_insert(true); + roles.entry(r.num).or_insert(true); old.get(*r) } other => Arc::new(other.clone()), @@ -516,21 +565,6 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap) -> &'static str { - match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => { - let was = refs(old, b.get(b"Annots")); - let added: Vec = refs(doc, d.get(b"Annots")).into_iter().filter(|r| !was.contains(r)).collect(); - added_annots(doc, &added) - } - None => kind::PAGE_CONTENT, - } -} - /// Whether the catalog `d` keeps every developer extension the signed catalog `b` declared, each /// with the same value: it may only declare more. fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { @@ -557,6 +591,14 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) { return kind::PAGE_CONTENT; } + // A signed form field or widget keeps that role: its value, appearance or signature may + // change, but turning it into a comment, another kind of annotation or another type of + // field is no form fill. + if before.is_some_and(|b| { + (b.name(b"Subtype") == Some(b"Widget") || b.contains(b"FT")) && (d.name(b"Subtype") != b.name(b"Subtype") || d.name(b"FT") != b.name(b"FT")) + }) { + return kind::OTHER; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs new file mode 100644 index 0000000000..ed338c24de --- /dev/null +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -0,0 +1,207 @@ +//! Changes made after signing are judged against what a reader resolves and by the role each +//! object held when signed: a newer generation, a repaired generation alias, or a widget +//! replaced, removed or moved never passes as a permitted change. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object}; + +use super::Modification; +use super::changes::classify_changes; + +/// One cross-reference row: an object written with header generation `.0` under row generation `.1`. +enum Row<'a> { + Obj(u16, u16, &'a str), +} + +/// A classic-xref file built one revision at a time; each later revision points back with /Prev. +#[derive(Clone)] +struct File { + bytes: Vec, + size: u32, + xref: Option, +} + +impl File { + fn new() -> Self { + File { bytes: b"%PDF-1.7\n".to_vec(), size: 1, xref: None } + } + + fn revision(mut self, rows: &[(u32, Row<'_>)]) -> Self { + let mut table = Vec::new(); + for (num, row) in rows { + self.size = self.size.max(num + 1); + let Row::Obj(header, generation, body) = row; + table.push(format!("{num} 1\n{:010} {generation:05} n \n", self.bytes.len())); + self.bytes.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + let at = self.bytes.len(); + self.bytes.extend_from_slice(b"xref\n"); + if self.xref.is_none() { + self.bytes.extend_from_slice(b"0 1\n0000000000 65535 f \n"); + } + self.bytes.extend_from_slice(table.concat().as_bytes()); + let prev = self.xref.map(|p| format!(" /Prev {p}")).unwrap_or_default(); + self.bytes.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R{prev} >>\nstartxref\n{at}\n%%EOF\n", self.size).as_bytes()); + self.xref = Some(at); + self + } + + fn open(&self) -> Document { + Document::open(Arc::new(self.bytes.clone())).unwrap() + } +} + +const PAGES: &str = "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"; +const TEXT_FIELD: &str = "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 20 120 40] /P 3 0 R >>"; +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 20 40 40] /Contents (note) >>"; + +fn page(extra: &str) -> String { + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] {extra} >>") +} + +fn r(num: u32, generation: u16) -> ObjRef { + ObjRef { num, generation } +} + +fn assert_refused(doc: &Document, old: Document, p: u8) { + let verdict = classify_changes(doc, Some(old), Some(p)); + assert!(matches!(verdict, Modification::Disallowed(_)), "P={p}: main returns {verdict:?}"); +} + +/// Two pages and a text field on page 3, listed in /Fields and in the page's inline /Annots. +fn form(page3: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(page3))), + (4, Row::Obj(0, 0, &page("/Annots []"))), + (5, Row::Obj(0, 0, TEXT_FIELD)), + (6, Row::Obj(0, 0, "[5 0 R]")), + ]) +} + +#[test] +fn content_stream_intermediate_generation_cover() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(""))), + (4, Row::Obj(0, 0, &page("/Contents 5 0 R"))), + (5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let now = signed + .clone() + .revision(&[(5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nQ q\nendstream"))]) + .revision(&[(5, Row::Obj(1, 1, "<< /Length 3 >>\nstream\nq Q\nendstream"))]); + let (doc, old) = (now.open(), signed.open()); + assert_eq!(doc.revisions().len(), 3); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.get(r(5, 1)), old.get(r(5, 0)), "the newest generation holds the signed bytes"); + assert_refused(&doc, old, 2); +} + +#[test] +fn widget_removed_from_inline_annots() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn widget_moved_between_pages() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + doc.update_dict(r(4, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(r(5, 0))]))).unwrap(); + doc.update_dict(r(5, 0), |w| w.set(b"P".to_vec(), Object::Ref(r(4, 0)))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn annots_moved_to_new_object_dropping_widget() { + let signed = form("/Annots 6 0 R"); + let mut doc = signed.open(); + let empty = doc.add(Object::Array(Vec::new())); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Ref(empty))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_page_relabelled_as_field() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 5 R 4 0 R] /Count 2 >>")), + (3, Row::Obj(5, 0, &page(""))), + (4, Row::Obj(0, 0, &page(""))), + ]); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| { + p.remove(b"Type"); + p.set(b"FT".to_vec(), Object::name("Tx")); + p.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +#[test] +fn resource_container_and_drawn_alias() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Contents 6 0 R /Resources << /Font 4 0 R /ExtGState << /GS1 4 5 R >> >>"))), + (4, Row::Obj(5, 0, "<< /F1 5 0 R /CA 1 >>")), + (5, Row::Obj(0, 0, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), + (6, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let mut doc = signed.open(); + doc.update_dict(r(4, 0), |d| d.set(b"FT".to_vec(), Object::name("Tx"))).unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +/// A field signed as `4 5 R` / `4 0 R` (header 5, row 0) that a later revision rewrites. +fn aliased_field(fields: &str, annots: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, &format!("<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> >>"))), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page(&format!("/Annots [{annots}]")))), + (4, Row::Obj(5, 0, TEXT_FIELD)), + ]) +} + +fn set_annots(doc: &mut Document, annot: ObjRef) { + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annot)]))).unwrap(); +} + +#[test] +fn repaired_alias_set_changes_widget_to_comment() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 0, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_slot_becomes_disjoint_generation() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 7, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn hidden_shared_alias_form_fill() { + let signed = aliased_field("4 5 R", "4 0 R"); + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let now = signed.clone().revision(&[(4, Row::Obj(5, 1, &filled))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 1)); + // At P=3 a relisted annotation is permitted whatever it is: only the generations refuse it. + for p in [2, 3] { + assert_refused(&doc, signed.open(), p); + } +} diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index d526b01e7c..791b2b503b 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -7,4 +7,5 @@ mod cms; mod crypto; mod encrypted_signing; mod pdf; +mod revision_classification; mod store_keys; diff --git a/crates/sign/tests/it/revision_classification.rs b/crates/sign/tests/it/revision_classification.rs new file mode 100644 index 0000000000..9d36c3f57e --- /dev/null +++ b/crates/sign/tests/it/revision_classification.rs @@ -0,0 +1,158 @@ +//! A certification signature stays valid only while what a reader shows is what was signed: +//! revisions appended by hand that reach back to an older generation, or that replace a signed +//! form field in place, are refused. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// One row of a hand-written revision: `num header obj body` under row generation `row`, or a +/// free row with its next generation. +enum Row<'a> { + Obj { num: u32, header: u16, row: u16, body: &'a str }, + Free { num: u32, next: u16 }, +} + +/// `base` with one more revision of `rows`, written as a classic table pointing back with /Prev. +fn append(base: &[u8], rows: &[Row<'_>]) -> Vec { + let doc = open(base); + let size = doc.trailer().get(b"Size").and_then(Object::as_int).unwrap(); + let root = doc.root().unwrap(); + let tail = String::from_utf8_lossy(&base[base.len().saturating_sub(64)..]).into_owned(); + let prev: usize = tail.rsplit("startxref").next().unwrap().split_whitespace().next().unwrap().parse().unwrap(); + let mut out = base.to_vec(); + let mut table = String::new(); + for row in rows { + match row { + Row::Obj { num, header, row, body } => { + table.push_str(&format!("{num} 1\n{:010} {row:05} n \n", out.len())); + out.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + Row::Free { num, next } => table.push_str(&format!("{num} 1\n0000000000 {next:05} f \n")), + } + } + let xref = out.len(); + out.extend_from_slice( + format!("xref\n{table}trailer\n<< /Size {size} /Root {} {} R /Prev {prev} >>\nstartxref\n{xref}\n%%EOF\n", root.num, root.generation) + .as_bytes(), + ); + out +} + +fn opts(certify: u8) -> SignOptions { + SignOptions { + page: 0, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(certify), + ..SignOptions::default() + } +} + +/// The certification signature of `bytes`, which must be refused. +fn assert_refused(bytes: &[u8]) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed).unwrap(); + assert!(matches!(s.modification, Modification::Disallowed(_)) && s.status != Status::Valid, "main returns {:?}, {:?}", s.status, s.modification); +} + +fn certify(unsigned: &[u8], p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + pdfcraft_sign::sign(&open(unsigned), &id, &opts(p)).unwrap() +} + +const PAGE2: &str = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>"; + +/// Revision 2 rotates page 2 (`4 0 obj`); revision 3 writes the signed page back as `4 1 obj`. +/// The page tree still says `4 0 R`. +fn intermediate_generation_cover(p: u8) { + let unsigned = write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>", + PAGE2, + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page one) Tj ET\nendstream", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page two) Tj ET\nendstream", + ]); + let signed = certify(&unsigned, p); + let rotated = PAGE2.replace(" >> >> >>", " >> >> /Rotate 180 >>"); + let two = append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: &rotated }]); + let three = append(&two, &[Row::Obj { num: 4, header: 1, row: 1, body: PAGE2 }]); + let doc = open(&three); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.revisions().len(), open(&signed).revisions().len() + 2); + assert_eq!(doc.get(pdfcraft_cos::ObjRef { num: 4, generation: 1 }), open(&signed).get(pdfcraft_cos::ObjRef { num: 4, generation: 0 })); + assert_refused(&three); +} + +#[test] +fn intermediate_generation_cover_p1() { + intermediate_generation_cover(1); +} + +#[test] +fn intermediate_generation_cover_p2() { + intermediate_generation_cover(2); +} + +#[test] +fn intermediate_generation_cover_p3() { + intermediate_generation_cover(3); +} + +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 200 40 220] /Contents (note) >>"; + +/// A page with text field `4 0 R`, listed in /Fields and in the page's inline /Annots, certified +/// allowing comments. +fn certified_form() -> Vec { + certify( + &write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [4 0 R] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + ]), + 3, + ) +} + +#[test] +fn widget_replaced_in_place_by_comment() { + let signed = certified_form(); + assert_refused(&append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} + +#[test] +fn widget_freed_then_reused_as_comment() { + let signed = certified_form(); + let freed = append(&signed, &[Row::Free { num: 4, next: 1 }]); + assert_refused(&append(&freed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} From de344a4bb1dd02bad3a9cffd13cf69a5ac73292b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 22:44:26 +0000 Subject: [PATCH 02/14] sign: compare annotation lists by what resolves, within a bound An annotation list is compared by the entries that resolve in each revision, and an unchanged list is no change. Expanding annotation lists is bounded per classification. A damaged slot rewritten with fewer accepted generations is the same slot when nothing signed refers to a generation it gave up. Co-Authored-By: Claude Opus 5.5 --- crates/sign/src/pdf/changes.rs | 142 +++++++++++++++--- .../src/pdf/revision_classification_tests.rs | 77 ++++++++++ 2 files changed, 196 insertions(+), 23 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 81467542e6..ef4b8868c6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -1,5 +1,11 @@ use super::types::{Modification, kind}; use super::{Arc, Dict, Document, HashMap, HashSet, ObjRef, Object}; +use std::cell::{Cell, OnceCell}; + +/// The most annotation-list entries one classification expands. Pages sharing one long /Annots +/// array, or many pages rewritten to list one, would otherwise cost pages × entries; past this +/// the lists are not compared, and the change is refused. +const MAX_ANNOT_ENTRIES: usize = 1 << 22; /// What later revisions changed, classified as Acrobat reports it, under DocMDP `p` (or none: /// an approval signature permits form fill, comments and further signatures). @@ -47,13 +53,31 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< .collect(); // The annotations the signed pages listed. One of them listed again elsewhere was moved or // copied there, not added. - let placed_annots: HashSet = - signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); + let annot_budget = Cell::new(MAX_ANNOT_ENTRIES); + let (mut placed_annots, mut expanded, mut annots_unbounded) = (HashSet::new(), HashSet::new(), false); + for pg in &signed_page_refs { + let page = old.get(*pg); + let Some(list) = page.as_dict().and_then(|d| d.get(b"Annots")) else { continue }; + // Pages sharing one array object list the same annotations. + if list.as_ref().is_some_and(|r| !expanded.insert(r.num)) { + continue; + } + let Some(found) = annot_refs(&old, Some(list), &annot_budget) else { + annots_unbounded = true; + break; + }; + placed_annots.extend(found.into_iter().map(|r| r.num)); + } // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); + if annots_unbounded { + disallowed.push(kind::OTHER); + } + // Every reference the signed revision makes, read only when a slot's generations narrowed. + let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. let same_place = |num: u32| -> bool { @@ -94,6 +118,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< signed_pages: &signed_pages, signed_annots: &signed_annots, placed_annots: &placed_annots, + annot_budget: &annot_budget, signed_resources: &signed_resources, old_info, dss: &dss, @@ -106,9 +131,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // it accepts others now, those references may reach nothing here while another reader // follows them back through /Prev to an intermediate revision, or to another alias, and // shows what that holds: the object compared below is not provably what is displayed. - // No writer changes a signed object's generation. + // No writer changes a signed object's generation. A damaged slot rewritten cleanly + // accepts fewer: that is the same slot when nothing signed refers to it by a generation + // it gave up. let signed_generations = old.accepted_generations(num); - if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let generations = doc.accepted_generations(num); + let narrowed_unreferenced = || { + !generations.is_empty() + && generations.iter().all(|g| signed_generations.contains(g)) + && signed_references.get_or_init(|| references(&old)).as_ref().is_some_and(|found: &HashSet| { + signed_generations.iter().filter(|g| !generations.contains(g)).all(|g| !found.contains(&ObjRef { num, generation: *g })) + }) + }; + if !signed_generations.is_empty() && signed_generations != generations && !narrowed_unreferenced() { let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); @@ -174,6 +209,7 @@ struct ChangedObjectClassifier<'a> { signed_pages: &'a HashSet, signed_annots: &'a HashSet, placed_annots: &'a HashSet, + annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, dss: &'a HashSet, @@ -188,8 +224,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { - Object::Array(now) if self.is_signed_annotation_array(r, before) => Some(self.annotation_array_change(now, before)), - Object::Dict(dict) => Some(self.dictionary_change(r, dict, before)), + Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), + Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), _ if before.is_none() => None, _ => Some(kind::OTHER), @@ -213,13 +249,21 @@ impl ChangedObjectClassifier<'_> { self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } - fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + fn annotation_array_change(&self, now: &Object, before: Option<&Object>) -> Option<&'static str> { + match (annot_refs(self.old, before, self.annot_budget), annot_refs(self.doc, Some(now), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + } } - /// A signed page's annotations going from `was` to `now`, compared by object number. - fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + /// A signed page's annotations going from `was` to `now`, compared by object number. Only an + /// entry that resolves is on the page: one rewritten to a generation its slot does not accept + /// draws nothing, whatever its number. `None` when the page shows the same annotations in + /// the same order. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> Option<&'static str> { + let live = + |doc: &Document, list: &[ObjRef]| -> Vec { list.iter().filter(|r| !matches!(*doc.get(**r), Object::Null)).copied().collect() }; + let (was, now) = (live(self.old, was), live(self.doc, now)); let had: HashSet = was.iter().map(|r| r.num).collect(); let kept: HashSet = now.iter().map(|r| r.num).collect(); let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); @@ -227,27 +271,31 @@ impl ChangedObjectClassifier<'_> { let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); // A signed annotation listed on another page was moved or copied there. let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + // The annotations kept, in their new order: another order draws another one on top. + let reordered = !now.iter().map(|r| r.num).filter(|num| had.contains(num)).eq(was.iter().map(|r| r.num)); if removed.iter().chain(&moved).any(|num| was_widget(*num)) { // A form field or signature widget taken off its page, or onto another: no permission covers it. - kind::OTHER - } else if !removed.is_empty() || !moved.is_empty() { - // Deleting or moving comments is a comment change (permitted at P 3 only). - kind::COMMENTS + Some(kind::OTHER) + } else if !removed.is_empty() || !moved.is_empty() || reordered { + // Deleting, moving or restacking comments is a comment change (permitted at P 3 only). + Some(kind::COMMENTS) + } else if added.is_empty() { + None } else { - added_annots(self.doc, &added) + Some(added_annots(self.doc, &added)) } } - fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { + fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> Option<&'static str> { let previous = before.and_then(Object::as_dict); if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. - kind::METADATA + Some(kind::METADATA) } else { - change_kind(self.doc, self.old, dict, previous) + Some(change_kind(self.doc, self.old, dict, previous)) } } @@ -255,11 +303,14 @@ impl ChangedObjectClassifier<'_> { /// rather than by any `/Type` its new version carries. Only its annotations may change (see /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped /// `/Type`) is a page content change. - fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> Option<&'static str> { match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), - None => kind::PAGE_CONTENT, + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => Some(kind::PAGE_CONTENT), + Some(b) => match (annot_refs(self.old, b.get(b"Annots"), self.annot_budget), annot_refs(self.doc, d.get(b"Annots"), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + }, + None => Some(kind::PAGE_CONTENT), } } @@ -439,6 +490,51 @@ fn added_annots(doc: &Document, added: &[ObjRef]) -> &'static str { } } +/// The references in the annotation list `o` (inline or indirect), its length spent from +/// `budget`. `None` past the budget. +fn annot_refs(doc: &Document, o: Option<&Object>, budget: &Cell) -> Option> { + let held; + let items = match o { + Some(Object::Ref(_)) => { + held = o.map(|o| doc.resolve(o)); + held.as_deref().and_then(Object::as_array) + } + other => other.and_then(Object::as_array), + }; + let Some(items) = items else { return Some(Vec::new()) }; + budget.set(budget.get().checked_sub(items.len())?); + Some(items.iter().filter_map(Object::as_ref).collect()) +} + +/// Every indirect reference in `doc`'s objects and trailer. `None` (nothing can be shown +/// unreferenced) when an object can't be read, or past a bound on references or nesting. +fn references(doc: &Document) -> Option> { + const MAX_REFERENCES: usize = 1 << 22; + const MAX_DEPTH: u8 = 64; + fn walk(o: &Object, depth: u8, found: &mut HashSet) -> bool { + if depth > MAX_DEPTH { + return false; + } + match o { + Object::Ref(r) => { + found.insert(*r); + found.len() <= MAX_REFERENCES + } + Object::Dict(d) => d.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Stream(s) => s.dict.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Array(a) => a.iter().all(|v| walk(v, depth + 1, found)), + _ => true, + } + } + let mut found = HashSet::new(); + for num in doc.object_numbers() { + if !walk(&*doc.try_get(num).ok()?, 0, &mut found) { + return None; + } + } + doc.trailer().iter().all(|(_, v)| walk(v, 0, &mut found)).then_some(found) +} + fn refs(doc: &Document, o: Option<&Object>) -> Vec { o.map(|o| doc.resolve(o)).and_then(|a| a.as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() } diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs index ed338c24de..b5e72fcae0 100644 --- a/crates/sign/src/pdf/revision_classification_tests.rs +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -205,3 +205,80 @@ fn hidden_shared_alias_form_fill() { assert_refused(&doc, signed.open(), p); } } + +fn verdict(doc: &Document, old: Document, p: Option) -> Modification { + classify_changes(doc, Some(old), p) +} + +/// A widget whose /Annots entry is rewritten to a generation its slot never had no longer +/// resolves: it is taken off the page, inline or through an indirect array. +#[test] +fn widget_dropped_through_a_dead_generation() { + for (page3, num) in [("/Annots 6 0 R", 6), ("/Annots [5 0 R]", 3)] { + let signed = form(page3); + let body = if num == 6 { "[5 3 R]".to_owned() } else { page("/Annots [5 3 R]") }; + let now = signed.clone().revision(&[(num, Row::Obj(0, 0, &body))]); + for p in [Some(2), Some(3), None] { + let v = verdict(&now.open(), signed.open(), p); + assert!(matches!(v, Modification::Disallowed(_)), "{page3} P={p:?}: {v:?}"); + } + } +} + +/// The same annotations, listed in a new array object, change nothing on the page; listed in +/// another order they change which is drawn on top. +#[test] +fn an_unchanged_annotation_list_is_no_change_and_a_reordered_one_is_a_comment_change() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Annots [4 0 R 5 0 R]"))), + (4, Row::Obj(0, 0, COMMENT)), + (5, Row::Obj(0, 0, COMMENT)), + ]); + let moved = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots 6 0 R"))), (6, Row::Obj(0, 0, "[4 0 R 5 0 R]"))]); + assert_eq!(verdict(&moved.open(), signed.open(), Some(2)), Modification::Allowed(Vec::new())); + let reordered = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots [5 0 R 4 0 R]")))]); + assert_eq!(verdict(&reordered.open(), signed.open(), Some(2)), Modification::Disallowed(vec![super::kind::COMMENTS])); +} + +/// Many signed pages sharing one long /Annots array, all rewritten to list it through a new +/// object, are judged in bounded time. +#[test] +fn shared_annotation_arrays_are_judged_in_bounded_time() { + const PAGES: u32 = 30_000; + let first_page = 10; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let annots: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let mut rows = vec![ + (1, "<< /Type /Catalog /Pages 2 0 R >>".to_owned()), + (2, format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>")), + (3, format!("[{annots}]")), + ]; + rows.extend((0..PAGES).map(|i| (first_page + i, page("/Annots 3 0 R")))); + let rows: Vec<(u32, Row<'_>)> = rows.iter().map(|(n, b)| (*n, Row::Obj(0, 0, b))).collect(); + let signed = File::new().revision(&rows); + let repointed: Vec = (0..PAGES).map(|_| page("/Annots 4 0 R")).collect(); + let mut later: Vec<(u32, Row<'_>)> = repointed.iter().enumerate().map(|(i, b)| (first_page + i as u32, Row::Obj(0, 0, b))).collect(); + let copy = format!("[{annots}]"); + later.push((4, Row::Obj(0, 0, ©))); + let now = signed.clone().revision(&later); + let (doc, old) = (now.open(), signed.open()); + let started = std::time::Instant::now(); + let _ = verdict(&doc, old, Some(3)); + assert!(started.elapsed() < std::time::Duration::from_secs(20), "took {:?}", started.elapsed()); +} + +/// A damaged signed slot (header 5, row 0) rewritten cleanly as `4 0 obj` keeps the generation +/// every signed reference uses: a form fill. Had anything signed referred to it as `4 5 R`, that +/// reference would no longer resolve, and the change is refused. +#[test] +fn a_repaired_slot_rewritten_cleanly_is_judged_by_its_references() { + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert_eq!(verdict(&now.open(), signed.open(), Some(2)), Modification::Allowed(vec![super::kind::FORM_FILL])); + let signed = aliased_field("4 5 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert!(matches!(verdict(&now.open(), signed.open(), Some(2)), Modification::Disallowed(_))); +} From e1a3c033aaf749a758010bd1c9342c426eee2b39 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 6 Nov 2026 00:19:58 +0000 Subject: [PATCH 03/14] sign: under no-changes certification, only invisible document timestamps may follow DocMDP P=1 allowed any change classified as a signature. A signature added after signing now breaks the certification unless it is a document timestamp drawing nothing (no positive-area /Rect, or Hidden/NoView). The added objects are judged directly, so a widget with no /T or one only in a page's /Annots is covered. The document security store and a genuine timestamp still pass. Co-Authored-By: Claude Opus 4.8 --- crates/sign/src/pdf/changes.rs | 51 ++++++++++++++++++++++++ crates/sign/tests/it/pdf.rs | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..3f77fd0554 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -136,9 +136,60 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< disallowed.push(what); } } + // DocMDP P=1 ("no changes allowed", ISO 32000-2 §12.8.2.2) permits only validation data to + // follow: the document security store, and document timestamps added for PAdES long-term + // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an + // approval signature, an unsigned signature widget, or a widget that paints over the signed + // page — is a change the certification forbids, however it is labelled. + if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + allowed.retain(|k| *k != kind::SIGNATURE); + if !disallowed.contains(&kind::SIGNATURE) { + disallowed.push(kind::SIGNATURE); + } + } if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } } +/// Whether every signature object `doc` gained or changed since the signed revision `old` is an +/// invisible document timestamp. A signature annotation (a widget, or a dictionary carrying a +/// signature value or role) that the signed revision did not hold byte for byte must be a +/// `/SubFilter /ETSI.RFC3161` value in a widget that shows nothing — no positive-area `/Rect`, +/// or the Hidden or NoView flag set. An approval signature, an empty signature widget, or a +/// visible one fails, whatever `/Type` or `/SubFilter` it claims. Judged on the objects +/// themselves, not on the `/AcroForm /Fields` listing, so a widget with no `/T` or one reached +/// only through a page's `/Annots` is covered too. +fn only_invisible_timestamps_added(doc: &Document, old: &Document) -> bool { + /// A widget shows nothing: no `/Rect` enclosing a positive area, or Hidden (2) / NoView (32). + fn invisible(doc: &Document, w: &Dict) -> bool { + let flags = w.get(b"F").and_then(|o| doc.resolve(o).as_int()).unwrap_or(0); + let area = + super::listing::nums(doc, w, b"Rect").filter(|r| r.len() == 4).is_some_and(|r| (r[0] - r[2]).abs() > 0.5 && (r[1] - r[3]).abs() > 0.5); + !area || flags & (2 | 32) != 0 + } + for num in doc.object_numbers() { + let new = doc.get(ObjRef { num, generation: doc.generation(num) }); + let Some(d) = new.as_dict() else { continue }; + // A signature object: a signature value, or a field or widget that carries one. + let is_value = matches!(d.name(b"Type"), Some(b"Sig" | b"DocTimeStamp")); + let is_field = d.name(b"FT") == Some(b"Sig") || d.name(b"Subtype") == Some(b"Widget"); + if !is_value && !is_field { + continue; + } + // Unchanged since signing: the certification already covered it. + if old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)).as_deref() == Some(&*new) { + continue; + } + let value = if is_value { Some(d.clone()) } else { d.get(b"V").map(|v| doc.resolve(v)).and_then(|v| v.as_dict().cloned()) }; + // A new or changed signature object passes only as an invisible document timestamp. A + // widget with no value (an empty signature field placed on the page) never does. + let timestamp = value.as_ref().is_some_and(super::validate::is_doc_timestamp); + if !timestamp || (is_field && !invisible(doc, d)) { + return false; + } + } + true +} + struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 869a4bebd5..57efadbaf3 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -351,6 +351,77 @@ fn later_changes_are_classified_under_the_signature_permissions() { assert_eq!((s.certify, s.status), (Some(1), Status::Invalid), "{:?}", s.modification); } +/// Certify `fixture` with "no changes allowed" (DocMDP P=1). +fn certified_p1() -> (DigitalId, TrustStore, Vec) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + (id, trust, signed) +} + +/// Add a signature widget to page 3's /Annots (not to /Fields), with value `v` and rectangle +/// `rect`. +fn add_sig_widget(doc: &mut Document, v: Option<&[(&[u8], Object)]>, rect: [i64; 4]) { + let mut w = Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"Rect".to_vec(), Object::Array(rect.map(Object::Int).to_vec())); + w.set(b"P".to_vec(), Object::Ref(ObjRef { num: 3, generation: 0 })); + if let Some(entries) = v { + let mut value = Dict::new(); + for (k, o) in entries { + value.set(k.to_vec(), o.clone()); + } + let vr = doc.add(Object::Dict(value)); + w.set(b"V".to_vec(), Object::Ref(vr)); + } + let wr = doc.add(Object::Dict(w)); + doc.update_dict(ObjRef { num: 3, generation: 0 }, |pg| { + let mut a = pg.get(b"Annots").and_then(Object::as_array).cloned().unwrap_or_default(); + a.push(Object::Ref(wr)); + pg.set(b"Annots".to_vec(), Object::Array(a)); + }) + .unwrap(); +} + +/// Under P=1, a signature added after signing breaks the certification unless it is an invisible +/// document timestamp. A visible overlay, an unsigned widget, an approval signature and a visible +/// widget wearing a timestamp /SubFilter each break it; a genuine document timestamp does not. +#[test] +fn only_an_invisible_document_timestamp_survives_a_no_changes_certification() { + let (_, trust, signed) = certified_p1(); + let check = |bytes: &[u8]| signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify == Some(1)).unwrap(); + assert_eq!(check(&signed).status, Status::Valid, "{:?}", check(&signed).details); + let approval: &[(&[u8], Object)] = &[(b"Type", Object::name("Sig")), (b"SubFilter", Object::name("adbe.pkcs7.detached"))]; + let timestamp: &[(&[u8], Object)] = &[(b"Type", Object::name("DocTimeStamp")), (b"SubFilter", Object::name("ETSI.RFC3161"))]; + // Each of these breaks the certification: it is a change P=1 does not allow. + for (what, v, rect) in [ + ("visible overlay widget", None, [0, 0, 300, 300]), + ("invisible unsigned widget", None, [0, 0, 0, 0]), + ("visible approval signature", Some(approval), [0, 0, 300, 300]), + ("invisible approval signature", Some(approval), [0, 0, 0, 0]), + ("visible widget wearing a timestamp subfilter", Some(timestamp), [0, 0, 300, 300]), + ] { + let edited = edit_after(&signed, |doc| add_sig_widget(doc, v, rect)); + let s = check(&edited); + assert!(matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::SIGNATURE)), "{what}: {:?}", s.modification); + assert_ne!(s.status, Status::Valid, "{what}"); + } +} + +/// A genuine document timestamp, which draws nothing, is validation data P=1 permits: the +/// certification stays valid. +#[test] +fn a_document_timestamp_keeps_a_no_changes_certification_valid() { + let (_, trust, signed) = certified_p1(); + let tsa = TestTsa { id: tsa_id(), time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 } }; + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let s = signatures(&open(&stamped), &stamped, &trust).into_iter().find(|s| s.certify == Some(1)).unwrap(); + assert_eq!(s.status, Status::Valid, "{:?}", s.details); + assert!(matches!(&s.modification, Modification::Allowed(k) if k.contains(&kind::SIGNATURE)), "{:?}", s.modification); +} + #[test] fn invisible_signatures_and_refusals() { let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); From ddeca878dff1ea6cc2fa55adb2db84513eecda1f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 21:40:30 -0500 Subject: [PATCH 04/14] sign: judge the signed page tree, catalog and form by their signed roles Rebased onto 102cae3b/9afa1266 for landing (#6358). The signed page tree's nodes, a /Kids array of its own, the catalog by object number and an indirect AcroForm keep the roles they had when signed; a field dropped from the form is no permitted change at any level; a signed widget's /Rect or /F is page content. Co-Authored-By: Martin Ball --- crates/sign/src/pdf/changes.rs | 117 ++++++++++--- crates/sign/tests/it/main.rs | 1 + crates/sign/tests/it/signed_structure.rs | 199 +++++++++++++++++++++++ 3 files changed, 295 insertions(+), 22 deletions(-) create mode 100644 crates/sign/tests/it/signed_structure.rs diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 03253b79fb..a44c08bb7c 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -68,14 +68,26 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< }; placed_annots.extend(found.into_iter().map(|r| r.num)); } + // The signed page tree's nodes: a change to one changes the pages, whatever it is labelled + // now. Every node, and every /Kids array object of its own, is read once; past the bound the + // tree is not known in full, and no change is judged against it. + let page_tree = page_tree_nodes(&old); + let page_tree_unbounded = page_tree.is_none(); + let (page_nodes, kids_arrays) = page_tree.unwrap_or_default(); // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + // The signed catalog's /AcroForm, when it is an object of its own. + let old_form = old.root().and_then(|c| old.get(c).as_dict().and_then(|c| c.get(b"AcroForm").and_then(Object::as_ref))).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); if annots_unbounded { disallowed.push(kind::OTHER); } + if page_tree_unbounded { + // Past the bound the tree is not known in full: no change is judged against it. + disallowed.push(kind::PAGES); + } // Every reference the signed revision makes, read only when a slot's generations narrowed. let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); @@ -117,6 +129,9 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + page_nodes: &page_nodes, + kids_arrays: &kids_arrays, + old_form, placed_annots: &placed_annots, annot_budget: &annot_budget, signed_resources: &signed_resources, @@ -259,10 +274,13 @@ struct ChangedObjectClassifier<'a> { old_xfa_data: &'a HashSet, signed_pages: &'a HashSet, signed_annots: &'a HashSet, + page_nodes: &'a HashSet, + kids_arrays: &'a HashSet, placed_annots: &'a HashSet, annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, + old_form: Option, dss: &'a HashSet, } @@ -275,6 +293,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { + // A signed page-tree node's /Kids array of its own lists the pages. + Object::Array(_) if before.is_some() && self.kids_arrays.contains(&r.num) => Some(kind::PAGES), Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), @@ -342,6 +362,15 @@ impl ChangedObjectClassifier<'_> { if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) + } else if previous.is_some() && self.page_nodes.contains(&r.num) { + // A signed page-tree node is a page-tree change, whatever its new version is labelled. + Some(kind::PAGES) + } else if let Some(previous) = previous.filter(|_| self.old.root().is_some_and(|root| root.num == r.num)) { + // The signed catalog, whatever /Type its new version carries. + Some(catalog_kind(self.doc, self.old, dict, previous)) + } else if let Some(previous) = previous.filter(|_| self.old_form == Some(r.num)) { + // The signed form dictionary, held in an object of its own, whatever keys it carries now. + Some(acroform_kind(self.doc, self.old, dict, Some(previous))) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. Some(kind::METADATA) @@ -601,6 +630,10 @@ fn acroform(doc: &Document) -> Option { fn acroform_kind(doc: &Document, old: &Document, now: &Dict, before: Option<&Dict>) -> &'static str { let fallback = acroform(old).unwrap_or_default(); let before = before.unwrap_or(&fallback); + // A field taken out of the form: no DocMDP permission covers dropping a signed field. + if lost_entries(doc, old, before.get(b"Fields"), now.get(b"Fields")) { + return kind::OTHER; + } let strip = |d: &Dict| -> Vec<(Vec, Object)> { let mut v: Vec<(Vec, Object)> = d.iter().filter(|(k, _)| !matches!(k.as_slice(), b"Fields" | b"SigFlags")).map(|(k, o)| (k.to_vec(), o.clone())).collect(); @@ -724,6 +757,62 @@ fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { } } +/// A change to the signed catalog `b`, now `d`. +fn catalog_kind(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> &'static str { + let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; + if without(d, keys) != without(b, keys) { + kind::STRUCTURE + } else if !extensions_kept(doc, old, d, b) { + // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A writer + // adding a /DSS also declares the ESIC extension it belongs to, so a declaration may be + // added, but none that was signed may change or go. + kind::STRUCTURE + } else if d.get(b"AcroForm") != b.get(b"AcroForm") { + // Compare the forms themselves, inline or not. + let now = acroform(doc).unwrap_or_default(); + acroform_kind(doc, old, &now, None) + } else if b.get(b"Perms") != d.get(b"Perms") { + kind::SIGNATURE + } else { + kind::METADATA + } +} + +/// The nodes of `doc`'s page tree and the /Kids arrays that are objects of their own, by number: +/// every dictionary with /Kids from the catalog's /Pages down. Each node, and each /Kids array +/// object, is read once. `None` past a million nodes. +fn page_tree_nodes(doc: &Document) -> Option<(HashSet, HashSet)> { + const MAX_NODES: usize = 1_000_000; + let (mut nodes, mut arrays) = (HashSet::new(), HashSet::new()); + let top = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|c| c.get(b"Pages").and_then(Object::as_ref))); + let mut stack: Vec = top.into_iter().collect(); + while let Some(r) = stack.pop() { + if nodes.len() >= MAX_NODES { + return None; + } + let node = doc.get(r); + // A leaf (a page) has no /Kids. + let Some(kids) = node.as_dict().and_then(|d| d.get(b"Kids")) else { continue }; + if !nodes.insert(r.num) || kids.as_ref().is_some_and(|a| !arrays.insert(a.num)) { + continue; + } + if let Some(kids) = doc.resolve(kids).as_array() { + stack.extend(kids.iter().filter_map(Object::as_ref)); + } + } + Some((nodes, arrays)) +} + +/// Whether a list of references (`/Fields`) no longer holds an entry it held when signed. An +/// entry is still held only by a reference that resolves now and that the signed revision +/// resolved too, so to the same slot (a repaired alias counts): one rewritten to a generation +/// the slot does not accept names nothing, and one to a newer generation names another object. +fn lost_entries(doc: &Document, old: &Document, was: Option<&Object>, now: Option<&Object>) -> bool { + let live = |d: &Document, r: &ObjRef| !matches!(*d.get(*r), Object::Null); + let now: HashSet = refs(doc, now).into_iter().filter(|r| live(doc, r) && live(old, r)).map(|r| r.num).collect(); + refs(old, was).iter().filter(|r| live(old, r)).any(|r| !now.contains(&r.num)) +} + fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) -> &'static str { let ty = d.name(b"Type"); // A later revision cannot make an existing object a signature by relabelling it: a signature @@ -746,31 +835,15 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) }) { return kind::OTHER; } + // Where a signed widget sits and whether it shows are what the page shows: filling the form + // changes its value and appearance, never its rectangle or its flags. + if before.is_some_and(|b| b.name(b"Subtype") == Some(b"Widget") && (d.get(b"Rect") != b.get(b"Rect") || d.get(b"F") != b.get(b"F"))) { + return kind::PAGE_CONTENT; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, - Some(b"Catalog") => { - let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; - return match before { - Some(b) if without(d, keys) != without(b, keys) => kind::STRUCTURE, - // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A - // writer adding a /DSS also declares the ESIC extension it belongs to, so a - // declaration may be added, but none that was signed may change or go. - Some(b) if !extensions_kept(doc, old, d, b) => kind::STRUCTURE, - Some(b) => { - if d.get(b"AcroForm") != b.get(b"AcroForm") { - // Compare the forms themselves, inline or not. - let now = acroform(doc).unwrap_or_default(); - acroform_kind(doc, old, &now, None) - } else if b.get(b"Perms") != d.get(b"Perms") { - kind::SIGNATURE - } else { - kind::METADATA - } - } - None => kind::FORM_FILL, - }; - } + Some(b"Catalog") => return before.map_or(kind::FORM_FILL, |b| catalog_kind(doc, old, d, b)), Some(b"Page") => { return match before { Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index 791b2b503b..bcd6334719 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -8,4 +8,5 @@ mod crypto; mod encrypted_signing; mod pdf; mod revision_classification; +mod signed_structure; mod store_keys; diff --git a/crates/sign/tests/it/signed_structure.rs b/crates/sign/tests/it/signed_structure.rs new file mode 100644 index 0000000000..471a43c8fe --- /dev/null +++ b/crates/sign/tests/it/signed_structure.rs @@ -0,0 +1,199 @@ +//! A certified document's page tree, catalog and form keep the roles they had when signed: a +//! later revision that relabels them, drops a field from the form, or hides or moves a signed +//! field's widget is no form fill. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object, PdfString, SaveOptions, write_incremental}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, kind, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn r(num: u32) -> ObjRef { + ObjRef { num, generation: 0 } +} + +/// Two pages, listed by the /Kids array object 6; page 3 holds text field 5, listed in /Fields. +fn form() -> Vec { + write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>", + "<< /Type /Pages /Kids 6 0 R /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [5 0 R] >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + "[3 0 R 4 0 R]", + ]) +} + +fn certified(p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let opts = SignOptions { + page: 1, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(p), + ..SignOptions::default() + }; + pdfcraft_sign::sign(&open(&form()), &id, &opts).unwrap() +} + +fn edit_after(signed: &[u8], f: impl FnOnce(&mut Document)) -> Vec { + let mut doc = open(signed); + f(&mut doc); + write_incremental(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap() +} + +/// The certification signature of `bytes`: its status and modification verdict. +fn verdict(bytes: &[u8]) -> (Status, Modification) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify.is_some()).unwrap(); + (s.status, s.modification) +} + +/// `bytes` is refused, for (among others) `expected`. +fn assert_refused(what: &str, bytes: &[u8], expected: &str) { + let (status, modification) = verdict(bytes); + assert!( + matches!(&modification, Modification::Disallowed(kinds) if kinds.contains(&expected)) && status != Status::Valid, + "{what}: {status:?} {modification:?}, expected {expected}" + ); +} + +#[test] +fn a_signed_page_tree_node_is_judged_as_the_page_tree() { + let signed = certified(2); + // Relabelled as a field and dropping a page. + assert_refused( + "node relabelled", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.remove(b"Type"); + n.set(b"T".to_vec(), PdfString::text("x")); + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(r(4))])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Turning every page through an inherited attribute. + assert_refused( + "inherited rotation", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.set(b"DR".to_vec(), Object::Dict(Default::default())); + n.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Reordering the pages through the node's /Kids array object. + assert_refused( + "pages reordered", + &edit_after(&signed, |doc| doc.set(r(6), Object::Array(vec![Object::Ref(r(4)), Object::Ref(r(3))]))), + kind::PAGES, + ); +} + +#[test] +fn a_signed_catalog_is_judged_as_the_catalog_whatever_its_type() { + let signed = certified(2); + assert_refused( + "catalog relabelled", + &edit_after(&signed, |doc| { + let content = doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(Default::default(), b"0 0 m 300 300 l S".to_vec()))); + let pages = doc.add(Object::Dict(Default::default())); + let mut page = pdfcraft_cos::Dict::new(); + page.set(b"Parent".to_vec(), Object::Ref(pages)); + page.set(b"MediaBox".to_vec(), Object::Array([0, 0, 300, 300].map(Object::Int).to_vec())); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + doc.update_dict(pages, |n| { + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(page)])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + c.remove(b"Type"); + c.set(b"FT".to_vec(), Object::name("Tx")); + c.set(b"Pages".to_vec(), Object::Ref(pages)); + }) + .unwrap(); + }), + kind::STRUCTURE, + ); +} + +#[test] +fn a_field_dropped_from_the_form_is_no_change_any_level_permits() { + for p in [1, 2] { + let signed = certified(p); + let set_fields = |fields: Vec| { + edit_after(&signed, move |doc| { + let root = doc.root().unwrap(); + let mut form = + doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm")).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap(); + let kept: Vec = fields + .into_iter() + .chain( + form.get(b"Fields") + .and_then(Object::as_array) + .into_iter() + .flatten() + .filter(|f| f.as_ref().is_some_and(|f| f.num != 5)) + .cloned(), + ) + .collect(); + form.set(b"Fields".to_vec(), Object::Array(kept)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(form))).unwrap(); + }) + }; + assert_refused(&format!("P={p} field dropped"), &set_fields(Vec::new()), kind::OTHER); + let dead = ObjRef { num: 5, generation: 3 }; + assert_refused(&format!("P={p} field listed by a dead generation"), &set_fields(vec![Object::Ref(dead)]), kind::OTHER); + // The field's slot taken by a newer generation: another object, listed in its place. + let newer = ObjRef { num: 5, generation: 1 }; + let replaced = edit_after(&set_fields(vec![Object::Ref(newer)]), |doc| { + let mut field = doc.get(r(5)).as_dict().cloned().unwrap(); + field.set(b"V".to_vec(), PdfString::text("Eve")); + doc.set(newer, Object::Dict(field)); + }); + assert_refused(&format!("P={p} field replaced by a newer generation"), &replaced, kind::OTHER); + } +} + +#[test] +fn a_signed_fields_widget_may_be_filled_but_not_hidden_or_moved() { + let signed = certified(2); + let edit = |f: &dyn Fn(&mut pdfcraft_cos::Dict)| edit_after(&signed, |doc| doc.update_dict(r(5), |w| f(w)).unwrap()); + let filled = edit(&|w| w.set(b"V".to_vec(), PdfString::text("Bob"))); + assert_eq!(verdict(&filled), (Status::Valid, Modification::Allowed(vec![kind::FORM_FILL]))); + assert_refused("hidden", &edit(&|w| w.set(b"F".to_vec(), Object::Int(2))), kind::PAGE_CONTENT); + assert_refused("moved", &edit(&|w| w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4]))), kind::PAGE_CONTENT); +} From 560c68db899ae332b9fda18677583ae9a45e3043 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:41:29 -0500 Subject: [PATCH 05/14] fix(sign): an existing object is judged by the role it had when signed Rebased #5187 onto 9afa1266. Landing the signed-role rule only: an existing object that gains form or annotation keys is judged by what it was when signed, and a signed annotation turned into another kind is no comment. The two P1 regressions the original PR carried are not landed: the signature decision still reads the new /FT (a signed widget relabelled as a text field is not signing), and the dropped-/Fields check runs before the form-key comparison (already carried by #6358's lost_entries). Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 23 ++++-- crates/sign/tests/it/pdf.rs | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index a44c08bb7c..8cfd823fb0 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -858,19 +858,32 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) Some(b"Pages") => return if before.is_some() { kind::PAGES } else { kind::STRUCTURE }, _ => {} } - match d.name(b"Subtype") { + // The role an existing object plays (annotation, form field, the form) is the one it had when + // signed: keys its new version carries are written by whoever changed it. An indirect + // /OCProperties rewritten with an added /FT still decides which layers show (#1787 judges a + // signed page's resources this way; this applies it to every other object). + let role = before.unwrap_or(d); + // Nor does an existing annotation or widget turned into another kind of annotation edit the + // one it was: an empty signature widget rewritten as a /Square that covers signed text is no + // signature change, and no comment either. + if before.is_some_and(|b| b.name(b"Subtype") != d.name(b"Subtype")) { + return kind::OTHER; + } + // The new version's /FT still decides whether this is a signature: a signed widget relabelled + // as a text field is not a signature change, whatever it was when signed. + match role.name(b"Subtype") { Some(b"Widget") => return if d.name(b"FT") == Some(b"Sig") && signature_ok { kind::SIGNATURE } else { kind::FORM_FILL }, Some(b"Link") => return kind::LINKS, - Some(_) if ty == Some(b"Annot") || d.contains(b"Rect") => return kind::COMMENTS, + Some(_) if role.name(b"Type") == Some(b"Annot") || role.contains(b"Rect") => return kind::COMMENTS, _ => {} } - if d.name(b"FT") == Some(b"Sig") && signature_ok { + if role.name(b"FT") == Some(b"Sig") && signature_ok { return kind::SIGNATURE; } - if d.contains(b"Fields") { + if role.contains(b"Fields") { return acroform_kind(doc, old, d, before); } - if d.contains(b"FT") || (d.contains(b"T") && d.contains(b"Kids")) || d.contains(b"DR") { + if role.contains(b"FT") || (role.contains(b"T") && role.contains(b"Kids")) || role.contains(b"DR") { return kind::FORM_FILL; } if before.is_none() { diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 57efadbaf3..8f327115d2 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -1620,6 +1620,145 @@ fn a_dictionary_labelled_metadata_is_not_a_metadata_change() { }); } +/// A one-page document whose text sits in an optional-content group (object 8) that the +/// catalog's indirect `/OCProperties` (object 7) lists as shown. +fn fixture_with_layer() -> Vec { + let content = b"/OC /oc1 BDC BT /F1 14 Tf 20 250 Td (Clause 4: the buyer pays) Tj ET EMC"; + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> /OCProperties 7 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Properties << /oc1 8 0 R >> >> /Annots [6 0 R] >>".to_vec(), + [format!("<< /Length {} >>\nstream\n", content.len()).as_bytes(), content, b"\nendstream"].concat(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>".to_vec(), + b"<< /OCGs [8 0 R] /D << /ON [8 0 R] >> >>".to_vec(), + b"<< /Type /OCG /Name (Clause 4) >>".to_vec(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// Hide the clause: rewrite `/OCProperties` (object 7) so its group starts off, adding `extra`. +fn hide_layer(doc: &mut Document, extra: &[(&str, Object)]) { + let mut d = dict_of(vec![("OCGs", Object::Array(vec![oref(8)])), ("D", Object::Dict(dict_of(vec![("OFF", Object::Array(vec![oref(8)]))])))]); + for (k, v) in extra { + d.set(k.as_bytes().to_vec(), v.clone()); + } + doc.set(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, Object::Dict(d)); +} + +#[test] +fn an_existing_object_given_form_keys_after_signing_is_still_judged() { + // An existing object is judged by the role it had when signed, not only when a signed page + // draws it (#1787): the catalog's indirect /OCProperties decides which layers show, and a + // rewrite that hides the clause stays a change whatever form-field keys it adds. + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &[])); + let zeros = Object::Array(vec![Object::Int(0); 4]); + for extra in [ + vec![("FT", Object::name("Tx"))], + vec![("DR", Object::Dict(pdfcraft_cos::Dict::new()))], + vec![("T", Object::String(PdfString::text("x"))), ("Kids", Object::Array(Vec::new()))], + vec![("Subtype", Object::name("Widget")), ("Rect", zeros.clone())], + vec![("Subtype", Object::name("Square")), ("Rect", zeros)], + ] { + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &extra)); + } +} + +#[test] +fn an_unsigned_signature_widget_turned_into_a_comment_is_no_edit_of_it() { + // The empty signature field "Approval" (object 6) rewritten as a white /Square annotation that + // covers the signed text. Judged by what it was, it is a signature widget; but an annotation + // of another kind is no edit that widget permits, at any level. + assert_refused_at_every_level(&fixture(), "other changes", |doc| { + let white = Object::Array(vec![Object::Int(1); 3]); + doc.set( + pdfcraft_cos::ObjRef { num: 6, generation: 0 }, + Object::Dict(dict_of(vec![ + ("Type", Object::name("Annot")), + ("Subtype", Object::name("Square")), + ("Rect", Object::Array(vec![Object::Int(0), Object::Int(0), Object::Int(300), Object::Int(300)])), + ("C", white.clone()), + ("IC", white), + ("P", oref(3)), + ])), + ); + }); +} + +/// `fixture()` with the form an indirect dictionary (object 7) listing the unsigned field. +fn fixture_with_indirect_acroform() -> Vec { + let objs: Vec<&[u8]> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm 7 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>", + b"<< /Length 44 >>\nstream\nBT /F1 14 Tf 20 250 Td (Contract text) Tj ET\nendstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>", + b"<< /Fields [6 0 R] >>", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +#[test] +fn the_forms_signed_fields_are_not_dropped_after_signing() { + // Signed, then timestamped. A later revision drops the form's /Fields, or empties it: the + // fields the timestamp covered, the signature among them, are gone from the form. That is no + // signature change, so the timestamp reports it as not permitted. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let tsa = TestTsa { + id: pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(), + time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 }, + }; + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture_with_indirect_acroform()), &id, &opts()).unwrap(); + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let mut failures = Vec::new(); + for (how, fields) in [("dropped", None), ("emptied", Some(Object::Array(Vec::new())))] { + let edited = edit_after(&stamped, |doc| { + doc.update_dict(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, |af| match &fields { + Some(f) => af.set(b"Fields".to_vec(), f.clone()), + None => { + af.remove(b"Fields"); + } + }) + .unwrap(); + }); + let stamp = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.doc_timestamp).expect("the document timestamp"); + if !matches!(stamp.modification, Modification::Disallowed(_)) { + failures.push(format!("/Fields {how}: {:?}, {:?}", stamp.status, stamp.modification)); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + /// A one-page document like `fixture()` (objects 1–6) plus a stream, object 7, with the given /// dictionary entries and body. `catalog_extra` goes into the catalog; the page draws object 7 /// as a Form XObject when `drawn`. From fbe079c953d3f2644ac59d40894a07fb061f21b0 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:43:14 -0500 Subject: [PATCH 06/14] fix(sign): compare the catalog's /Perms by content under P=1 Rebased #5173 onto 9afa1266. Its later-signature predicate is superseded by #6364's object-based check, so only the /Perms rule lands: under DocMDP P=1 a /Perms that differs from the signed catalog's, by content, is a structure change however it was rewritten. Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 73 +++++++++++++++++++++++++++++++++- crates/sign/tests/it/pdf.rs | 29 ++++++++++++++ parity/acrobat-features.toml | 4 +- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 8cfd823fb0..712c9fcfff 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -217,13 +217,30 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an // approval signature, an unsigned signature widget, or a widget that paints over the signed // page — is a change the certification forbids, however it is labelled. - if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + if p == Some(1) { + only_validation_data(&mut allowed, &mut disallowed, doc, &old); + } + if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } +} + +/// DocMDP P=1 ("no changes allowed") permits only validation data to follow the signed revision: +/// the security store, and document timestamps that draw nothing. A later approval signature, a +/// visible widget or a changed catalog `/Perms` breaks the certification. +fn only_validation_data(allowed: &mut Vec<&'static str>, disallowed: &mut Vec<&'static str>, doc: &Document, old: &Document) { + if allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, old) { allowed.retain(|k| *k != kind::SIGNATURE); if !disallowed.contains(&kind::SIGNATURE) { disallowed.push(kind::SIGNATURE); } } - if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } + // The catalog's /Perms names the certification signature itself. Elsewhere a change to it is + // judged as signing; under P=1 dropping or redirecting it is a change like any other. + // Compared by content: an indirect /Perms rewritten in place keeps its reference (and, shared + // with the trailer's /Info, would otherwise pass as a metadata change). + let perms = |d: &Document| d.root().and_then(|r| d.get(r).as_dict().and_then(|c| c.get(b"Perms").map(|p| (*d.resolve(p)).clone()))); + if perms(doc) != perms(old) && !disallowed.contains(&kind::STRUCTURE) { + disallowed.push(kind::STRUCTURE); + } } /// Whether every signature object `doc` gained or changed since the signed revision `old` is an @@ -892,3 +909,55 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) } kind::OTHER } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + /// A PDF of `objs` (numbered from 1) whose trailer also holds `trailer_extra`. + fn pdf(objs: &[&str], trailer_extra: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer_extra} >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out + } + + /// A P=1 signed revision whose catalog `/Perms` and trailer `/Info` are one indirect + /// dictionary (object 3). A later revision points its `/DocMDP` at another signature + /// dictionary already in the file, in place: the reference stays, no object is added, and + /// as the signed `/Info` the object reads as metadata, but the certification's own record + /// has changed. + #[test] + fn a_perms_dictionary_shared_with_info_rewritten_in_place_is_a_structure_change() { + let signed = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R /Perms 3 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /DocMDP 4 0 R >>", + "<< /Type /Sig >>", + "<< /Type /Sig >>", + ], + "/Info 3 0 R", + ); + let old = Document::open(Arc::new(signed.clone())).unwrap(); + let mut doc = Document::open(Arc::new(signed)).unwrap(); + let other = ObjRef { num: 5, generation: 0 }; + doc.update_dict(ObjRef { num: 3, generation: 0 }, |p| p.set(b"DocMDP".to_vec(), Object::Ref(other))).unwrap(); + let edited = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions { object_streams: false, ..Default::default() }).unwrap(); + let doc = Document::open(Arc::new(edited)).unwrap(); + match classify_changes(&doc, Some(old), Some(1)) { + Modification::Disallowed(kinds) => assert!(kinds.contains(&kind::STRUCTURE), "{kinds:?}"), + other => panic!("the certification's /Perms changed in place under P=1: {other:?}"), + } + } +} diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 8f327115d2..8c9a827e6d 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -3164,3 +3164,32 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val all.iter().map(|s| (&s.status, &s.modification)).collect::>() ); } + +#[test] +fn the_catalogs_perms_do_not_change_under_no_changes() { + // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later + // revision that drops it, or points it elsewhere, breaks the certification. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let certified = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + let elsewhere = dict_of(vec![("DocMDP", oref(5))]); + for (how, perms) in [("dropped", None), ("pointed elsewhere", Some(Object::Dict(elsewhere)))] { + let edited = edit_after(&certified, |doc| { + let root = doc.root().unwrap(); + doc.update_dict(root, |c| match &perms { + Some(p) => c.set(b"Perms".to_vec(), p.clone()), + None => { + c.remove(b"Perms"); + } + }) + .unwrap(); + }); + let s = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.field == "Signature1").unwrap(); + assert!( + s.status == Status::Invalid && matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::STRUCTURE)), + "/Perms {how} under P=1: {:?}, {:?}", + s.status, + s.modification + ); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ec144d3a07..10b040e09f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6535,8 +6535,8 @@ milestone = "M9" status = "shipped" commands = ["sign.panel"] tools = ["sign_list"] -evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill"] -notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision" +evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill", "crates/sign/tests/it/pdf.rs::the_catalogs_perms_do_not_change_under_no_changes"] +notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision. Under P=1 a change to the catalog's /Perms breaks the certification." [[feature]] id = "sign.view-signed-version" From d7ce875dd692109789bcdad1b683ffc5a2d07dc7 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 10:15:07 +0200 Subject: [PATCH 07/14] redact: independent redaction proof machinery (snapshot, sweep, budget) (#5557) * redact: add the independent redaction proof machinery * redact: harden the proof's paint check and count removed codes The manifest counts character codes, not raw bytes: a composite font's codes are several bytes, so the byte length overstated what was cut. Annotation appearances are checked whatever their storage (a direct dictionary has no reference of its own to dedupe on), and a stream shared by several annotations is checked at each annotation's own placement. The graphics-state stack and the clip holes of one walk are bounded; deeper or more is unverifiable, never unbounded memory. Even-odd clips cut holes only where the rule can be told cheaply - all subpaths axis-aligned, disjoint, few - and anything else keeps its paint reported, failing closed. The /W widths of a descendant font cost bounded work, like a CMap. * redact: settle the survivors a page sweep counted before its work ran out A page that the search budget stops on left the counts made on earlier pages unsettled: the survivors were seen but never reported, and the regions that owned them were never marked. The sweep now stops and still settles, so a located survivor is reported where it sits (the page reads unswept either way, and the proof still fails). * redact: prove the redacted output of every supported encryption The authenticated-reopen regression now covers all four crypt algorithms the writer supports, not only AES-128: the caller's credentials prove the saved output, and empty or wrong credentials still fail closed. * redact: answer the review round on the proof A glyph lying under several regions is now accounted for by every one of them, so overlapping marks each carry their own share in a shared proof instead of the first one absorbing the rest and the others reading as if no text was there. Each finding is recorded once per region as it is found, so a page that paints over a region a thousand times neither grows the report nor the memory with repeats. The byte search walks the haystack once (the pattern's prefix function is built once), so a needle of one letter over a haystack full of it can no longer spend comparisons the search budget never charged for. A numeric XML character reference may carry any number of leading zeros; the padded form now decodes like the short one, so a survivor spelled that way in the XMP is still found. The /W widths of a descendant font are paid for from the shared work budget: a form invoked again rebuilds its fonts, and a per-font counter alone let the same array be paid for many times over. * redact: sweep the proof's survivors out of every place a reader could look Plant the removed text where the review listed it and expect the sweep to report it: a compressed object stream, a page thumbnail, a form field value, an XFA packet, and an object nothing refers to. A file whose page content is bigger than the proof may decode for one stream is now shown to come out unverified on its surfaces and its region, never clean. * redact: spare the names a redacted word legitimately sits in, and charge failed decodes A redacted word that is also a structural name (a /Subtype, a /Filter, a /BaseFont) is structure, not a leftover copy: the name surface is now swept against the matches the input itself had, and only a surplus of matches is a survivor. The input's names are counted when the snapshot is taken, for the region's needles and for strings added later alike. A decode that fails after doing part of its work paid nothing: the proof's cumulative decode allowance now charges every attempt, failures included, so the same bomb cannot be decoded (and fail) again and again for free. * redact: count the input's names once each, and say so when the count runs out The name allowance counted one entry per occurrence, so a document whose pages repeat the same structural name paid for the same scan again and again; names are now kept once, with their count. A name search that runs out of work read as no matches, which would have swept names with none allowed: that is reported on the text surface now, and the proof fails rather than pass on an uncounted allowance. * Bound redaction proof searches across many needles Assisted-by: OpenAI Codex * Redaction proof: drop find_all, which nothing calls Co-Authored-By: Claude Opus 5.5 * Cargo.lock: put aho-corasick in its sorted place Co-Authored-By: Claude Opus 5.5 * Redaction proof: review notes A word split across a TJ array is looked for whole. Overlapping marks count a removed copy once. A spent decode budget is a problem, not a raw-only stream. Short needles stay out of the attachment text search, and an indirect /Decode counts. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + crates/redact/Cargo.toml | 2 + crates/redact/src/image.rs | 2 +- crates/redact/src/interp.rs | 100 +- crates/redact/src/lib.rs | 11 +- crates/redact/src/tests_verify.rs | 1380 ++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 711 +++++++++++ crates/redact/src/verify/geometry.rs | 500 ++++++++ crates/redact/src/verify/limits.rs | 42 + crates/redact/src/verify/mod.rs | 389 ++++++ crates/redact/src/verify/search_tests.rs | 34 + crates/redact/src/verify/snapshot.rs | 532 +++++++++ crates/redact/src/verify/sweep.rs | 759 ++++++++++++ crates/redact/src/verify/util.rs | 301 +++++ 14 files changed, 4759 insertions(+), 15 deletions(-) create mode 100644 crates/redact/src/tests_verify.rs create mode 100644 crates/redact/src/verify/extract.rs create mode 100644 crates/redact/src/verify/geometry.rs create mode 100644 crates/redact/src/verify/limits.rs create mode 100644 crates/redact/src/verify/mod.rs create mode 100644 crates/redact/src/verify/search_tests.rs create mode 100644 crates/redact/src/verify/snapshot.rs create mode 100644 crates/redact/src/verify/sweep.rs create mode 100644 crates/redact/src/verify/util.rs diff --git a/Cargo.lock b/Cargo.lock index 4816a44ec0..e1f271ed12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4610,6 +4619,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4620,6 +4630,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index fe21d13b32..c02335e3c4 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +aho-corasick = "1" pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } @@ -20,6 +21,7 @@ pdfcraft-xfa = { workspace = true } # CCITT Group 4 and JBIG2 encoding for cleared one-bit images. pdfcraft-filters = { workspace = true } thiserror = { workspace = true } +sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 4b3c44b0fd..a99e179895 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -12,7 +12,7 @@ use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed), or to be stored /// again without shared `/JBIG2Globals` (see [`without_globals`]). -const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; +pub(crate) const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; fn components(doc: &Document, s: &Stream) -> Option { let cs = doc.resolve(s.dict.get(b"ColorSpace")?); diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 460085a320..7a2cd26eca 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -53,6 +53,23 @@ pub(crate) enum Mode { Verify, } +/// A run of glyphs cut out under a region, kept for the redaction proof +/// ([`Snapshot::capture`](crate::verify::Snapshot::capture)). +pub(crate) struct Removed { + /// Index of the region (in `Scope::rects`) the run lay under. + pub(crate) rect: usize, + /// The character codes as they stood in the string. + pub(crate) raw: Vec, + /// What the font says they showed (codes without a known meaning are left out). + pub(crate) text: String, + /// How many character codes the run holds (a composite font's codes are several bytes). + pub(crate) codes: usize, + /// Which text operator showed it, counted from the start of the page. + pub(crate) op: usize, + /// Where in that operator it starts: the operand's place in a `TJ` array, and the byte in it. + pub(crate) at: (usize, usize), +} + #[derive(Clone)] struct Gs { ctm: Matrix, @@ -124,6 +141,12 @@ pub(crate) struct Scope<'a> { xobject_oc_cache: HashMap, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Collects the glyph runs removed under a region in apply mode (`None` = not collected). + /// The redaction proof's snapshot is what sets it: it must know what applying is about to + /// remove, by the interpreter's own judgement, without trusting anything else to it. + pub removed: Option>, + /// How many text operators have been shown, for [`Removed::op`]. + shown: usize, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -149,6 +172,8 @@ impl<'a> Scope<'a> { ocmd_cache: HashMap::new(), xobject_oc_cache: HashMap::new(), layer_blocks: 0, + removed: None, + shown: 0, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -169,21 +194,33 @@ impl<'a> Scope<'a> { self.rects.iter().any(|r| contains(*r, b, 0.01)) } - /// Does a glyph box count as under a region? Any real overlap does (more than a fifth of + /// Which regions a glyph box counts as under? Any real overlap does (more than a fifth of /// the glyph or a point, whichever is less, in both directions), so neighbours that only /// touch an edge survive while partly covered glyphs go. Degenerate boxes (zero-size text) - /// count when their origin is inside. - fn glyph_hit(&self, b: [f64; 4], origin: (f64, f64)) -> bool { + /// count when their origin is inside. Empty = under no region (a glyph kept). A glyph may + /// lie under several regions at once (overlapping marks); each is reported. + fn glyph_rects(&self, b: [f64; 4], origin: (f64, f64)) -> Vec { let (gw, gh) = (b[2] - b[0], b[3] - b[1]); if gw < 0.01 || gh < 0.01 { - return self.rects.iter().any(|r| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]); + return self + .rects + .iter() + .enumerate() + .filter(|(_, r)| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]) + .map(|(i, _)| i) + .collect(); } let (mx, my) = ((gw * 0.2).min(1.0), (gh * 0.2).min(1.0)); - self.rects.iter().any(|r| { - let ox = b[2].min(r[2]) - b[0].max(r[0]); - let oy = b[3].min(r[3]) - b[1].max(r[1]); - ox > mx && oy > my - }) + self.rects + .iter() + .enumerate() + .filter(|(_, r)| { + let ox = b[2].min(r[2]) - b[0].max(r[0]); + let oy = b[3].min(r[3]) - b[1].max(r[1]); + ox > mx && oy > my + }) + .map(|(i, _)| i) + .collect() } } @@ -223,6 +260,8 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The regions it lies under (empty for hidden text). + rects: Vec, } /// Process a scope's content streams (state flows from one stream to the next). @@ -644,11 +683,14 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +/// In apply mode the runs removed under each region are also handed to `scope.removed`. +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + scope.shown += 1; + let op = scope.shown; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -661,7 +703,7 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let bytes = &s.bytes; @@ -674,18 +716,50 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let degenerate = b[2] - b[0] < 0.01 && b[3] - b[1] < 0.01; matches!(gs.render_mode, 3 | 7) || !(overlaps(page, b, 0.0) || degenerate) } - None => scope.glyph_hit(b, origin), + None => { + rects = scope.glyph_rects(b, origin); + !rects.is_empty() + } }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, rects }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } + if scope.mode == Mode::Apply + && let Some(sink) = scope.removed.as_mut() + { + // Consecutive removed glyphs under one region are one run, recorded for + // every region the glyphs lie under: overlapping marks each account for + // their own share of what was cut. + let mut under: Vec = glyphs.iter().flat_map(|g| g.rects.iter().copied()).collect(); + under.sort_unstable(); + under.dedup(); + for r in under { + let mut k = 0; + while let Some(g) = glyphs.get(k) { + if !g.rects.contains(&r) { + k += 1; + continue; + } + let mut end = k; + while glyphs.get(end + 1).is_some_and(|n| n.rects.contains(&r)) { + end += 1; + } + let span = g.bytes.start..glyphs.get(end).map_or(g.bytes.end, |l| l.bytes.end); + let at = (place, span.start); + let raw = bytes.get(span).unwrap_or_default().to_vec(); + sink.push(Removed { rect: r, text: f.decode(&raw), raw, codes: end - k + 1, op, at }); + k = end + 1; + } + } + } let mut run: Vec = Vec::new(); for g in &glyphs { if g.hit { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index b6a8885c03..6bb8e5cc06 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,10 +27,14 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_verify; pub(crate) mod type3; pub(crate) mod type3_tables; +pub mod verify; use interp::{Mode, Scope, process}; +pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -59,6 +63,11 @@ pub enum RedactError { "redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)" )] OverlayLimit, + /// A stream the redaction created to draw the marks' boxes and labels can't be read back, so + /// the proof can't tell it apart from content that was there before and nothing can be + /// exempted from the sweep. + #[error("a stream the redaction created can't be read back, so the result can't be proven")] + OverlayUnreadable, /// The structure tree couldn't be cleaned completely (#3302): an element written in place that /// would need changing, an element whose inherited page can't be found, or a tree too large to /// walk. Tags that kept the removed text or annotation would undo the redaction. @@ -164,7 +173,7 @@ pub struct Report { pub overlays: Vec, } -fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { +pub(crate) fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { let v: Vec = doc.resolve(o?).as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == 4 && v.iter().all(|x| x.is_finite())).then(|| [v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]) } diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs new file mode 100644 index 0000000000..ee443f3209 --- /dev/null +++ b/crates/redact/src/tests_verify.rs @@ -0,0 +1,1380 @@ +//! The independent redaction proof: what it catches, what it refuses to call clean. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Algorithm, Dict, Document, NewEncryption, Object, SaveOptions, Stream, write_full, write_incremental}; +use sha2::Digest; + +use super::*; +use crate::verify::tests_support::{self, count_text_operators, survivor_encodings}; +use crate::verify::{Status, Surface, SurfaceReport, Verdict}; + +const SECRET: &str = "SECRETVALUE"; + +pub(super) fn stream(dict: &str, data: &[u8]) -> Vec { + let mut v = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + v.extend_from_slice(data); + v.extend_from_slice(b"\nendstream"); + v +} + +pub(super) fn pdf_bytes(objs: Vec>, trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +pub(super) fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// Every glyph is 500 units wide (5 pt at 10 pt). +pub(super) const FONT: &str = + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor << /Ascent 800 /Descent -200 >> >>"; + +/// A composite font without a /ToUnicode map: its text can't be decoded. +const OPAQUE_FONT: &str = "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 >>] >>"; + +/// Options of a one-page file: catalog, page and trailer entries, page resources, and more +/// objects (numbered from 7). +#[derive(Default)] +struct Spec<'a> { + catalog: &'a str, + page: &'a str, + trailer: &'a str, + resources: &'a str, + extra: Vec>, +} + +/// "Public" at the top, the secret lower down at x 10..65, y 100..110. +const CONTENT: &[u8] = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + +pub(super) const REGION: [f64; 4] = [8.0, 95.0, 70.0, 112.0]; + +fn build_with(content: &[u8], spec: Spec<'_>) -> Vec { + let mut objs: Vec> = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {} >>", spec.catalog).into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R {} /Resources << /Font << /F1 5 0 R >> {} >> >>", + spec.page, spec.resources + ) + .into_bytes(), + stream("", content), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + objs.extend(spec.extra); + pdf_bytes(objs, spec.trailer) +} + +pub(super) fn mark(doc: &mut Document, rects: &[[f64; 4]]) { + let shape = Shape::Redact { quads: rects.iter().map(|r| rect_quad(*r)).collect(), overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Tester".into() }, &Meta::default()).unwrap(); +} + +/// The standard page with `spec`, the secret's region marked. +fn marked(spec: Spec<'_>) -> Document { + marked_content(CONTENT, spec) +} + +fn marked_content(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[REGION]); + doc +} + +/// Apply, and tell the snapshot which streams the apply created (its overlay boxes and labels). +fn applied(doc: &mut Document, snap: &mut Snapshot) -> Result { + let result = apply(doc, None); + if let Ok(report) = &result { + snap.set_overlay_streams(doc, &report.overlays)?; + } + result +} + +/// Capture, apply, prove. +pub(super) fn run(doc: &mut Document) -> (Result, Proof) { + let mut snap = Snapshot::capture(doc, None).unwrap(); + let result = applied(doc, &mut snap); + let proof = snap.prove(doc, None, &ProofOptions::default()); + (result, proof) +} + +/// Prove a document the redaction was never applied to (as a full save would write it). +pub(super) fn prove_unapplied(doc: &mut Document) -> Proof { + let snap = Snapshot::capture(doc, None).unwrap(); + doc.require_full_save(); + snap.prove(doc, None, &ProofOptions::default()) +} + +pub(super) fn surface(p: &Proof, s: Surface) -> &SurfaceReport { + p.surfaces.iter().find(|r| r.surface == s).unwrap() +} + +fn residue(r: &Result) -> bool { + matches!(r, Err(RedactError::Residue(n)) if *n > 0) +} + +#[test] +fn the_proof_reads_a_pages_streams_as_the_one_stream_they_are() { + // The TJ array of the secret ends the first piece and its operator starts the second + // (#153): the proof reads the pieces as the one stream they are, so the removed text is + // neither reported as a survivor nor does the page count as unreadable. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td [(SECRETVALUE)]"), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"TJ ET BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let (r, proof) = run(&mut doc); + let r = r.unwrap(); + assert_eq!(r.glyphs, 11, "every glyph of the split string is removed: {r:?}"); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.survivors(), 0); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); +} + +#[test] +fn a_clean_redaction_passes_and_the_neighbour_is_untouched() { + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries.len(), 1); + let e = &proof.entries[0]; + assert_eq!((e.status, e.page, e.region), (Status::Verified, 0, REGION)); + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len())); + let content = page_content(&doc); + assert!(content.contains("(Public) Tj") && !content.contains(SECRET), "{content}"); + for s in &proof.surfaces { + assert!(matches!(s.verdict, Verdict::Clean | Verdict::Absent), "{s:?}"); + } + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Clean); +} + +#[test] +fn the_manifest_has_digests_and_counts_but_no_plaintext() { + let mut doc = marked(Spec::default()); + // The digest of the original content: a shared proof must not carry it. + let original: String = sha2::Sha256::digest(page_content(&doc).as_bytes()).iter().map(|b| format!("{b:02x}")).collect(); + let (r, proof) = run(&mut doc); + r.unwrap(); + let e = &proof.entries[0]; + let after = e.page_sha256_after.as_deref().unwrap(); + assert!(after.len() == 64 && after.bytes().all(|c| c.is_ascii_hexdigit())); + assert_ne!(after, original); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(&original), "{shown}"); + } + // Two glyph-carrying operators before; the removed one left a number-only TJ, which carries + // no glyphs and is not counted. + assert_eq!((e.text_ops_before, e.text_ops_after), (Some(2), Some(1))); + assert!(e.removed_text.is_none()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } + + // Plaintext only on request. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + assert!(!format!("{snap:?}").contains(SECRET)); + applied(&mut doc, &mut snap).unwrap(); + let with = snap.prove(&doc, None, &ProofOptions { include_plaintext: true }); + assert_eq!(with.entries[0].removed_text.as_deref(), Some(&[SECRET.to_string()][..])); +} + +#[test] +fn survivors_do_not_echo_the_text_either() { + let mut doc = marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } +} + +#[test] +fn numeric_only_tj_is_not_glyph_carrying() { + assert_eq!(count_text_operators(b"BT (Tj) Tj [ -250 ] TJ [(x) -3 (y)] TJ () Tj [] TJ ET"), 2); + assert_eq!(count_text_operators(b"BT (a) ' 1 2 (b) \" [ 10 -10 ] TJ ET"), 2); +} + +#[test] +fn a_survivor_in_the_info_dictionary() { + let mut doc = + marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (Report SECRETVALUE) /Author (Someone) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_xmp() { + let xmp = stream("/Type /Metadata /Subtype /XML", b"Quarterly & SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_long_numeric_reference_in_xmp_is_a_survivor() { + // A character reference may carry any number of leading zeros: the padded form reads the + // same, and is swept the same. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_an_annotation_string() { + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +// ── Every place a reader could recover the removed text from ──────────────────────────────── +// Each test plants the removed text in one location and expects the sweep to report it there. + +#[test] +fn a_survivor_in_a_compressed_object_stream_is_found() { + // A full save packs non-stream objects into compressed object streams: a copy of the + // removed text left in one (an annotation away from the region) is neither in the stored + // bytes nor on any page, only inside the /ObjStm. The sweep reads the saved file like any + // reader, objects in object streams included, and reports the copy. + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.windows(b"/ObjStm".len()).any(|w| w == b"/ObjStm"), "the output packs objects into object streams"); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the plaintext is compressed away"); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Clean, "the copy sits in the object stream, not the stored bytes"); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_page_thumbnail_is_found() { + // The page's /Thumb holds a picture of the page as it was, secret included (uncompressed + // pixels, so the stored bytes are the pixels). A thumbnail is not page content, so only + // the object sweep can find it. + let thumb = stream("/Type /XObject /Subtype /Image /Width 7 /Height 3 /ColorSpace /DeviceGray /BitsPerComponent 8", b"junk SECRETVALUE junk"); + let mut doc = marked(Spec { page: "/Thumb 7 0 R", extra: vec![thumb], ..Spec::default() }); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_form_field_value_is_found() { + // The redacted text also sits in the form: a text field whose /V holds it. The field + // dictionary is an object like any other, and its strings are swept wherever the field + // hangs (AcroForm and the page's annotations both point at it). + let field = b"<< /Type /Annot /Subtype /Widget /Rect [200 20 280 40] /FT /Tx /T (form) /V (note: SECRETVALUE) >>".to_vec(); + let spec = Spec { catalog: "/AcroForm << /Fields [7 0 R] >>", page: "/Annots [7 0 R]", extra: vec![field], ..Spec::default() }; + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + // The field's copy is the only place left (the save renumbers objects, so the place is + // named, not numbered from the input file; both views of the string count as survivors). + assert!(s.survivors >= 1, "{}", proof.to_text()); + assert_eq!(s.locations.len(), 1, "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_xfa_packet_is_found() { + // An XFA form mirrors its field values in the /XFA packets, XML of their own. Redaction + // strips an XFA form outright (crates/xfa does that, not this machinery); the proof must + // not take the stripper on trust: whatever it left (a shape it refused, a leftover + // packet) is swept like any stream, and what the packet holds is reported. + let xfa = stream("", b"
SECRETVALUE
"); + let spec = Spec { catalog: "/AcroForm << /XFA 7 0 R /Fields [] >>", extra: vec![xfa], ..Spec::default() }; + let doc = marked(spec); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_unused_object_is_found() { + // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that + // only walks the document tree misses it; the sweep walks every object the + // cross-reference table knows, orphans included, and names the object it sits in. + let orphan = format!("<< /Type /Annot /Subtype /Text /Contents (note: {SECRET}) >>").into_bytes(); + let doc = marked(Spec { extra: vec![orphan], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +/// The page draws "(Image)" under the marked region, and keeps an image XObject whose +/// `/Subtype /Image` is the same word. +fn image_word_doc() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let content = b"BT /F1 10 Tf 10 100 Td (Image) Tj ET"; + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[0.0, 90.0, 40.0, 115.0]]); + doc +} + +#[test] +fn a_word_that_remains_in_a_structural_name_does_not_fail_the_proof() { + // "Image" is redacted from the page, and is also the /Subtype of the image the page keeps. + // The name is structure redaction never edits, not a leftover copy: the proof must pass, + // and the name surface must read clean, not unswept. + let mut doc = image_word_doc(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Clean); + + // But a name the input did not have is a copy like any other: a surplus of name matches + // is a survivor, wherever it sits. + let snap = Snapshot::capture(&image_word_doc(), None).unwrap(); + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", catalog: "/Extra << /Stolen /Image >>", extra: vec![image], ..Spec::default() }; + let mut shown = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec)); + mark(&mut shown, &[[0.0, 90.0, 40.0, 115.0]]); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +fn attachment_spec(payload: &[u8]) -> Spec<'static> { + Spec { + catalog: "/Names << /EmbeddedFiles << /Names [(a.bin) 7 0 R] >> >>", + extra: vec![b"<< /Type /Filespec /F (a.bin) /EF << /F 8 0 R >> >>".to_vec(), stream("/Type /EmbeddedFile", payload)], + ..Spec::default() + } +} + +#[test] +fn a_survivor_in_an_attachment() { + let mut doc = marked(attachment_spec(b"the SECRETVALUE is attached")); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); + + // UTF-16LE with a byte order mark. + let mut le = vec![0xFF, 0xFE]; + le.extend(SECRET.encode_utf16().flat_map(u16::to_le_bytes)); + let mut doc = marked(attachment_spec(&le)); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_a_nested_attachment() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", Spec::default()); + let middle = build_with(b"", attachment_spec(&inner)); + let mut doc = marked(attachment_spec(&middle)); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::EmbeddedFiles); + assert_eq!(s.verdict, Verdict::Survivor); + assert!(s.locations.iter().any(|l| l.matches(" > ").count() >= 1), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_clean_nested_attachment_is_swept_and_clean() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (harmless) Tj ET", Spec::default()); + let mut doc = marked(attachment_spec(&inner)); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Clean); +} + +#[test] +fn unsweepable_attachments_fail_closed() { + for payload in [&b"PK\x03\x04 zipped"[..], b"%PDF-1.7 not really a pdf", b"\x1f\x8b gzip"] { + let mut doc = marked(attachment_spec(payload)); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Unswept, "{}", String::from_utf8_lossy(payload)); + assert!(!proof.passed(), "a proof that could not sweep must not pass"); + } +} + +#[test] +fn an_undecodable_stream_fails_closed() { + let junk = stream("/Filter /NoSuchDecode", b"\x00\x01\x02 junk"); + let mut doc = marked(Spec { catalog: "/Extra 7 0 R", extra: vec![junk], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn an_incremental_remnant_fails_the_proof() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + // A "redaction" saved as an incremental update: the page shows something else now, but the + // first revision still holds the original content. + let mut other = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let new = other.add(Object::Stream(Stream::flate(Dict::new(), b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"))); + let page = pdfcraft_model::pages(&other).swap_remove(0); + other.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + let saved = write_incremental(&other, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Survivor); + // The page text itself is clean: only the file structure gives it away. + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); + + // The real redaction, saved as apply() requires, is a single clean revision. + applied(&mut doc, &mut snap).unwrap(); + let full = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&full, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn the_secret_in_each_encoding_is_found() { + // Spelled as PDF syntax in a catalog entry. + let utf16: String = SECRET.encode_utf16().flat_map(u16::to_be_bytes).map(|b| format!("{b:02X}")).collect(); + let utf16le: String = SECRET.encode_utf16().flat_map(u16::to_le_bytes).map(|b| format!("{b:02X}")).collect(); + let hex8: String = SECRET.bytes().map(|b| format!("{b:02x}")).collect(); + let forms = [ + format!("({SECRET})"), + format!(""), + format!(""), + format!("", hex8.to_uppercase()), + format!("<{hex8}>"), + format!("<{utf16}>"), + "(\\123ECRETVALUE)".to_string(), + ]; + for f in forms { + let mut doc = marked(Spec { catalog: &format!("/Extra {f}"), ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed(), "{f}: {}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{f}: {}", proof.to_text()); + } + + // Spelled as bytes in the file, including the backslash-escaped literal form. + let paren = "SECR(ET)X\\Y"; + let enc = survivor_encodings(paren); + assert!(enc.len() >= 10); + for spelling in enc { + let mut hay = b"junk ".to_vec(); + hay.extend(&spelling); + hay.extend(b" junk"); + assert!(tests_support::found(paren, &hay), "{}", String::from_utf8_lossy(&spelling)); + } + assert!(!tests_support::found(paren, b"nothing to see")); +} + +#[test] +fn a_rotated_page_is_proven_too() { + let mut doc = marked(Spec { page: "/Rotate 90", ..Spec::default() }); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::Verified); +} + +#[test] +fn a_region_without_text_is_recorded_as_such() { + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + assert!(proof.passed()); +} + +fn opaque_spec() -> Spec<'static> { + Spec { resources: "/Font << /F1 5 0 R /F2 7 0 R >>", extra: vec![OPAQUE_FONT.as_bytes().to_vec()], ..Spec::default() } +} + +#[test] +fn text_that_cannot_be_decoded_makes_a_region_unverifiable() { + let mut doc = open(build_with(b"BT /F2 10 Tf 10 200 Td <00410042> Tj ET", opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(proof.entries[0].status, Status::Unverifiable); + assert!(!proof.passed(), "no PASS while a region is unverifiable"); +} + +#[test] +fn removed_text_of_an_undecodable_font_is_swept_by_its_codes() { + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].status, Status::Verified, "{}", proof.to_text()); + + // Left in place, the codes are still on the page and the proof says so. + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn codes_removed_counts_codes_not_bytes() { + // A composite font's codes are two bytes each: the manifest counts the codes, not the bytes. + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].codes_removed, 4, "{}", proof.to_text()); + + // A simple font's codes are one byte each: the two counts agree. + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!((proof.entries[0].runs_removed, proof.entries[0].codes_removed), (1, SECRET.len())); +} + +#[test] +fn overlapping_marks_each_account_for_their_own_share() { + // Two regions over the same text: each gets the runs, codes and status it is owed — the + // second one is not waved through as "no text in the region". + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries.len(), 2); + for e in &proof.entries { + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len()), "{}", proof.to_text()); + assert_eq!(e.status, Status::Verified, "{}", proof.to_text()); + } + + // Left in place, both regions name the survivor as theirs. + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert!(proof.entries.iter().all(|e| matches!(e.status, Status::Failed | Status::Unverifiable)), "{}", proof.to_text()); +} + +#[test] +fn nothing_applied_means_survivors_where_the_text_was() { + let mut doc = marked(Spec::default()); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn a_document_that_would_save_incrementally_is_not_proven() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + // A reopened copy no longer carries the full-save requirement. + let reopened = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let proof = snap.prove(&reopened, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); +} + +fn image_page() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0xFFu8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + marked_region(b"q 100 0 0 100 100 100 cm /Im1 Do Q 0 0 1 rg 120 120 60 60 re f", spec) +} + +fn marked_region(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[100.0, 100.0, 150.0, 150.0]]); + doc +} + +#[test] +fn images_and_paths_under_a_region_are_checked_independently() { + let mut doc = image_page(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + + // The same page unredacted: the image and the path are both still there. + let proof = prove_unapplied(&mut image_page()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed); + assert!(e.detail.contains("image") && e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn a_shading_under_a_region_must_be_clipped() { + let sh = + b"<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> >>" + .to_vec(); + let spec = || Spec { resources: "/Shading << /Sh1 7 0 R >>", extra: vec![sh.clone()], ..Spec::default() }; + let mut doc = marked_region(b"/Sh1 sh", spec()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + + let proof = prove_unapplied(&mut marked_region(b"/Sh1 sh", spec())); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn an_even_odd_clip_around_the_region_hides_what_it_cuts_out() { + // How kept content is told to hide: an even-odd clip of the page and the region. A path + // painted inside the region under that clip is clipped away; the same path under a clip + // the rule cannot read (a triangle, overlapping inners), or with no clip, is reported. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |tail: &str| -> Proof { + let content = format!("BT /F1 10 Tf 10 200 Td (Public) Tj ET {tail}"); + let mut shown = marked_content(content.as_bytes(), Spec::default()); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let proof = prove("q 0 0 300 300 re 8 95 62 17 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert!(proof.passed(), "{}", proof.to_text()); + + // A triangle is not a rectangle: no hole is cut, and the paint is reported. + let proof = prove("q 0 0 300 300 re 8 95 m 70 95 l 8 112 l h W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + // Overlapping inners change the parity: no hole is cut either. + let proof = prove("q 0 0 300 300 re 8 95 62 17 re 5 90 70 25 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + + // Unclipped, the same path is a plain finding. + let proof = prove("0 0 0 rg 10 100 20 10 re f"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); +} + +#[test] +fn a_graphics_state_stack_too_deep_to_follow_is_unverifiable() { + // Saved states carry their clip holes with them: a `q` stack deep enough to grow without + // bound is refused, not followed. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut deep = Vec::new(); + for _ in 0..crate::verify::limits::MAX_GS_STACK + 8 { + deep.extend_from_slice(b"q "); + } + deep.extend_from_slice(b"0 0 0 rg 10 100 20 10 re f"); + let mut shown = marked_content(&deep, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("too deeply"), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_page_full_of_paint_reports_its_finding_once() { + // Fifty paths over a region are one finding: the report names the problem once, not once + // per operator, and the report does not grow with the operator count. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut content = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET ".to_vec(); + for k in 0..50 { + content.extend_from_slice(&format!("0 0 1 rg {} 100 4 4 re f ", 10 + k).into_bytes()); + } + let mut shown = marked_content(&content, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert_eq!(proof.entries[0].detail.matches("a path still paints").count(), 1, "{}", proof.to_text()); +} + +#[test] +fn text_kept_elsewhere_does_not_fail_the_proof() { + // The same word is redacted in one place and legitimately remains in another. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let mut doc = marked_content(content, Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("remain in retained content")); + + // A copy that should have gone is still caught, by count. + let proof = prove_unapplied(&mut marked_content(content, Spec::default())); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn the_proof_text_keeps_its_honesty_wording() { + // The closing caveat of a proof is a promise to whoever reads a shared proof: pin it so the + // wording cannot silently drop out of the text for either verdict. + let honesty = "Evidence, not a guarantee"; + let passed = Proof { entries: vec![], surfaces: vec![], failures: vec![] }; + assert!(passed.passed()); + assert!(passed.to_text().contains(honesty), "{}", passed.to_text()); + + let failed = Proof { entries: vec![], surfaces: vec![], failures: vec!["the marked text is still readable".into()] }; + assert!(!failed.passed()); + assert!(failed.to_text().contains(honesty), "{}", failed.to_text()); +} + +// ── Independence, scope and bounds of the sweep ───────────────────────────────────────────── + +/// Hang `obj` off the catalog (as `/Extra`) so the saved file keeps it. +pub(super) fn attach(doc: &mut Document, obj: Object) { + let r = doc.add(obj); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(r))).unwrap(); +} + +/// Add a content stream (with `dict`) after the page's own. +fn append_content(doc: &mut Document, dict: Dict, data: &[u8]) { + let new = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + let page = pdfcraft_model::pages(doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); +} + +#[test] +fn a_glyph_the_interpreter_kept_is_found_by_placement_alone() { + // The snapshot is taken where the region holds no text, so it has nothing to search for: + // only the independent placement of glyphs can notice the text in the output. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut shown = marked(Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert!(!proof.passed()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("glyph(s) still show"), "{}", e.detail); + assert_eq!(proof.survivors(), 0, "no needle was involved"); + // The neighbour at y 200 is outside the region and is not reported. + assert!(!e.detail.contains("12 glyph"), "{}", e.detail); +} + +#[test] +fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { + // The secret drawn through a rotated/translated text matrix inside a scaled form: the + // placement has to follow `Tm` and the form's `/Matrix`, not the raw operands. + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix [1 0 0 1 0 0] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 1 0 0 1 10 100 Tm (SECRETVALUE) Tj ET", + ); + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut doc = marked_content(b"/Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("glyph(s) still show")); +} + +#[test] +fn an_ambiguous_needle_is_still_swept_outside_page_content() { + // The word also stays on the page legitimately (so page text, the raw file and decoded + // streams can only count it), but a copy in the Info dictionary is not page content. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let spec = Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }; + let mut doc = marked_content(content, spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); + + // Likewise an XMP packet and an annotation string. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked_content(content, Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::Xmp).verdict, Verdict::Survivor); + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (SECRETVALUE) >>".to_vec(); + let mut doc = marked_content(content, Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::ObjectStrings).verdict, Verdict::Survivor); +} + +#[test] +fn a_pcmark_key_in_the_input_does_not_exempt_a_stream() { + // A content stream that claims to be a redaction box paints inside the region, and was not + // created by this run: it counts as paint like any other. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut d = Dict::new(); + d.set(b"PCMark".to_vec(), Object::Name(b"Redaction".to_vec())); + append_content(&mut doc, d, b"0 0 0 rg 8 95 62 17 re f"); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn only_the_streams_this_run_created_are_exempt() { + // The same stream, declared as created by this run, is the box that was drawn on purpose. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let new = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"0 0 0 rg 8 95 62 17 re f".to_vec()))); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); + doc.require_full_save(); + snap.set_overlay_streams(&doc, &[new]).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn overlay_text_does_not_collide_with_what_was_removed() { + // "Account" is redacted and the label says "Account [redacted]": the label is not a survivor. + let content = b"BT /F1 10 Tf 10 100 Td (Account 1234) Tj ET"; + let mut doc = open(build_with(content, Spec::default())); + let shape = Shape::Redact { quads: vec![rect_quad([8.0, 95.0, 100.0, 112.0])], overlay: "Account [redacted]".into(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "T".into() }, &Meta::default()).unwrap(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); +} + +#[test] +fn removed_annotation_and_field_text_is_searched_for_too() { + let spec = || Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Subject (call back ANNOTWORDS) >>".to_vec()], ..Spec::default() }; + // Not asked for: the proof does not know about it. + let mut doc = marked(spec()); + let (_, proof) = run(&mut doc); + assert!(proof.passed(), "{}", proof.to_text()); + // Asked for: it is found where it was left. + let mut doc = marked(spec()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&[" call back ANNOTWORDS ".to_string(), "x".to_string(), String::new()]); + let result = applied(&mut doc, &mut snap); + assert!(result.is_ok() || residue(&result)); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + // And an absent string is clean. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&["never present anywhere".to_string()]); + applied(&mut doc, &mut snap).unwrap(); + assert!(snap.prove(&doc, None, &ProofOptions::default()).passed()); +} + +#[test] +fn a_damaged_stream_tail_is_unswept_not_truncated() { + let text: Vec = (0..4000u32).map(|i| b'a' + (i * 7 % 23) as u8).collect(); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + // Nothing was removed (the snapshot's region is blank), so the only issue is the stream. + let snap = Snapshot::capture(&marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()), None).unwrap(); + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + attach(&mut doc, Object::Stream(s)); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_damaged_page_content_tail_makes_the_page_unreadable() { + let text: Vec = b"BT /F1 10 Tf 10 100 Td (filler) Tj ET\n".repeat(200); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + let mut doc = marked(Spec::default()); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(s)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{}", proof.to_text()); +} + +#[test] +fn a_malformed_form_matrix_or_bbox_is_unswept() { + for dict in ["/Matrix [1 0 0]", "/Matrix [1 0 0 1 0 (x)]", "/BBox [0 0 300]"] { + let form = stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {dict}"), b"BT /F1 10 Tf 10 100 Td (x) Tj ET"); + let spec = Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }; + let mut doc = marked_content(b"/Fm1 Do", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{dict}: {}", proof.to_text()); + assert!(!proof.passed()); + assert!(proof.entries[0].detail.contains("malformed"), "{dict}: {}", proof.to_text()); + } +} + +#[test] +fn images_hidden_in_patterns_soft_masks_and_type3_glyphs_are_not_skipped() { + let image = "BI /W 1 /H 1 /CS /G /BPC 8 ID \u{ff} EI"; + let tile = + |data: &str| stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", data.as_bytes()); + let group = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency >> /Resources << >>", image.as_bytes()); + let proc = stream("", format!("1000 0 d0 {image}").as_bytes()); + let cases: Vec<(&str, Spec)> = vec![ + ("pattern", Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile(image)], ..Spec::default() }), + ( + "soft mask", + Spec { resources: "/ExtGState << /G1 << /SMask << /Type /Mask /S /Luminosity /G 7 0 R >> >> >>", extra: vec![group], ..Spec::default() }, + ), + ( + "type3", + Spec { + resources: "/Font << /F3 << /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 7 0 R >> /Encoding << /Type /Encoding /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] >> >>", + extra: vec![proc], + ..Spec::default() + }, + ), + ]; + for (what, spec) in cases { + // The region holds no text and no visible image: only the hidden image is in question. + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Unverifiable, "{what}: {}", proof.to_text()); + assert!(e.detail.contains("pattern, soft mask or Type3"), "{what}: {}", e.detail); + assert!(!proof.passed()); + } + // The same resources without an image are fine. + let spec = Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile("0 0 5 5 re f")], ..Spec::default() }; + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion, "{}", proof.to_text()); +} + +#[test] +fn the_sweep_is_bounded() { + // A CMap whose ranges rewrite the same 64Ki codes again and again costs work, whatever the map size. + let mut cmap = b"1 begincodespacerange <0000> endcodespacerange\n".to_vec(); + for _ in 0..200 { + cmap.extend_from_slice(b"100 beginbfrange\n"); + for _ in 0..100 { + cmap.extend_from_slice(b"<0000> <0041>\n"); + } + cmap.extend_from_slice(b"endbfrange\n"); + } + let start = std::time::Instant::now(); + assert!(!tests_support::cmap_is_complete(&cmap)); + assert!(start.elapsed().as_secs() < 20, "bounded work"); + // A small CMap is read in full. + assert!(tests_support::cmap_is_complete(b"1 beginbfchar <01> <0041> endbfchar")); + // Nested objects, cumulative decoding and searching all stop and say so. + assert!(tests_support::walks_in_full(60)); + assert!(!tests_support::walks_in_full(200)); + assert_eq!(tests_support::decode_budget_runs_out(), (true, true)); + assert_eq!(tests_support::search_budget_runs_out(), (true, true)); + // A decode that fails past the one-stream cap pays for the attempt: the same bomb cannot + // be decoded (and fail) again and again for free. + assert_eq!(tests_support::failed_decode_charges_the_budget(), (true, true, true)); +} + +#[test] +fn the_search_finds_what_is_there_whatever_the_prefix() { + assert!(tests_support::found("aaaab", b"xxaaaaaaab")); + assert!(tests_support::found("needle", b"needle")); + assert!(!tests_support::found("needle", b"needl")); + assert!(!tests_support::found("aaab", b"aaaaaaaaaa")); + assert!(tests_support::found("abc", b"zabc")); + // The worst shape for a skip-based search (a needle of one letter and a haystack full of + // it) is walked once per byte, and still finds what is there. + let needle = format!("{}b", "a".repeat(64)); + let mut hay = vec![b'a'; 20_000]; + assert!(!tests_support::found(&needle, &hay)); + hay.extend_from_slice(needle.as_bytes()); + assert!(tests_support::found(&needle, &hay)); +} + +#[test] +fn the_sweep_charge_stays_fair_for_a_bulk_of_needles() { + // 80 removed strings swept over one 5 MiB haystack (a stream decoded in full): the search + // really is one pass per needle, but the budget must not charge the haystack once per + // needle — that charge (haystack × needle × spelling: 80 × 20 × 5 MiB, about twice the 4 GiB + // work budget) would refuse an ordinary bulk redaction out of hand. The haystack is charged + // once per scan and a needle's own charge is capped, so the sweep stays under the budget. + let hay = vec![b'x'; 5 << 20]; + let word = "SECRETVALUE-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + assert_eq!(tests_support::bulk_counts(&hay, word, 80, 0), Some(0)); + // Genuinely unbounded work still fails closed: the region scanned is charged in full, so a + // haystack that no longer fits in the remaining budget is an unswept surface, never a pass. + assert_eq!(tests_support::bulk_counts(&hay, word, 80, crate::verify::tests_support::work_budget() - (1 << 20)), None); +} + +#[test] +fn survivors_seen_before_the_work_limit_are_still_reported() { + // The page sweep stops when the work budget runs out, but the survivors it has already + // counted are settled: a survivor on an earlier page is reported where it sits, not hidden + // behind the limit (the page still reads unswept, so the proof still fails). + let (reported, unswept) = tests_support::page_sweep_survivors_survive_the_limit(); + assert!(reported, "a survivor counted before the limit is reported"); + assert!(unswept, "the page the limit stopped on is unswept"); +} + +#[test] +fn a_file_past_the_proof_budget_is_unverified_never_clean() { + // A page whose content is bigger than the proof may decode for one stream: the budget + // stops the sweep, and the result must say so. The surfaces read unswept (a page content + // stream is never excused as raw-only), the region reads unverifiable, the proof fails. + // Work that stopped is "not verified", never a pass. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let content = Stream::flate(Dict::new(), &big); + drop(big); + let mut doc = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default())); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(content)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed(), "{}", proof.to_text()); + assert!(!proof.failures.is_empty(), "{}", proof.to_text()); + for s in [Surface::ExtractedText, Surface::DecodedStreams] { + let r = surface(&proof, s); + assert_eq!(r.verdict, Verdict::Unswept, "{s:?}: {}", proof.to_text()); + assert_eq!((r.survivors, r.raw_only), (0, 0), "{s:?}: {}", proof.to_text()); + } + assert!(matches!(proof.entries[0].status, Status::Unverifiable | Status::Failed), "the region is not verified: {}", proof.to_text()); +} + +#[test] +fn font_widths_parsing_shares_the_work_budget() { + // A form invoked again rebuilds its fonts, so the `/W` array is paid for from the shared + // work budget, not from a per-font counter that every rebuild resets. + let (full, refused) = tests_support::font_widths_charge_shares_the_budget(); + assert!(full, "a small /W array is read in full"); + assert!(refused, "a spent work budget refuses the /W parsing"); +} + +#[test] +fn an_unreadable_overlay_stream_is_its_own_error() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + let broken = doc.add(Object::Stream(Stream::from_raw(dict, b"not deflate data".to_vec()))); + let err = snap.set_overlay_streams(&doc, &[broken]).unwrap_err(); + assert_eq!(err, RedactError::OverlayUnreadable); + assert!(!err.to_string().contains("page 0"), "{err}"); +} + +// ── The caller's credentials ───────────────────────────────────────────────────────────────── + +#[test] +fn a_nonempty_user_password_document_is_proven_with_those_credentials() { + // The redacted document is protected with a user password and saved: the proof's subject is + // now an encrypted file, and only the caller's own credentials can read it back. Every + // supported encryption algorithm. + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "pw-secret", + owner_password: "owner-secret", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + + // The credentials the caller holds prove the saved output. + let proof = snap.prove(&doc, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // An already saved file too. + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // Empty and wrong credentials fail closed: the output can't be read back, which is an + // unswept proof, never a pass. + for pw in [None, Some("wrong")] { + let proof = snap.prove_saved(&saved, pw, &ProofOptions::default()); + assert!(!proof.passed(), "{alg:?} {pw:?}: {}", proof.to_text()); + for s in &proof.surfaces { + assert_eq!(s.verdict, Verdict::Unswept, "{alg:?} {pw:?}: {}", proof.to_text()); + } + } + } +} + +// ── Compressed copies inside binary streams ────────────────────────────────────────────────── + +/// A stream object whose stored bytes are the Flate encoding of `data` (as `Stream::flate` +/// writes them), with the extra dictionary entries `dict`. +fn flate_object(dict: &str, data: &[u8]) -> Vec { + stream(&format!("{dict} /Filter /FlateDecode"), &Stream::flate(Dict::new(), data).raw) +} + +/// A one-page file whose resources point at a fake font: /F2 is a /TrueType font whose +/// /FontDescriptor carries `font_file` as its /FontFile2 (objects 7, 8 and 9). The font carries +/// `/Widths` (object 6) so the interpreter resolves it; nothing on the page draws with it. +fn font_file_spec(font_file: Vec) -> Spec<'static> { + Spec { + resources: "/Font << /F1 5 0 R /F2 7 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /TrueType /BaseFont /X /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor 8 0 R >>".to_vec(), + b"<< /Type /FontDescriptor /FontName /X /Flags 4 /FontFile2 9 0 R >>".to_vec(), + font_file, + ], + ..Spec::default() + } +} + +#[test] +fn a_compressed_copy_in_a_fake_font_file_is_found() { + // The visible copy is redacted normally, but a Flate-compressed copy of the secret hides in + // a fake /FontFile2. /Length1 makes the sweep class the stream as a binary font program, so + // its stored bytes hold no plaintext: only the decoded data gives the copy away. + let spec = font_file_spec(flate_object("/Length1 900", SECRET.as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor, "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert!(s.locations.iter().any(|l| l.contains("(decoded)")), "{:?}", s.locations); + assert!(!proof.passed()); +} + +#[test] +fn a_legitimate_font_file_is_swept_and_clean() { + // A font program that holds none of what was removed (as real ones do not) is searched and + // lets the redaction pass: no false survivor. + let spec = font_file_spec(flate_object("/Length1 900", b"\x00\x01\x00\x00\x00\x10glyph tables")); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Clean, "{}", proof.to_text()); +} + +#[test] +fn a_font_program_that_will_not_decode_is_searched_as_stored_bytes() { + // An unrelated, damaged font program the page reaches is not a reason to refuse: nothing + // redacted touched it, so its stored bytes are searched and it is counted as raw only. + let raw = Stream::flate(Dict::new(), b"a font program without the secret").raw.as_ref().to_vec(); + let cut = raw.len() - 8; + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..cut])); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Clean, 1), "{}", proof.to_text()); +} + +/// The standard page plus an image (object 7) drawn away from the secret. +fn image_doc(image: Vec) -> Document { + let mut content = CONTENT.to_vec(); + content.extend_from_slice(b" q 20 0 0 20 200 200 cm /Im1 Do Q"); + marked_content(&content, Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }) +} + +const IMG: &str = "/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"; + +fn assert_redacted_cleanly(mut doc: Document) -> Proof { + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); + proof +} + +#[test] +fn an_unrelated_truncated_flate_image_does_not_fail_the_proof() { + let raw = Stream::flate(Dict::new(), &[0x80u8; 400]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_image_over_the_stream_cap_does_not_fail_the_proof() { + // Highly compressible: stored in a few dozen KiB, decoded past MAX_STREAM. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let raw = Stream::flate(Dict::new(), &big).raw.as_ref().to_vec(); + drop(big); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_truncated_font_program_does_not_fail_the_redaction() { + let raw = Stream::flate(Dict::new(), b"glyph tables").raw.as_ref().to_vec(); + let doc = marked(font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..raw.len() - 6]))); + assert_redacted_cleanly(doc); +} + +#[test] +fn a_secret_in_the_stored_bytes_of_a_font_program_is_still_found() { + // The font program cannot be decoded (it claims Flate but holds plain bytes), so only its + // stored bytes are searched: the copy of the secret in them is a survivor. + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.to_text().contains(SECRET)); + assert!(!proof.passed()); +} + +/// The standard page plus an /Indexed colour space whose lookup table is object 7. +fn indexed_doc(lookup: Vec) -> Document { + marked_content(CONTENT, Spec { resources: "/ColorSpace << /CS1 [/Indexed /DeviceRGB 255 7 0 R] >>", extra: vec![lookup], ..Spec::default() }) +} + +#[test] +fn an_indexed_lookup_stream_is_searched_as_stored_bytes_only() { + // Unrelated and damaged: nothing redacted touched it, so it must not fail the proof. + let raw = Stream::flate(Dict::new(), &[0x40u8; 300]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(indexed_doc(stream("/Filter /FlateDecode", &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); + // A secret stored raw inside one that will not decode is still found. + let mut doc = indexed_doc(stream("/Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +/// Capture a sound document, then damage the stream `num` (a Flate stream cut short, with the +/// extra dictionary entries `dict`) and prove it. +fn prove_damaged(mut doc: Document, num: u32, dict: &str, data: &[u8]) -> Proof { + let snap = Snapshot::capture(&doc, None).unwrap(); + let raw = Stream::flate(Dict::new(), data).raw.as_ref().to_vec(); + let cut = raw[..raw.len() - 8].to_vec(); + let broken = open(pdf_bytes(vec![b"<< /Type /Catalog >>".to_vec(), stream(&format!("{dict} /Filter /FlateDecode"), &cut)], "")); + let obj = broken.get(ObjRef::new(2, 0)).as_ref().clone(); + doc.set(ObjRef::new(num, 0), obj); + doc.require_full_save(); + snap.prove(&doc, None, &ProofOptions::default()) +} + +#[test] +fn damaged_content_appearance_and_metadata_still_fail_closed() { + // A truncated second content stream. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let proof = prove_damaged(doc, 7, "", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + assert!(!surface(&proof, Surface::DecodedStreams).problems.is_empty(), "{}", proof.to_text()); + + // A truncated appearance stream. + let annot = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 8 0 R >> >>".to_vec(); + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot, ap], ..Spec::default() }); + doc.require_full_save(); + let proof = prove_damaged(doc, 8, "/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + + // A truncated XMP packet. + let xmp = stream("/Type /Metadata /Subtype /XML", b"some metadata of the file"); + let doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let proof = prove_damaged(doc, 7, "/Type /Metadata /Subtype /XML", b"some metadata of the file"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::Xmp).problems.is_empty(), "{}", proof.to_text()); +} + +#[test] +fn an_annotation_appearance_is_checked_however_it_is_stored() { + // The region holds no text, so only the paint check can speak. An annotation written in + // place (a direct dictionary, no reference of its own) is checked like an indirect one, and + // a stream shared by two annotations is checked at each annotation's own placement, so the + // copy that lands over the region is seen even though the first placement does not reach it. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |spec: Spec<'_>| -> Proof { + let mut shown = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + + let proof = prove(Spec { + page: "/Annots [<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>]", + extra: vec![ap], + ..Spec::default() + }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + let far = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 7 0 R >> >>".to_vec(); + let near = b"<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>".to_vec(); + let proof = prove(Spec { page: "/Annots [8 0 R 9 0 R]", extra: vec![ap, far, near], ..Spec::default() }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); +} + +#[test] +fn a_null_entry_in_the_contents_array_is_an_empty_piece() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R null] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + assert_redacted_cleanly(doc); +} + +/// The page's decoded content streams, joined (what the redaction left on the page). +fn page_content(doc: &Document) -> String { + let p = &pdfcraft_model::pages(doc)[0]; + let (_, data) = page_streams(doc, &p.dict, 0).unwrap(); + data.iter().map(|d| String::from_utf8_lossy(d).into_owned()).collect::>().join("\n") +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs new file mode 100644 index 0000000000..f6db96229b --- /dev/null +++ b/crates/redact/src/verify/extract.rs @@ -0,0 +1,711 @@ +//! A minimal text extractor, independent of the interpreter's font metrics and hit test. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::util::{Ctx, latin1, resolve_strict, squash}; +use crate::annots_of; +use crate::verify::limits::{MAX_CMAP, MAX_CMAP_ENTRIES, MAX_CMAP_WORK, MAX_DEPTH}; + +/// Is this one of the glyph-carrying text-showing operators (`Tj`, `'`, `"`, and `TJ` with at +/// least one non-empty string)? A `TJ` of numbers only (the gap redaction leaves for a removed +/// string) moves the cursor but shows nothing, so it does not count. +pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { + let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); + match op.op.as_slice() { + b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), + b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + _ => false, + } +} + +/// A page's text as two whitespace-free views: decoded text, and the character codes as Latin-1 +/// (what fonts without a usable mapping show). +#[derive(Default)] +pub(super) struct Extracted { + pub(super) text: String, + pub(super) raw: String, + pub(super) glyph_ops: usize, + /// Glyph-carrying operators whose text could not be decoded. + pub(super) undecodable_ops: usize, + /// Per region asked about: glyphs whose centre this extractor places inside it. + pub(super) shown_in: Vec, + /// Glyph-carrying operators drawn where this extractor can't place them (tiling patterns, + /// or an annotation appearance without a usable `/Rect`). + pub(super) unplaced: usize, +} + +/// How one font turns codes into text. +#[derive(Clone, Default)] +pub(super) struct Decoder { + pub(super) map: HashMap, + /// Bytes per code. + pub(super) width: usize, + /// No way to read these codes as text (composite font without a mapping). + pub(super) opaque: bool, +} + +impl Decoder { + fn latin1() -> Self { + Decoder { map: HashMap::new(), width: 1, opaque: false } + } + + /// `Err` when the font's own data can't be read in full (a damaged or oversized CMap). + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let composite = font.name(b"Subtype") == Some(b"Type0"); + let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; + let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); + if let Some(Object::Stream(s)) = tu.as_deref() { + let data = ctx.decode(s)?; + if data.len() > MAX_CMAP { + return Err(()); + } + let (map, width) = parse_cmap(&data).ok_or(())?; + if !map.is_empty() { + d.map = map; + if let Some(w) = width { + d.width = w; + } + return Ok(d); + } + } + if composite { + d.opaque = true; + return Ok(d); + } + // Simple font: Latin-1, with the /Differences of the encoding. + if let Some(enc) = font.get(b"Encoding").map(|e| doc.resolve(e)) + && let Some(diff) = enc.as_dict().and_then(|e| e.get(b"Differences")).map(|x| doc.resolve(x)) + && let Some(items) = diff.as_array() + { + let mut code = 0u32; + for it in items { + match &*doc.resolve(it) { + Object::Int(i) => code = u32::try_from(*i).unwrap_or(0), + Object::Name(n) => { + if let Some(c) = std::str::from_utf8(n).ok().and_then(pdfcraft_fonts::pdf::glyph_unicode) { + d.map.insert(code, c.to_string()); + } + code = code.saturating_add(1); + } + _ => {} + } + } + } + Ok(d) + } + + /// Each code of a shown string, with the text it gives (`None`: no way to read it as text). + fn split(&self, s: &[u8]) -> Vec<(u32, Option)> { + let w = self.width.max(1); + s.chunks(w) + .map(|c| { + let code = code_of(c); + let text = if self.opaque { + None + } else { + match self.map.get(&code) { + Some(t) => Some(t.clone()), + None if self.map.is_empty() || w == 1 && code < 256 => Some(char::from(u8::try_from(code).unwrap_or(b'?')).to_string()), + None => None, + } + }; + (code, text) + }) + .collect() + } + + /// The text of a shown string. + fn decode(&self, s: &[u8]) -> String { + self.split(s).into_iter().filter_map(|(_, t)| t).collect() + } +} + +/// A font as the geometric check sees it: its decoder and its own simple width model. +#[derive(Clone)] +pub(super) struct Font { + pub(super) dec: Decoder, + /// Advance per code, in text space units per 1 em of font size (0.5 = half an em). + pub(super) widths: HashMap, + /// Advance of a code `widths` doesn't list. + pub(super) default_width: f64, + pub(super) vertical: bool, +} + +/// The width a font without usable `/Widths` is assumed to have (half an em). +pub(super) const FLAT_WIDTH: f64 = 0.5; + +impl Font { + fn flat() -> Self { + Font { dec: Decoder::latin1(), widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false } + } + + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let dec = Decoder::new(doc, font, ctx)?; + let mut f = Font { dec, widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false }; + let num = |o: &Object| doc.resolve(o).as_f64(); + if font.name(b"Subtype") == Some(b"Type0") { + f.vertical = font.get(b"Encoding").map(|e| doc.resolve(e)).and_then(|e| e.as_name().map(|n| n.ends_with(b"-V"))).unwrap_or(false); + let desc = font.get(b"DescendantFonts").map(|d| doc.resolve(d)).and_then(|d| d.as_array().and_then(|a| a.first().cloned())); + let desc = desc.and_then(|d| doc.resolve(&d).as_dict().cloned()); + if let Some(d) = desc { + f.default_width = d.get(b"DW").and_then(num).map_or(1.0, |w| w / 1000.0); + // The array is charged to the shared work budget as it is taken (a form + // invoked again rebuilds its fonts, so a per-font counter alone would let the + // same work be paid for many times), and every entry is paid for there too, + // whether or not it grows the map (a range may rewrite the same codes again + // and again, and the array is bounded only by the file). + let items: Vec = match d.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + Some(a) => { + if !ctx.charge(a.len()) { + return Err(()); + } + a + } + None => Vec::new(), + }; + let mut work = 0usize; + let mut step = |n: usize| -> Result<(), ()> { + work = work.saturating_add(n); + if work <= MAX_CMAP_WORK && ctx.charge(n) { Ok(()) } else { Err(()) } + }; + let mut i = 0; + while let Some(first) = items.get(i).and_then(num) { + let first = first.max(0.0) as u32; + match items.get(i + 1).map(|o| doc.resolve(o)) { + Some(a) if a.as_array().is_some() => { + step(a.as_array().map_or(0, Vec::len))?; + for (k, w) in a.as_array().into_iter().flatten().enumerate() { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w / 1000.0); + } + } + i += 2; + } + Some(last) => { + let (Some(last), Some(w)) = (last.as_f64(), items.get(i + 2).and_then(num)) else { break }; + let last = (last.max(0.0) as u32).min(first.saturating_add(0xFFFF)); + // Each code the range covers is paid for, grown or not (a span is at + // most 0x10000 codes, so the count always fits). + step(last.saturating_sub(first) as usize + 1)?; + for c in first..=last { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + f.widths.insert(c, w / 1000.0); + } + i += 3; + } + None => break, + } + } + } + } else { + // Glyph space units per em: 1000, or what a Type3 font's matrix says. + let scale = if font.name(b"Subtype") == Some(b"Type3") { + font.get(b"FontMatrix").map(|m| doc.resolve(m)).and_then(|m| m.as_array().and_then(|a| a.first().and_then(num))).unwrap_or(0.001) + } else { + 0.001 + }; + let first = font.get(b"FirstChar").and_then(num).map_or(0, |v| v.max(0.0) as u32); + let missing = font + .get(b"FontDescriptor") + .map(|d| doc.resolve(d)) + .and_then(|d| d.as_dict().and_then(|d| d.get(b"MissingWidth").and_then(num))) + .map(|w| w * scale); + if let Some(ws) = font.get(b"Widths").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + if ws.len() > MAX_CMAP_ENTRIES { + return Err(()); + } + for (k, w) in ws.iter().enumerate() { + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w * scale); + } + } + f.default_width = missing.unwrap_or(0.0); + } else if let Some(std) = pdfcraft_fonts::Std14::from_base_font(&String::from_utf8_lossy(font.name(b"BaseFont").unwrap_or(b""))) { + // No /Widths: a standard font has exact published widths, by WinAnsi code (Symbol, + // ZapfDingbats and other encodings are refused before the proof is reached). + for code in 0..=255u8 { + f.widths.insert(u32::from(code), std.width(code) / 1000.0); + } + f.default_width = if std.is_fixed_pitch() { 0.6 } else { 0.0 }; + } + } + Ok(f) + } + + fn width(&self, code: u32) -> f64 { + self.widths.get(&code).copied().unwrap_or(self.default_width) + } +} + +pub(super) fn hex_token(tok: &[u8]) -> Option> { + let digits: Vec = tok.iter().filter_map(|c| (*c as char).to_digit(16).map(|d| d as u8)).collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + Some(digits.chunks(2).map(|p| p.iter().fold(0u8, |a, d| (a << 4) | d)).collect()) +} + +pub(super) fn utf16_text(b: &[u8]) -> String { + if let [c] = b { + return char::from(*c).to_string(); + } + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) +} + +pub(super) fn code_of(b: &[u8]) -> u32 { + b.iter().fold(0u32, |a, x| a.wrapping_shl(8) | u32::from(*x)) +} + +pub(super) enum Tok { + Hex(Vec), + Word(String), + Open, + Close, +} + +pub(super) fn cmap_tokens(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = data.get(i) { + match c { + b'<' if data.get(i + 1) != Some(&b'<') => { + let end = data.get(i..).and_then(|d| d.iter().position(|x| *x == b'>')).map_or(data.len(), |e| i + e); + if let Some(h) = data.get(i + 1..end).and_then(hex_token) { + out.push(Tok::Hex(h)); + } + i = end + 1; + } + b'[' => { + out.push(Tok::Open); + i += 1; + } + b']' => { + out.push(Tok::Close); + i += 1; + } + b'%' => { + while data.get(i).is_some_and(|x| !matches!(x, b'\n' | b'\r')) { + i += 1; + } + } + c if c.is_ascii_alphabetic() => { + let start = i; + while data.get(i).is_some_and(|x| x.is_ascii_alphanumeric()) { + i += 1; + } + out.push(Tok::Word(String::from_utf8_lossy(data.get(start..i).unwrap_or_default()).into_owned())); + } + _ => i += 1, + } + } + out +} + +/// `bfchar` / `bfrange` entries of a ToUnicode CMap, and the code length of its codespace. +/// `None` when the CMap is bigger than [`MAX_CMAP_ENTRIES`] entries or costs more than +/// [`MAX_CMAP_WORK`] steps: a map that is cut short would read as "no such code". +pub(super) fn parse_cmap(data: &[u8]) -> Option<(HashMap, Option)> { + let toks = cmap_tokens(data); + let mut map: HashMap = HashMap::new(); + let mut width = None; + // Every step is paid for, whether or not it grows the map (a range may rewrite the same + // codes again and again). + let mut work = 0usize; + let mut step = |n: usize| -> Option<()> { + work = work.saturating_add(n); + (work <= MAX_CMAP_WORK).then_some(()) + }; + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + let Tok::Word(w) = t else { continue }; + match w.as_str() { + "begincodespacerange" => { + if let Some(Tok::Hex(h)) = toks.get(i) { + width = Some(h.len()); + } + } + "beginbfchar" => { + while let (Some(Tok::Hex(src)), Some(Tok::Hex(dst))) = (toks.get(i), toks.get(i + 1)) { + step(1)?; + map.insert(code_of(src), utf16_text(dst)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 2; + } + } + "beginbfrange" => { + while let (Some(Tok::Hex(lo)), Some(Tok::Hex(hi))) = (toks.get(i), toks.get(i + 1)) { + let (lo, hi) = (code_of(lo), code_of(hi)); + let span = hi.saturating_sub(lo).min(0xFFFF); + match toks.get(i + 2) { + Some(Tok::Hex(dst)) => { + let base = utf16_text(dst); + step(usize::try_from(span).ok()?.saturating_add(1))?; + for k in 0..=span { + let mut chars: Vec = base.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = char::from_u32(u32::from(*last).saturating_add(k)).unwrap_or('\u{fffd}'); + } + map.insert(lo.saturating_add(k), chars.into_iter().collect()); + } + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 3; + } + Some(Tok::Open) => { + i += 3; + let mut k = 0u32; + while let Some(Tok::Hex(d)) = toks.get(i) { + step(1)?; + map.insert(lo.saturating_add(k), utf16_text(d)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + k = k.saturating_add(1); + i += 1; + } + i += 1; // the closing bracket + } + _ => break, + } + } + } + _ => {} + } + } + Some((map, width)) +} + +pub(super) fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { + res.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default() +} + +pub(super) fn push_squashed(into: &mut String, s: &str) { + into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); +} + +/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error +/// (guessing the identity would put its content in the wrong place). +pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + match dict.get(b"Matrix") { + None => Ok(Matrix::IDENTITY), + Some(m) => { + let m = doc.resolve(m); + let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + } + } +} + +/// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. +pub(super) fn bbox_of(doc: &Document, dict: &Dict) -> Result, ()> { + let Some(b) = dict.get(b"BBox") else { return Ok(None) }; + let b = doc.resolve(b); + let v: Vec = + b.as_array().ok_or(())?.iter().map(|x| doc.resolve(x).as_f64().filter(|v| v.is_finite()).ok_or(())).collect::>()?; + match v[..] { + [a, b, c, d] => Ok(Some([a.min(c), b.min(d), a.max(c), b.max(d)])), + _ => Err(()), + } +} + +/// The content streams of a page as `(raw object, decoded)`, a missing or unreadable entry an +/// error. `skip_overlay` leaves out the streams this run drew the boxes and labels with. +pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: bool) -> Result, ()> { + let list: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*resolve_strict(doc, c)? { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut data: Vec = Vec::new(); + for o in &list { + match &*resolve_strict(doc, o)? { + Object::Stream(s) => { + let d = ctx.decode(s)?; + if skip_overlay && ctx.is_overlay(&d) { + continue; + } + data.extend(d); + data.push(b'\n'); + } + // A null entry is an empty piece, as for the redaction itself (ISO 32000-2 7.3.9). + Object::Null => {} + _ => return Err(()), + } + } + Ok(data) +} + +/// The text of a page: its content, the forms and patterns it reaches (visible or not), and the +/// appearance streams of its annotations; and which of `regions` still hold glyphs. +pub(super) fn extract_page(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> Result { + let data = page_content(doc, page, ctx, true)?; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut ex = Extractor { doc, ctx, regions, out: Extracted { shown_in: vec![0; regions.len()], ..Extracted::default() }, stack: Vec::new() }; + ex.scope(&data, &res, Matrix::IDENTITY, 0, true)?; + for a in annots_of(doc, page) { + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = + d.get(b"Rect").and_then(|r| doc.resolve(r).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect::>())); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec<(Option, Object)> = match &*s { + Object::Stream(_) => vec![(state.as_ref(), (*s).clone())], + Object::Dict(sub) => sub.iter().map(|(_, v)| (v.as_ref(), (*doc.resolve(v)).clone())).collect(), + _ => Vec::new(), + }; + for (r, o) in streams { + if let Object::Stream(st) = o { + // Where the appearance lands on the page (ISO 32000-2 §12.5.5), if it can be told. + let ctm = rect.as_deref().and_then(|r| appearance_matrix(doc, &st, r)); + // A font of the appearance is looked up in the appearance's own resources + // first (what the redaction itself does), then in the page's. + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + let parent = own.as_ref().unwrap_or(&res); + ex.stream(&st, r, parent, ctm.unwrap_or_default(), 1, ctm.is_some())?; + } + } + } + } + Ok(ex.out) +} + +/// The matrix that maps an appearance stream's form space onto an annotation's `/Rect` +/// (ISO 32000-2 §12.5.5): the appearance's `/Matrix`, then the scale that fits the mapped +/// `/BBox` into the rectangle. `None` when that can't be told (no usable `/Rect`, `/BBox` or +/// `/Matrix`, or a mapped box that degenerates). +pub(crate) fn appearance_matrix(doc: &Document, ap: &Stream, rect: &[f64]) -> Option { + let [rx0, ry0, rx1, ry1] = rect[..] else { return None }; + let bbox = bbox_of(doc, &ap.dict).ok()??; + let m = matrix_of(doc, &ap.dict).ok()?; + let t = m.bbox(bbox); + let (tw, th) = (t[2] - t[0], t[3] - t[1]); + if tw <= 0.0 || th <= 0.0 { + return None; + } + let (rx0, rx1, ry0, ry1) = (rx0.min(rx1), rx0.max(rx1), ry0.min(ry1), ry0.max(ry1)); + let (sx, sy) = ((rx1 - rx0) / tw, (ry1 - ry0) / th); + let a = Matrix([sx, 0.0, 0.0, sy, rx0 - t[0] * sx, ry0 - t[1] * sy]); + Some(m.then(&a)) +} + +/// Text-state parameters that `q`/`Q` save and restore. +#[derive(Clone)] +pub(super) struct TextGs { + pub(super) ctm: Matrix, + pub(super) tc: f64, + pub(super) tw: f64, + pub(super) th: f64, + pub(super) tl: f64, + pub(super) rise: f64, + pub(super) size: f64, + pub(super) font: Vec, +} + +pub(super) struct Extractor<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) out: Extracted, + pub(super) stack: Vec, +} + +impl Extractor<'_> { + /// A form or pattern stream. `ctm` maps its space onto the page's; `placed` is false where + /// that mapping is not known. + fn stream(&mut self, s: &Stream, id: Option, parent: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if let Some(r) = id { + if self.stack.contains(&r) { + return Ok(()); + } + self.stack.push(r); + } + let result = (|| { + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) + })(); + if id.is_some() { + self.stack.pop(); + } + result + } + + /// Show the strings of one text-showing operator: extract their text, and place each glyph + /// to count the ones inside a region. `tm` moves along with the pen. + fn show_text(&mut self, font: &Font, op: &pdfcraft_content::Op, gs: &TextGs, tm: &mut Matrix, placed: bool) { + let o = op.op.as_slice(); + // Strings and TJ adjustments, in order. + let items: Vec<&Object> = match o { + b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => op.operands.last().into_iter().collect(), + }; + let mut any_text = false; + for it in items { + if let Some(n) = it.as_f64() { + let d = -n / 1000.0 * gs.size; + let m = if font.vertical { Matrix::translate(0.0, d) } else { Matrix::translate(d * gs.th, 0.0) }; + *tm = m.then(tm); + continue; + } + let Some(s) = it.as_string() else { continue }; + let t = font.dec.decode(&s.bytes); + any_text |= !squash(&t).is_empty(); + push_squashed(&mut self.out.text, &t); + push_squashed(&mut self.out.raw, &latin1(&s.bytes)); + for (code, text) in font.dec.split(&s.bytes) { + let w = font.width(code); + let space = text.as_deref().is_some_and(|t| t == " "); + // The glyph's centre, in text space (the box runs from 0.2 em below the + // baseline to 0.8 em above), and where the pen moves to. + let (centre, advance) = if font.vertical { + let adv = -gs.size + gs.tc + if space { gs.tw } else { 0.0 }; + ((0.0, gs.rise - 0.5 * gs.size), Matrix::translate(0.0, adv)) + } else { + let adv = (w * gs.size + gs.tc + if space { gs.tw } else { 0.0 }) * gs.th; + ((0.5 * w * gs.size * gs.th, gs.rise + 0.3 * gs.size), Matrix::translate(adv, 0.0)) + }; + let visible = text.as_deref().is_none_or(|t| !squash(t).is_empty()); + if placed && visible && !self.regions.is_empty() { + let (x, y) = tm.then(&gs.ctm).apply(centre.0, centre.1); + for (r, n) in self.regions.iter().zip(self.out.shown_in.iter_mut()) { + if x >= r[0] && x <= r[2] && y >= r[1] && y <= r[3] { + *n += 1; + } + } + } + *tm = advance.then(tm); + } + } + if carries_glyphs(op) { + self.out.glyph_ops += 1; + if !any_text { + self.out.undecodable_ops += 1; + } + if !placed { + self.out.unplaced += 1; + } + } + } + + fn scope(&mut self, data: &[u8], res: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if depth > MAX_DEPTH { + return Err(()); + } + let doc = self.doc; + let fonts = res_dict(doc, res, b"Font"); + let xobjects = res_dict(doc, res, b"XObject"); + let mut cache: HashMap, Font> = HashMap::new(); + let mut gs = TextGs { ctm, tc: 0.0, tw: 0.0, th: 1.0, tl: 0.0, rise: 0.0, size: 0.0, font: Vec::new() }; + let mut saved: Vec = Vec::new(); + let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); + for op in parse(data).ops { + let o = op.op.as_slice(); + match o { + b"q" => { + if saved.len() >= 4096 { + return Err(()); + } + saved.push(gs.clone()); + } + b"Q" => { + if let Some(g) = saved.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"BT" => { + tm = Matrix::IDENTITY; + tlm = Matrix::IDENTITY; + } + b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + b"Tf" => { + gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num(1).unwrap_or(0.0); + } + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if o == b"TD" { + gs.tl = -y; + } + tlm = Matrix::translate(x, y).then(&tlm); + tm = tlm; + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + tm = Matrix(m); + tlm = tm; + } + } + b"T*" => { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + b"Tj" | b"TJ" | b"'" | b"\"" => { + if matches!(o, b"'" | b"\"") { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + if o == b"\"" + && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + { + (gs.tw, gs.tc) = (aw, ac); + } + if !cache.contains_key(&gs.font) { + let f = match fonts.get(&gs.font).map(|f| doc.resolve(f)) { + Some(f) => f.as_dict().map_or_else(|| Ok(Font::flat()), |d| Font::new(doc, d, self.ctx))?, + None => Font::flat(), + }; + cache.insert(gs.font.clone(), f); + } + let Some(font) = cache.get(&gs.font) else { continue }; + self.show_text(font, &op, &gs, &mut tm, placed); + } + b"Do" => { + let Some(name) = op.name(0) else { continue }; + let Some(entry) = xobjects.get(name) else { continue }; + if let Object::Stream(s) = &*doc.resolve(entry) + && s.dict.name(b"Subtype") == Some(b"Form") + { + let m = matrix_of(doc, &s.dict)?; + bbox_of(doc, &s.dict)?; + self.stream(s, entry.as_ref(), res, m.then(&gs.ctm), depth + 1, placed)?; + } + } + _ => {} + } + } + // Tiling patterns draw content of their own, repeated where the pattern is used: its + // glyphs can't be placed. + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + if let Object::Stream(s) = &*doc.resolve(p) { + self.stream(s, p.as_ref(), res, Matrix::IDENTITY, depth + 1, false)?; + } + } + Ok(()) + } +} diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs new file mode 100644 index 0000000000..fb69f91351 --- /dev/null +++ b/crates/redact/src/verify/geometry.rs @@ -0,0 +1,500 @@ +//! The non-text content under the regions: images, paths and shadings, and the placement of the +//! glyphs still shown, checked independently of the interpreter. + +use std::collections::HashSet; + +use pdfcraft_content::{Matrix, contains, overlaps, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::extract::{appearance_matrix, bbox_of, matrix_of, page_content, res_dict}; +use super::util::{Ctx, image_codec}; +use crate::verify::limits::{MAX_CLIP_SUBPATHS, MAX_DEPTH, MAX_GS_STACK, MAX_HOLES, MAX_PIXELS}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum Finding { + /// This many glyphs are placed inside the region. + Text(usize), + Image, + Path, + Shading, + Missing, + Unverifiable(&'static str), +} + +#[derive(Clone)] +pub(super) struct Gs { + pub(super) ctm: Matrix, + /// Device-space boxes cut out of the clip (even-odd clips with a hole). + pub(super) holes: Vec<[f64; 4]>, +} + +pub(super) struct Geo<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) found: Vec<(usize, Finding)>, + /// What `found` already holds: each finding is recorded once per region, so a page that + /// paints thousands of times cannot grow the report (or the memory) with repeats. + reported: HashSet<(usize, Finding)>, + /// Pattern, soft-mask and Type3 resources already searched for images. + pub(super) searched: HashSet, +} + +/// What still paints inside `regions` on a page of the output (the overlay boxes excepted): +/// images with pixels there, paths and shadings not clipped away from there. +pub(super) fn check_geometry(doc: &Document, page: &Dict, regions: &[[f64; 4]], ctx: &Ctx) -> Vec<(usize, Finding)> { + let unverifiable = |why: &'static str| (0..regions.len()).map(|i| (i, Finding::Unverifiable(why))).collect(); + // The redaction boxes are painted on purpose. + let Ok(data) = page_content(doc, page, ctx, true) else { return unverifiable("the page content can't be read in full") }; + let mut geo = Geo { doc, ctx, regions, found: Vec::new(), reported: HashSet::new(), searched: HashSet::new() }; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + geo.run(&data, &res, Matrix::IDENTITY, Vec::new(), 0); + geo.appearances(page, &res); + geo.found +} + +impl Geo<'_> { + /// Record a finding for region `i`, once: a page that paints over a region a thousand + /// times is one finding, not a thousand. + fn report(&mut self, i: usize, f: Finding) { + if self.reported.insert((i, f.clone())) { + self.found.push((i, f)); + } + } + + /// The appearances of the annotations still on the page, checked like the page itself: their + /// content is placed with the appearance matrix (ISO 32000-2 §12.5.5) and swept for images, + /// paths and shadings under the regions. An appearance that has content but can't be placed + /// makes every region unverifiable: the redaction refuses such documents, so one in the + /// output means what it paints can't be vouched for. + fn appearances(&mut self, page: &Dict, res: &Dict) { + let doc = self.doc; + for a in crate::annots_of(doc, page) { + // A direct dictionary has no reference of its own; it is checked like an indirect + // one (what its appearance paints does not depend on how it is stored). + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + // A redaction mark's appearance outlines the marked area on purpose (and applying + // removes the mark whole): it is exempt like the boxes drawn over the regions. + if d.name(b"Subtype") == Some(b"Redact") { + continue; + } + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = crate::rect_of(doc, d.get(b"Rect")); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec = match &*s { + Object::Stream(st) => vec![st.clone()], + Object::Dict(sub) => sub + .iter() + .filter_map(|(_, v)| match &*doc.resolve(v) { + Object::Stream(st) => Some(st.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + for st in streams { + // A stream shared by several annotations is checked at each annotation's + // own placement: where it lands comes from each `/Rect`, so the first + // placement tells nothing about the others. Repeated decodes spend the + // same bounded budget as everything else. + let Ok(data) = self.ctx.decode(&st) else { + self.unverifiable("an annotation appearance can't be read in full"); + continue; + }; + if data.is_empty() { + continue; + } + let Some(ctm) = rect.as_ref().and_then(|r| appearance_matrix(doc, &st, r)) else { + self.unverifiable("an annotation appearance can't be placed, so what it paints can't be checked"); + continue; + }; + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(res), ctm, Vec::new(), 1); + } + } + } + } + + fn touching(&self, b: [f64; 4]) -> Vec { + (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| overlaps(*r, b, 0.0))).collect() + } + + /// Does something painting over `within` reach region `i` through the clip? Only the part of + /// the region the painting could touch has to be cut out. + fn unclipped(&self, i: usize, within: [f64; 4], gs: &Gs) -> bool { + let Some(r) = self.regions.get(i) else { return false }; + let reach = [r[0].max(within[0]), r[1].max(within[1]), r[2].min(within[2]), r[3].min(within[3])]; + !gs.holes.iter().any(|h| contains(*h, reach, 0.01)) + } + + fn run(&mut self, data: &[u8], res: &Dict, ctm: Matrix, holes: Vec<[f64; 4]>, depth: usize) { + if depth > MAX_DEPTH { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable("forms are nested too deep to check")); + } + return; + } + let xobjects = res_dict(self.doc, res, b"XObject"); + self.hidden_images(res, depth); + let mut gs = Gs { ctm, holes }; + let mut stack: Vec = Vec::new(); + // The current path in device space: its subpaths, and the clip operator it carries. + let mut sub: Vec> = Vec::new(); + let mut clip: Option = None; + for op in parse(data).ops { + let o = op.op.as_slice(); + if matches!(o, b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" | b"W" | b"W*") { + extend_path(&op, &gs.ctm, &mut sub, &mut clip); + continue; + } + if matches!(o, b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n") { + self.paint_path(o, &mut gs, &sub, clip); + sub.clear(); + clip = None; + continue; + } + sub.clear(); + clip = None; + match o { + b"q" => { + // The saved state carries the holes with it, so an unbounded stack of them + // is unbounded memory: past the cap the walk says so instead of growing. + if stack.len() >= MAX_GS_STACK { + return self.unverifiable("the content stacks graphics states too deeply to check"); + } + stack.push(gs.clone()); + } + b"Q" => { + if let Some(g) = stack.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"sh" => { + for i in 0..self.regions.len() { + if self.unclipped(i, [f64::MIN, f64::MIN, f64::MAX, f64::MAX], &gs) { + self.report(i, Finding::Shading); + } + } + } + b"BI" => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + self.report(i, Finding::Image); + } + } + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let obj = self.doc.resolve(entry); + let Object::Stream(s) = &*obj else { continue }; + match s.dict.name(b"Subtype") { + Some(b"Image") => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + let Some(region) = self.regions.get(i) else { continue }; + match image_cleared(self.doc, self.ctx, s, &gs.ctm, *region) { + Ok(true) => {} + Ok(false) => self.report(i, Finding::Image), + Err(()) => self.report(i, Finding::Unverifiable("an image under the region can't be inspected")), + } + } + } + Some(b"Form") => self.form(s, &gs, res, depth), + _ => {} + } + } + _ => {} + } + } + } + + /// A path-painting operator `o` ends the current path: report what it paints over a region, + /// and let an even-odd clip add its holes to `gs`. + fn paint_path(&mut self, o: &[u8], gs: &mut Gs, sub: &[Vec<(f64, f64)>], clip: Option) { + let pts: Vec<(f64, f64)> = sub.iter().flatten().copied().collect(); + let bbox = pts.iter().fold(None, |b: Option<[f64; 4]>, &(u, v)| { + Some(match b { + None => [u, v, u, v], + Some(b) => [b[0].min(u), b[1].min(v), b[2].max(u), b[3].max(v)], + }) + }); + if o != b"n" + && let Some(b) = bbox + { + for i in self.touching(b) { + if self.unclipped(i, b, gs) { + self.report(i, Finding::Path); + } + } + } + // An even-odd clip whose subpaths are all axis-aligned rectangles lets through what an + // odd number of the inner rectangles covers. Where a region lies under the outer box + // and an odd number of inner ones, it is cut out of the clip; under an even number it + // is painted and must be reported. Anything that is not a stock of disjoint rectangles + // (a triangle, a rotated box, overlapping inners, too many of them) cannot be told + // cheaply, so no hole is cut: what the clip lets through is then reported as paint + // (fail closed). This is how redaction itself clips regions out of what it leaves. + if clip != Some(true) { + return; + } + let Some(rects) = sub.iter().map(|s| axis_aligned_rect(s)).collect::>>() else { return }; + if rects.len() < 2 || rects.len() > MAX_CLIP_SUBPATHS { + return; + } + let (outer, inners) = (rects[0], &rects[1..]); + // Inner rectangles must be disjoint, or their overlaps would change the parity the + // reader draws with. + let disjoint = inners.iter().enumerate().all(|(x, a)| inners.iter().skip(x + 1).all(|b| !overlaps(*a, *b, 0.0))); + let regions: Vec<[f64; 4]> = self.regions.to_vec(); + let mut cut = Vec::new(); + for r in regions { + if contains(outer, r, 0.0) && disjoint && inners.iter().filter(|b| contains(**b, r, 0.0)).count() % 2 == 1 { + cut.push(r); + } + } + if gs.holes.len() + cut.len() > MAX_HOLES { + return self.unverifiable("the content cuts clips with too many pieces to check"); + } + gs.holes.extend(cut); + } + + fn unverifiable(&mut self, why: &'static str) { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable(why)); + } + } + + fn form(&mut self, s: &Stream, gs: &Gs, parent: &Dict, depth: usize) { + let (Ok(m), Ok(bbox)) = (matrix_of(self.doc, &s.dict), bbox_of(self.doc, &s.dict)) else { + self.unverifiable("a form under the region has a malformed /Matrix or /BBox"); + return; + }; + let fctm = m.then(&gs.ctm); + if let Some(bb) = bbox + && self.touching(fctm.bbox(bb)).is_empty() + { + return; + } + let Ok(data) = self.ctx.decode(s) else { + self.unverifiable("a form under the region can't be decoded in full"); + return; + }; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(parent), fctm, gs.holes.clone(), depth + 1); + } + + /// Images drawn from tiling patterns, soft-mask groups and Type3 glyph procedures can't be + /// placed or cleared by this check: if `res` reaches any, every region is unverifiable. + fn hidden_images(&mut self, res: &Dict, depth: usize) { + let doc = self.doc; + let mut streams: Vec<(Option, Stream)> = Vec::new(); + let take = |o: &Object, streams: &mut Vec<(Option, Stream)>| { + if let Object::Stream(s) = &*doc.resolve(o) { + streams.push((o.as_ref(), s.clone())); + } + }; + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + take(p, &mut streams); + } + for (_, g) in res_dict(doc, res, b"ExtGState").iter() { + if let Some(Object::Dict(sm)) = doc.resolve(g).as_dict().and_then(|g| g.get(b"SMask")).map(|m| (*doc.resolve(m)).clone()).as_ref() + && let Some(group) = sm.get(b"G") + { + take(group, &mut streams); + } + } + for (_, f) in res_dict(doc, res, b"Font").iter() { + let Some(font) = doc.resolve(f).as_dict().cloned() else { continue }; + if font.name(b"Subtype") != Some(b"Type3") { + continue; + } + if let Some(procs) = font.get(b"CharProcs").and_then(|c| doc.resolve(c).as_dict().cloned()) { + for (_, p) in procs.iter() { + take(p, &mut streams); + } + } + } + for (id, s) in streams { + if let Some(r) = id + && !self.searched.insert(r) + { + continue; + } + match self.has_image(&s, res, depth + 1) { + Ok(false) => {} + Ok(true) => self.unverifiable("an image inside a pattern, soft mask or Type3 glyph can't be placed or cleared"), + Err(()) => self.unverifiable("a pattern, soft mask or Type3 glyph can't be read in full"), + } + } + } + + /// Does the content of `s` (forms it draws included) draw an image? + fn has_image(&mut self, s: &Stream, parent: &Dict, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(()); + } + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + let res = own.as_ref().unwrap_or(parent); + let xobjects = res_dict(self.doc, res, b"XObject"); + for op in parse(&data).ops { + match op.op.as_slice() { + b"BI" => return Ok(true), + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; + match x.dict.name(b"Subtype") { + Some(b"Image") => return Ok(true), + Some(b"Form") => { + if let Some(r) = entry.as_ref() + && !self.searched.insert(r) + { + continue; + } + if self.has_image(x, res, depth + 1)? { + return Ok(true); + } + } + _ => {} + } + } + _ => {} + } + } + Ok(false) + } +} + +/// Extend the current path (`sub`: its subpaths in device space; `clip`: the clip operator it +/// carries) by one construction operator. +fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { + let o = op.op.as_slice(); + let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); + let at = |x: f64, y: f64| ctm.apply(x, y); + match o { + b"m" => { + if let [x, y] = nums[..] { + sub.push(vec![at(x, y)]); + } + } + b"l" | b"c" | b"v" | b"y" => { + if let Some(cur) = sub.last_mut() { + cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + } + } + b"re" => { + if let [x, y, w, h] = nums[..] { + sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); + } + } + b"W" => *clip = Some(false), + b"W*" => *clip = Some(true), + _ => {} + } +} + +/// The axis-aligned rectangle a four-corner subpath draws, in device space: `None` for anything +/// else (open paths, more or fewer corners, skewed or rotated boxes, non-finite coordinates). +fn axis_aligned_rect(pts: &[(f64, f64)]) -> Option<[f64; 4]> { + // `re` leaves its rectangle open: the four corners are the whole subpath. A rectangle drawn + // by hand closes by repeating the first point at the end. Both spell the same four corners. + let pts = match pts { + [a, _, _, _, e] if a == e => &pts[..4], + _ => pts, + }; + let [p0, p1, p2, p3] = pts else { return None }; + let finite = |p: &(f64, f64)| p.0.is_finite() && p.1.is_finite(); + if !finite(p0) || !finite(p1) || !finite(p2) || !finite(p3) { + return None; + } + // The edges alternate between horizontal and vertical, whichever corner they start from and + // whichever way the rectangle is wound. + let flat = |a: &(f64, f64), b: &(f64, f64)| a.1 == b.1 && a.0 != b.0; + let steep = |a: &(f64, f64), b: &(f64, f64)| a.0 == b.0 && a.1 != b.1; + let even = flat(p0, p1) && steep(p1, p2) && flat(p2, p3) && steep(p3, p0); + let odd = steep(p0, p1) && flat(p1, p2) && steep(p2, p3) && flat(p3, p0); + if !even && !odd { + return None; + } + let (x0, x1) = (p0.0.min(p2.0), p0.0.max(p2.0)); + let (y0, y1) = (p0.1.min(p2.1), p0.1.max(p2.1)); + (x0 < x1 && y0 < y1).then_some([x0, y0, x1, y1]) +} + +/// Are the samples of image `s` whose centres fall in `region` all clear (zero; for image masks, +/// the sample that does not paint)? `Err` when the image can't be inspected. +#[allow(clippy::result_unit_err)] +pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, region: [f64; 4]) -> Result { + if image_codec(&s.dict) { + // Still encoded pixels: nothing was cleared. + return Ok(false); + } + let (w, h) = (s.dict.int(b"Width").ok_or(())?, s.dict.int(b"Height").ok_or(())?); + if w <= 0 || h <= 0 || (w as u64).saturating_mul(h as u64) > MAX_PIXELS { + return Err(()); + } + let (w, h) = (usize::try_from(w).map_err(|_| ())?, usize::try_from(h).map_err(|_| ())?); + let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + let (ncomp, bpc) = if mask { + (1, 1) + } else { + let cs = doc.resolve(s.dict.get(b"ColorSpace").ok_or(())?); + let name = match &*cs { + Object::Name(n) => n.clone(), + Object::Array(a) => a.first().and_then(Object::as_name).ok_or(())?.to_vec(), + _ => return Err(()), + }; + let n = match name.as_slice() { + b"DeviceGray" | b"G" | b"CalGray" | b"Indexed" | b"I" | b"Separation" => 1, + b"DeviceRGB" | b"RGB" | b"CalRGB" | b"Lab" => 3, + b"DeviceCMYK" | b"CMYK" => 4, + // One component per colorant name. + b"DeviceN" => doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?).as_array().ok_or(())?.len(), + b"ICCBased" => match &*doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?) { + Object::Stream(icc) => usize::try_from(icc.dict.int(b"N").ok_or(())?).map_err(|_| ())?, + _ => return Err(()), + }, + _ => return Err(()), + }; + (n, usize::try_from(s.dict.int(b"BitsPerComponent").ok_or(())?).map_err(|_| ())?) + }; + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { + return Err(()); + } + let data = ctx.decode(s)?; + let bits = ncomp.checked_mul(bpc).ok_or(())?; + let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); + if data.len() < row.checked_mul(h).ok_or(())? { + return Err(()); + } + let inverted = s.dict.get(b"Decode").is_some_and(|d| { + let d = doc.resolve(d); + d.as_array().and_then(|a| a.first()).is_some_and(|first| doc.resolve(first).as_f64() == Some(1.0)) + }); + let want = u8::from(mask && !inverted); + for j in 0..h { + let v = 1.0 - (j as f64 + 0.5) / h as f64; + // Rows that don't reach the region hold nothing to check. + let ((ax, ay), (bx, by)) = (ctm.apply(0.0, v), ctm.apply(1.0, v)); + if !overlaps(region, [ax.min(bx), ay.min(by) - 0.01, ax.max(bx), ay.max(by) + 0.01], 0.0) { + continue; + } + for i in 0..w { + let (x, y) = ctm.apply((i as f64 + 0.5) / w as f64, v); + if x < region[0] || x > region[2] || y < region[1] || y > region[3] { + continue; + } + let start = j.checked_mul(row).and_then(|v| v.checked_mul(8)).and_then(|v| v.checked_add(i.checked_mul(bits)?)).ok_or(())?; + for bit in start..start.checked_add(bits).ok_or(())? { + let byte = data.get(bit / 8).ok_or(())?; + if (byte >> (7 - bit % 8)) & 1 != want { + return Ok(false); + } + } + } + } + Ok(true) +} diff --git a/crates/redact/src/verify/limits.rs b/crates/redact/src/verify/limits.rs new file mode 100644 index 0000000000..e2972bb5f6 --- /dev/null +++ b/crates/redact/src/verify/limits.rs @@ -0,0 +1,42 @@ +//! Bounds of the redaction proof (`verify`). Hitting one is a failure the proof reports, never +//! a quiet skip: a proof that cannot sweep everything it must is not a passing proof. + +/// Most decoded bytes of one stream the proof reads. +pub(crate) const MAX_STREAM: usize = 64 << 20; +/// Most decoded bytes the whole proof reads, over every stream and every pass. +pub(crate) const MAX_TOTAL_DECODED: u64 = 2 << 30; +/// Most units of linear search work (input, patterns and reported matches) in one proof. +pub(crate) const MAX_WORK: u64 = 4 << 30; +/// Nested attachment levels that are swept (deeper ones are reported as unswept). +pub(crate) const MAX_NESTING: usize = 4; +/// Nesting bound of object walks (deeper is reported, never skipped silently). +pub(crate) const MAX_WALK: usize = 64; +/// Deepest graphics-state stack the paint check follows (saved states carry their clip holes +/// with them, so an unbounded stack is unbounded memory). Deeper is unverifiable. +pub(crate) const MAX_GS_STACK: usize = 4096; +/// Most clip holes one walk may cut (each is kept until the state that made it is restored). +/// More is unverifiable. +pub(crate) const MAX_HOLES: usize = 4096; +/// Most subpaths of one even-odd clip the hole rule examines (it compares them pairwise). +/// More, and no hole is cut: what the clip lets through is reported as paint. +pub(crate) const MAX_CLIP_SUBPATHS: usize = 64; +/// Most decoded bytes of an embedded CMap. +pub(crate) const MAX_CMAP: usize = 4 << 20; +/// Most CMap range steps one font may cost (a range that never grows the map still costs). +pub(crate) const MAX_CMAP_WORK: usize = 4 << 20; +/// Most entries one ToUnicode CMap may define. +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 20; +/// Needles shorter than this (in characters) are too common to look for in raw bytes. +pub(crate) const MIN_NEEDLE: usize = 3; +/// Smallest needle looked for in the raw file and in decoded streams. Those hold object syntax, +/// numbers and binary data, where three characters match by chance (`/Length 612`, a +/// `/MediaBox`, pixels); shorter needles are still swept on page text, strings, names, Info, XMP +/// and attachment text. +pub(crate) const MIN_STRUCTURED_NEEDLE: usize = 6; +/// Deepest nesting of forms, patterns and appearance streams the proof's independent walks +/// follow: the bound the interpreter itself walks forms at, so the proof reaches everything a +/// redaction could have removed and the two cannot drift apart. +pub(crate) const MAX_DEPTH: usize = crate::interp::MAX_DEPTH; +/// Most pixels an image may have for the proof to inspect it (larger ones are removed by +/// [`apply`](crate::apply), and the proof's pixel look has the same ceiling: `crate::image`). +pub(crate) const MAX_PIXELS: u64 = crate::image::MAX_PIXELS; diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs new file mode 100644 index 0000000000..b374f93fdc --- /dev/null +++ b/crates/redact/src/verify/mod.rs @@ -0,0 +1,389 @@ +//! An independent proof that redaction removed what it claimed to remove. +//! +//! [`apply`](crate::apply) cuts glyphs out with its own interpreter, so asking that interpreter +//! whether anything is left would only repeat its own mistakes. The proof therefore works from +//! the other end, and does not trust the interpreter's font metrics or hit test: +//! +//! 1. **Before** applying, [`Snapshot::capture`] records, per redaction region, the glyph runs the +//! interpreter is about to remove (their character codes and the text the font gives them), a +//! digest of each page's content, and the text of the whole document as a separate extractor +//! sees it. The strings the sweep looks for therefore *start* from what the interpreter +//! removed; a glyph it wrongly kept is caught by the geometric check below, not by a needle. +//! 2. **After** applying, [`Snapshot::prove`] serializes the document and sweeps the *output* +//! for those strings on every surface a reader could recover them from: the raw file, every +//! string object, every decoded stream (orphans included), the text of every page (read by a +//! minimal extractor of its own, not the interpreter's font metrics or hit test), the Info +//! dictionary, the XMP packet, embedded files (PDF attachments are swept recursively) and the +//! revision structure. Each string is looked for in every encoding a PDF can hold it in. +//! 3. It places every glyph still shown on the redacted pages with its own text-state tracking +//! and width model (the font's `/Widths`, else a flat guess) and fails a region when the +//! centre of a surviving glyph lies inside it, whatever the interpreter decided. +//! 4. It also re-checks the non-text content under each region — on the page and in the +//! appearance streams of the annotations left on it: no image still carries pixels there, and +//! no path or shading paints there unclipped. +//! +//! The proof fails closed: a survivor, but also anything it could not sweep (page content, a form +//! or an appearance stream, XMP, or an attachment that won't decode in full or can't be opened; a +//! nesting or work limit hit; an appearance with content that can't be placed on its page), makes +//! it fail. The one exception is binary data that redaction +//! never edits: a font program, colour profile, `/Indexed` lookup table, function sample table or +//! Flate/LZW image that is damaged or too long to decode is searched as stored bytes and reported +//! as `raw_only`, not as unswept. A proof that cannot fail is worthless. +//! +//! Work is bounded: every stream is decoded strictly under `limits::MAX_STREAM`, all decodes together +//! under `limits::MAX_TOTAL_DECODED`, and all searching under `limits::MAX_WORK`. A document that needs more +//! is reported as unswept, not skipped (binary data as above excepted). +//! +//! The result is a [`Proof`]: a per-region manifest (page, region, glyph counts, SHA-256 of +//! the page content after) and a verdict per surface. It holds counts and digests, +//! never the redacted text, unless [`ProofOptions::include_plaintext`] asks for it. +//! +//! Limits, stated plainly: text that exists only as pixels of an image is not text; a string +//! that also appears in content that was *not* redacted is counted, over the whole document, on +//! the page text, the raw file and the decoded streams (those hold the kept copy too) and fails +//! on a surplus; names are swept against the matches the input itself had, since a redacted word +//! that is also a `/Subtype` or a `/BaseFont` is structure, not a leftover copy, and a surplus of +//! matches still fails; every other surface is swept with no allowance. A word of six characters or more +//! that also spells a structural name (`/Filter`, `/MediaBox`) can still be reported by the raw-file and +//! decoded-stream surfaces, which search stored bytes and cannot tell a name from a copy: that is +//! over-reporting, never a pass. The raw file and the decoded +//! streams are searched for strings of at least six characters only (shorter ones match object +//! syntax and numbers by chance), skipping the trailer's file identifier and the +//! pixels of an image codec (DCT, JPX, JBIG2, CCITT), which are searched as stored bytes only; shorter strings are still found on the page text, +//! in strings, names, Info, XMP and attachment text. A region without extractable text is reported as such, and as +//! unverifiable when the page draws text this extractor cannot decode. The geometric check +//! places glyphs with the font's `/Widths` only: for a font without them the widths are a guess, +//! so a neighbouring glyph may be reported when the guess puts its centre inside a region. +//! `runs_removed` and `codes_removed` are exact counts of what was cut: a proof that is shared +//! discloses how much text sat under each region (never what it said). + +use std::fmt::Write as _; + +use pdfcraft_cos::ObjRef; + +mod extract; +mod geometry; +pub(crate) mod limits; +#[cfg(test)] +mod search_tests; +mod snapshot; +mod sweep; +mod util; + +pub use snapshot::Snapshot; +/// What the proof established for one region. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + /// What was removed was swept for on every surface and is gone. + Verified, + /// The region held no extractable text: nothing was removed, and that is recorded as such. + VerifiedNoTextInRegion, + /// Neither "text was removed" nor "there was none" can be checked (the page draws text this + /// extractor can't decode, or an image under the region can't be inspected). The proof can't + /// pass while any region is unverifiable. + Unverifiable, + /// A survivor was found, or the mechanical evidence is missing. + Failed, +} + +/// The places a redacted string is looked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Surface { + RawBytes, + ObjectStrings, + DecodedStreams, + ExtractedText, + Info, + Xmp, + EmbeddedFiles, + Revisions, +} + +const SURFACES: [Surface; 8] = [ + Surface::RawBytes, + Surface::ObjectStrings, + Surface::DecodedStreams, + Surface::ExtractedText, + Surface::Info, + Surface::Xmp, + Surface::EmbeddedFiles, + Surface::Revisions, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// Swept, nothing found. + Clean, + /// Swept, a redacted string (or an earlier revision) is still there. + Survivor, + /// Could not be swept; the proof fails. + Unswept, + /// There is nothing of this kind in the document. + Absent, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SurfaceReport { + pub surface: Surface, + pub verdict: Verdict, + /// Streams, strings, pages or files looked at. + pub scanned: usize, + /// Streams scanned as stored bytes only: image-codec pixels (not text), and font, colour or + /// image streams that could not be decoded in full. + pub raw_only: usize, + pub survivors: usize, + /// Where each survivor sits ("object 12", "page 1", "attachment #1 > object 4"); never the text. + pub locations: Vec, + /// Why the surface could not be swept. + pub problems: Vec, +} + +/// The manifest entry of one redaction region. +#[derive(Clone, Debug, PartialEq)] +pub struct Excision { + /// 0-based page. + pub page: usize, + /// The region in user space. + pub region: [f64; 4], + /// The Redact annotation it came from. + pub mark: ObjRef, + pub status: Status, + pub detail: String, + /// Glyph runs (consecutive glyphs of one string) removed under the region, and their + /// character codes. + pub runs_removed: usize, + pub codes_removed: usize, + /// SHA-256 of the page's decoded content streams, joined, after redaction. There is no digest + /// of the content before: a proof is meant to be shared, and a hash of the original page + /// would let a reader confirm a guess at what it said. + pub page_sha256_after: Option, + /// Glyph-carrying text operators on the page (forms included) before and after. A `TJ` of + /// numbers only (the gap left for a removed string) carries no glyphs and does not count. + pub text_ops_before: Option, + pub text_ops_after: Option, + /// The removed text, only when [`ProofOptions::include_plaintext`] is set. + pub removed_text: Option>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Proof { + pub entries: Vec, + pub surfaces: Vec, + /// Located, plain reasons for every failure (never the redacted text). + pub failures: Vec, +} + +impl Proof { + /// Survivors found across all surfaces. + pub fn survivors(&self) -> usize { + self.surfaces.iter().map(|s| s.survivors).sum() + } + + /// Surfaces that could not be swept. + pub fn unswept(&self) -> usize { + self.surfaces.iter().filter(|s| s.verdict == Verdict::Unswept || !s.problems.is_empty()).count() + } + + pub fn unverifiable(&self) -> usize { + self.entries.iter().filter(|e| e.status == Status::Unverifiable).count() + } + + /// Pass only when nothing failed, nothing is unverifiable and no surface went unswept. + pub fn passed(&self) -> bool { + self.failures.is_empty() + && self.survivors() == 0 + && self.unswept() == 0 + && self.entries.iter().all(|e| matches!(e.status, Status::Verified | Status::VerifiedNoTextInRegion)) + } + + /// A human-readable report: the verdict, the manifest and the surfaces. It names places and + /// counts, and the removed text only if it was asked for. + pub fn to_text(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "Redaction proof: {}", if self.passed() { "PASS" } else { "FAIL" }); + for f in &self.failures { + let _ = writeln!(s, " failure: {f}"); + } + for (i, e) in self.entries.iter().enumerate() { + let _ = writeln!( + s, + " region {} page {} {:?} {:?}: {} run(s), {} code(s) removed; text ops {:?} -> {:?}; page sha256 after {}{}", + i + 1, + e.page + 1, + e.region, + e.status, + e.runs_removed, + e.codes_removed, + e.text_ops_before, + e.text_ops_after, + e.page_sha256_after.as_deref().unwrap_or("-"), + if e.detail.is_empty() { String::new() } else { format!(" ({})", e.detail) }, + ); + } + for r in &self.surfaces { + let _ = writeln!( + s, + " {:?}: {:?} ({} scanned, {} raw only, {} survivor(s)){}", + r.surface, + r.verdict, + r.scanned, + r.raw_only, + r.survivors, + if r.locations.is_empty() { String::new() } else { format!(" at {}", r.locations.join(", ")) }, + ); + for p in &r.problems { + let _ = writeln!(s, " unswept: {p}"); + } + } + let _ = writeln!(s, " Evidence, not a guarantee: text that exists only as pixels of an image is not text, and is not searched."); + s + } +} + +/// What a proof may contain. +#[derive(Clone, Debug, Default)] +pub struct ProofOptions { + /// Include the removed text in the manifest. Off by default: a proof is meant to be shared, + /// and must not carry what it proves was removed. + pub include_plaintext: bool, +} + +/// Handles for the tests to reach the byte sweep without a whole document. +#[cfg(test)] +pub(crate) mod tests_support { + use pdfcraft_content::parse; + use pdfcraft_cos::{Dict, Object, Stream}; + + use super::extract::{carries_glyphs, parse_cmap}; + use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; + use super::sweep::{Needle, add_needle, bytes_counts, bytes_hits, walk_strings}; + use super::util::Ctx; + pub(crate) use super::util::survivor_encodings; + + /// The proof's whole search-work budget (for tests that pre-spend part of it). + pub(crate) fn work_budget() -> u64 { + MAX_WORK + } + + /// The byte sweep of `hay` for a bulk of `count` distinct needles, on a context already + /// `spent` units into its work budget: `Some` (with the needles that occur) while the sweep + /// stays under the budget, `None` once it ran out. + pub(crate) fn bulk_counts(hay: &[u8], text: &str, count: usize, spent: u64) -> Option { + let mut needles: Vec = Vec::new(); + for i in 0..count { + add_needle(&mut needles, false, format!("{text}{i:04}"), Some(i)); + } + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + bytes_counts(&needles, hay, MIN_STRUCTURED_NEEDLE, &ctx).map(|hits| hits.len()) + } + + /// The glyph-carrying text-showing operators of a decoded content stream. + pub fn count_text_operators(decoded: &[u8]) -> usize { + parse(decoded).ops.iter().filter(|op| carries_glyphs(op)).count() + } + + /// Does the byte sweep for `text` find it in `hay`? + pub fn found(text: &str, hay: &[u8]) -> bool { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, text.to_string(), Some(0)); + bytes_hits(&needles, hay, 3, &Ctx::new(Default::default())).is_some_and(|h| !h.is_empty()) + } + + /// Does this CMap parse completely (not cut short by a bound)? + pub fn cmap_is_complete(data: &[u8]) -> bool { + parse_cmap(data).is_some() + } + + /// Is an object nested `depth` levels deep walked in full? + pub fn walks_in_full(depth: usize) -> bool { + let mut o = Object::Null; + for _ in 0..depth { + o = Object::Array(vec![o]); + } + walk_strings(&o, 0, &mut |_| {}) + } + + /// Does decoding fail once the cumulative decode budget is used up (and not before)? + pub fn decode_budget_runs_out() -> (bool, bool) { + let ctx = Ctx::new(Default::default()); + let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let before = ctx.decode(&s).is_ok(); + let spent = Ctx::with_decode_left(50); + (before, spent.decode(&s).is_err()) + } + + /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb + /// cannot be decoded (and fail) again and again for free? + /// `(refused, charged, then refused for want of allowance)`. + pub fn failed_decode_charges_the_budget() -> (bool, bool, bool) { + let big = vec![0u8; MAX_STREAM + (1 << 20)]; + let bomb = Stream::flate(Dict::new(), &big); + drop(big); + let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); + let refused = ctx.decode(&bomb).is_err(); + let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; + let gone = ctx.decode(&small).is_err(); + (refused, charged, gone) + } + + /// Does the byte search report `None` once the work budget is used up (and not before)? + pub fn search_budget_runs_out() -> (bool, bool) { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, "secret".to_string(), Some(0)); + let ctx = Ctx::new(Default::default()); + let before = bytes_hits(&needles, b"some bytes", 3, &ctx).is_some(); + ctx.work.set(MAX_WORK); + (before, bytes_hits(&needles, b"some bytes", 3, &ctx).is_none()) + } + + /// Is a survivor counted on an earlier page still reported when the work budget runs out on + /// a later one? `(survivor reported, page left unswept)`. + pub fn page_sweep_survivors_survive_the_limit() -> (bool, bool) { + use super::Surface; + use super::sweep::{Acc, sweep_pages}; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"), + b"<< /Type /Font /Subtype /TrueType /BaseFont /Arial >>".to_vec(), + crate::tests_verify::stream("", &format!("BT /F1 10 Tf 10 100 Td ({}) Tj ET", "x".repeat(700_000)).into_bytes()), + ]; + let doc = pdfcraft_cos::Document::open(std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, ""))).unwrap(); + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRETVALUE".to_string(), Some(0)); + let mut acc = Acc::new(Ctx::new(Default::default())); + // Page 1 (the survivor's) fits in what is left; page 2 does not. + acc.ctx.work.set(MAX_WORK - 100_000); + sweep_pages(&needles, &doc, 0, "", &mut acc); + let text = acc.reps.get(&Surface::ExtractedText).unwrap(); + (text.survivors > 0, !text.problems.is_empty()) + } + + /// Does a font's `/W` parsing answer to the shared work budget, and not only to its own + /// counter (a form invoked again rebuilds its fonts, and would pay for the same array + /// again)? `(read in full, refused on a spent budget)`. + pub fn font_widths_charge_shares_the_budget() -> (bool, bool) { + use super::extract::extract_page; + let font = b"<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 1000 /W [1 65535 250] >>] >>".to_vec(); + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F2 5 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F2 10 Tf 10 100 Td <00410042> Tj ET"), + font, + ]; + let bytes = std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, "")); + let probe = |spent: u64| -> bool { + let doc = pdfcraft_cos::Document::open(bytes.clone()).unwrap(); + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + let page = &pdfcraft_model::pages(&doc)[0]; + extract_page(&doc, &page.dict, &ctx, &[]).is_ok() + }; + (probe(0), !probe(MAX_WORK - 10_000)) + } +} diff --git a/crates/redact/src/verify/search_tests.rs b/crates/redact/src/verify/search_tests.rs new file mode 100644 index 0000000000..4b8d84c243 --- /dev/null +++ b/crates/redact/src/verify/search_tests.rs @@ -0,0 +1,34 @@ +use super::{ + sweep::{add_needle, bytes_counts, bytes_hits}, + util::Ctx, +}; +use std::collections::HashSet; + +#[test] +fn a_thousand_needles_share_one_multimegabyte_scan() { + let mut needles = Vec::new(); + for i in 0..1000 { + add_needle(&mut needles, false, format!("SECRET_{i:04}"), None); + } + let hay = vec![b'z'; 8 << 20]; + assert_eq!(bytes_hits(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); +} + +#[test] +fn overlapping_encodings_count_one_occurrence() { + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRET42".into(), None); + let text: Vec = "SECRET42".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let mut hay = vec![0xfe, 0xff]; + hay.extend(text); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(vec![(0, 1)])); +} + +#[test] +fn match_count_exhaustion_is_incomplete_not_a_truncated_pass() { + let mut needles = Vec::new(); + add_needle(&mut needles, true, "SECRET42".into(), None); + let hay = b"SECRET42 ".repeat((1 << 16) + 1); + assert!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())).is_none()); +} diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs new file mode 100644 index 0000000000..fe74571bb4 --- /dev/null +++ b/crates/redact/src/verify/snapshot.rs @@ -0,0 +1,532 @@ +//! The snapshot taken before applying (what is about to go) and the proof run on the output. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full}; + +use super::extract::extract_page; +use super::geometry::{Finding, check_geometry}; +use super::sweep::{Acc, Needle, add_needle, bytes_hits, sweep_doc, walk_strings}; +use super::util::{Ctx, count, latin1, sha256, squash, words}; +use super::{Excision, Proof, ProofOptions, SURFACES, Status, Surface}; +use crate::interp::{Mode, Removed, Scope, process}; +use crate::verify::limits::MIN_NEEDLE; +use crate::{Mark, RedactError, Report, page_streams}; + +pub(super) struct RegionSnap { + pub(super) page: usize, + pub(super) region: [f64; 4], + pub(super) mark: ObjRef, + pub(super) runs: Vec<(Vec, String)>, + /// The runs of one text operator joined, where it split them: `[(sec) (ret)] TJ` reads as + /// `secret`, which a copy elsewhere in the file holds whole. + pub(super) joined: Vec<(Vec, String)>, + /// The character codes the runs hold (a composite font's codes are several bytes). + pub(super) codes: usize, +} + +pub(super) struct PageSnap { + pub(super) sha: Option, + pub(super) text_ops: Option, + pub(super) undecodable_ops: usize, +} + +/// What redaction is about to remove, recorded before it does. +pub struct Snapshot { + regions: Vec, + pages: BTreeMap, + needles: Vec, + /// Things that made the "before" side unreadable (reported on the text surface). + problems: Vec, + /// The document's page text before redaction (decoded view, code view), for the allowance + /// of needles added later. + src: (String, String), + /// The names of the document (structure redaction never edits), for the name allowance of + /// needles added later. + names: BTreeMap, usize>, + /// Digests of the decoded content of the streams this run created to draw the redaction + /// boxes and labels (see [`Snapshot::set_overlay_streams`]). + overlay: HashSet, +} + +impl std::fmt::Debug for Snapshot { + // The snapshot holds what is about to be redacted: never print it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Snapshot").field("regions", &self.regions.len()).field("needles", &self.needles.len()).finish() + } +} + +impl Snapshot { + /// Record what redacting the marks on `pages` (0-based; `None` = all) will remove. Call it + /// before [`apply`](crate::apply); it changes nothing. + pub fn capture(doc: &Document, pages: Option<&[usize]>) -> Result { + let all = crate::marks(doc); + let chosen: Vec<&Mark> = all.iter().filter(|m| pages.is_none_or(|p| p.contains(&m.page))).collect(); + if chosen.is_empty() { + return Err(RedactError::NothingToApply); + } + Snapshot::capture_marks(doc, &chosen) + } + + pub(crate) fn capture_marks(doc: &Document, chosen: &[&Mark]) -> Result { + let mut by_page: Vec = chosen.iter().map(|m| m.page).collect(); + by_page.sort_unstable(); + by_page.dedup(); + let page_list = pdfcraft_model::pages(doc); + let ctx = Ctx::new(HashSet::new()); + let mut regions: Vec = Vec::new(); + let mut pages = BTreeMap::new(); + let mut images = pdfcraft_cos::ImageBudget::new(doc); + // What was cut, each copy once, however many overlapping marks it lay under: the allowance + // for what stays is what the source holds less this. + let (mut cut_text, mut cut_raw) = (String::new(), String::new()); + for &pi in &by_page { + let page = page_list.get(pi).ok_or(RedactError::Unreadable(pi + 1))?; + let page_marks: Vec<&&Mark> = chosen.iter().filter(|m| m.page == pi).collect(); + let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); + let (_, data) = page_streams(doc, &page.dict, pi)?; + let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + // The interpreter's own removal path, on a scratch copy: what it takes out is what + // has to be gone. + let mut scratch = doc.clone(); + let mut sink = Report::default(); + let removed: Vec = { + let mut scope = Scope::new(&rects, Mode::Apply, &mut sink, pdfcraft_cos::DecodeBudget::page(), &mut images); + scope.removed = Some(Vec::new()); + process(&mut scratch, &mut scope, &data, &resources, Matrix::IDENTITY); + scope.removed.take().unwrap_or_default() + }; + let lines = joined(&removed); + let mut seen = HashSet::new(); + for line in lines.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))) { + cut_text.push('\u{1}'); + cut_text.push_str(&squash(&line.text)); + cut_raw.push('\u{1}'); + cut_raw.push_str(&squash(&latin1(&line.raw))); + } + let before = extract_page(doc, &page.dict, &ctx, &[]); + pages.insert( + pi, + PageSnap { + sha: Some(sha256(&data)), + text_ops: before.as_ref().ok().map(|e| e.glyph_ops), + undecodable_ops: before.as_ref().map_or(0, |e| e.undecodable_ops), + }, + ); + let mut k = 0; + for m in &page_marks { + for r in &m.rects { + let hits: Vec<&Removed> = removed.iter().filter(|x| x.rect == k).collect(); + let runs = hits.iter().map(|x| (x.raw.clone(), x.text.clone())).collect(); + let codes = hits.iter().map(|x| x.codes).sum(); + let joined = lines.iter().filter(|l| l.rect == k && l.pieces > 1).map(|l| (l.raw.clone(), l.text.clone())).collect(); + regions.push(RegionSnap { page: pi, region: *r, mark: m.obj, runs, joined, codes }); + k += 1; + } + } + } + // The document's text as the independent extractor reads it, for the allowance of + // strings that legitimately remain elsewhere. + let (mut src_text, mut src_raw) = (String::new(), String::new()); + let mut problems = Vec::new(); + for (i, p) in page_list.iter().enumerate() { + match extract_page(doc, &p.dict, &ctx, &[]) { + Ok(e) => { + src_text.push_str(&e.text); + src_raw.push_str(&e.raw); + } + Err(()) => { + problems.push(format!("page {} of the source can't be read, so strings left on it can't be told from removed ones", i + 1)) + } + } + } + let mut needles: Vec = Vec::new(); + for (ri, r) in regions.iter().enumerate() { + for (raw, text) in r.runs.iter().chain(&r.joined) { + let raw_s = latin1(raw); + for (raw_view, whole) in [(false, text.trim().to_string()), (true, raw_s)] { + let mut shown: Vec = Vec::new(); + if !squash(&whole).is_empty() { + shown.push(whole.clone()); + shown.extend(words(&whole).into_iter().filter(|w| *w != whole).map(str::to_string)); + } + for s in shown { + add_needle(&mut needles, raw_view, s, Some(ri)); + } + } + } + } + for (src, cut, raw) in [(&src_text, &cut_text, false), (&src_raw, &cut_raw, true)] { + match (super::sweep::key_counts(&needles, src, raw, &ctx), super::sweep::key_counts(&needles, cut, raw, &ctx)) { + (Some(source), Some(removed)) => { + let removed: BTreeMap = removed.into_iter().collect(); + for (i, total) in source { + if let Some(n) = needles.get_mut(i) { + n.allowance = total.saturating_sub(removed.get(&i).copied().unwrap_or(0)); + } + } + } + _ => problems.push("the source text allowances could not be counted in full".into()), + } + } + // The names the removed text legitimately sits in (a redacted word that is also a + // `/Subtype`, a `/Filter`, a `/BaseFont`): names are structure redaction never edits, + // so the output may hold the input's own matches, and only a surplus is a survivor. + let names = all_names(doc); + let name_allow: Vec> = needles.iter().map(|n| count_name_matches(&names, n, &ctx)).collect(); + if name_allow.iter().any(Option::is_none) { + problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + for (n, allow) in needles.iter_mut().zip(name_allow) { + n.name_allowance = allow.unwrap_or(0); + } + Ok(Snapshot { regions, pages, needles, problems, src: (src_text, src_raw), names, overlay: HashSet::new() }) + } + + /// Tell the proof which streams this run created to draw the redaction boxes and labels + /// (`created`: references into `doc`, the document after `apply`). Call it before + /// [`prove`](Snapshot::prove) / [`prove_saved`](Snapshot::prove_saved). + /// + /// Those streams paint over the regions on purpose and may show label text, so the proof + /// leaves them out of the text sweep and the paint check. They are recognised by the digest + /// of their decoded content (a save renumbers objects, so references would not survive it), + /// and *only* those: a stream that merely carries a `/PCMark /Redaction` key in the input is + /// not exempt. Streams that can't be read here are an error; nothing is then exempted. + pub fn set_overlay_streams(&mut self, doc: &Document, created: &[ObjRef]) -> Result<(), RedactError> { + let ctx = Ctx::new(HashSet::new()); + let mut digests = HashSet::new(); + for r in created { + let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; + let Object::Stream(s) = &*obj else { continue }; + let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + digests.insert(sha256(&[data])); + } + self.overlay = digests; + Ok(()) + } + + /// Also look for these strings in the output: text that left the document by another route + /// than a page's glyphs (a removed annotation's `/Contents` or `/RC`, a form field's value). + /// Each is searched whole, on every surface. A string that also legitimately remains as page + /// text is counted on the page-content surfaces only (as for the removed glyph runs). + /// Strings that are blank or shorter than the minimum needle are ignored. + pub fn add_removed_strings(&mut self, strings: &[String]) { + for s in strings { + let text = s.trim(); + let key = squash(text); + if key.chars().count() < MIN_NEEDLE { + continue; + } + let known = self.needles.iter().any(|n| !n.raw_view && n.key == key); + // No owner: a survivor fails the proof through its surface; no region carries it. + add_needle(&mut self.needles, false, text.to_string(), None); + if !known { + let ctx = Ctx::new(HashSet::new()); + let matches = self.needles.last().map(|n| count_name_matches(&self.names, n, &ctx)); + if let Some(n) = self.needles.last_mut() { + n.allowance = count(&self.src.0, &n.key); + n.name_allowance = matches.flatten().unwrap_or(0); + } + if matches == Some(None) { + self.problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + } + } + } + + /// Serialize `doc` the way a save would and sweep the result. The output is re-opened with + /// `password`, the credentials the caller opened the document with (`None` tries the empty + /// password): a save of an encrypted document is itself encrypted, and a proof that could + /// not read its own subject would prove nothing. + pub fn prove(&self, doc: &Document, password: Option<&str>, opts: &ProofOptions) -> Proof { + match write_full(doc, &SaveOptions::default()) { + Ok(bytes) => self.prove_inner(&bytes, Some(doc), password, opts), + Err(e) => { + let mut p = self.prove_inner(&[], None, password, opts); + p.failures.push(format!("the document could not be serialized for the sweep: {e}")); + p + } + } + } + + /// Sweep an already saved file (for example an incrementally saved one, whose earlier + /// revision still holds the original content). The file is opened with `password`, the + /// credentials it is read with. + pub fn prove_saved(&self, bytes: &[u8], password: Option<&str>, opts: &ProofOptions) -> Proof { + self.prove_inner(bytes, None, password, opts) + } + + fn prove_inner(&self, bytes: &[u8], live: Option<&Document>, password: Option<&str>, opts: &ProofOptions) -> Proof { + let mut acc = Acc::new(Ctx::new(self.overlay.clone())); + let mut failures: Vec = Vec::new(); + for p in &self.problems { + acc.problem(Surface::ExtractedText, p.clone()); + } + // Always the caller's credentials, never unconditionally the empty password: a + // document whose user password is not empty must be provable with that password, and + // must fail closed (an unswept output, never a false pass) without it. + let parsed = if bytes.is_empty() { None } else { Document::open_with_password(Arc::new(bytes.to_vec()), password).ok() }; + match &parsed { + Some(out) => sweep_doc(&self.needles, out, bytes, 0, "", &mut acc), + None => { + let why = "the output could not be read back".to_string(); + for s in SURFACES { + acc.problem(s, why.clone()); + } + failures.push(why); + } + } + if live.is_some_and(|d| !d.full_save_required()) { + acc.survivor(Surface::Revisions, "the document would be saved as an incremental update, keeping the original revision".into(), None); + } + + // The non-text content under each region, on the output. + let (findings, after) = match &parsed { + Some(out) => self.check_output(out, &acc.ctx), + None => Default::default(), + }; + let mut entries = Vec::with_capacity(self.regions.len()); + for (i, r) in self.regions.iter().enumerate() { + let (entry, failure) = self.entry(i, r, &findings, &after, &acc.failed_needles, opts); + failures.extend(failure); + entries.push(entry); + } + let surfaces = acc.finish(); + for s in &surfaces { + if s.survivors > 0 { + failures.push(format!("{:?}: {} survivor(s) at {}", s.surface, s.survivors, s.locations.join(", "))); + } + for p in &s.problems { + failures.push(format!("{:?} could not be swept: {p}", s.surface)); + } + } + Proof { entries, surfaces, failures } + } + + /// The non-text content under each region of the output (what still paints or shows there), + /// and per page the digest and glyph-operator count of what `apply` left. + fn check_output(&self, out: &Document, ctx: &Ctx) -> (HashMap>, PagesAfter) { + let mut findings: HashMap> = HashMap::new(); + let mut after: PagesAfter = BTreeMap::new(); + let out_pages = pdfcraft_model::pages(out); + for &pi in self.pages.keys() { + let idx: Vec = (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| r.page == pi)).collect(); + let Some(page) = out_pages.get(pi) else { + for &i in &idx { + findings.entry(i).or_default().push(Finding::Missing); + } + continue; + }; + let rects: Vec<[f64; 4]> = idx.iter().filter_map(|&i| self.regions.get(i).map(|r| r.region)).collect(); + let (sha, ops, shown) = page_after(out, &page.dict, ctx, &rects); + after.insert(pi, (sha, ops)); + match shown { + Some(shown) => { + for (k, n) in shown.into_iter().enumerate() { + if let (Some(&i), true) = (idx.get(k), n > 0) { + findings.entry(i).or_default().push(Finding::Text(n)); + } + } + } + None => { + for &i in &idx { + findings + .entry(i) + .or_default() + .push(Finding::Unverifiable("the page text can't be placed, so what still shows under the region can't be checked")); + } + } + } + for (k, f) in check_geometry(out, &page.dict, &rects, ctx) { + if let Some(&i) = idx.get(k) { + findings.entry(i).or_default().push(f); + } + } + } + (findings, after) + } + + /// The manifest entry of region `i`, and why it failed when it did. + fn entry( + &self, + i: usize, + r: &RegionSnap, + findings: &HashMap>, + after: &PagesAfter, + failed_needles: &HashSet, + opts: &ProofOptions, + ) -> (Excision, Option) { + let mut failure = None; + let snap = self.pages.get(&r.page); + let (sha_after, ops_after) = after.get(&r.page).cloned().unwrap_or((None, None)); + let sha_before = snap.and_then(|s| s.sha.clone()); + let owned: Vec<&Needle> = self.needles.iter().filter(|n| n.owners.contains(&i)).collect(); + let survived = self.needles.iter().enumerate().filter(|(ni, n)| n.owners.contains(&i) && failed_needles.contains(ni)).count(); + let mut detail: Vec = Vec::new(); + let mut status = if r.runs.is_empty() { Status::VerifiedNoTextInRegion } else { Status::Verified }; + let fs = findings.get(&i).map(Vec::as_slice).unwrap_or_default(); + if survived > 0 { + status = Status::Failed; + detail.push(format!("{survived} removed string(s) still found")); + } + for f in fs { + match f { + Finding::Text(n) => detail.push(format!("{n} glyph(s) still show inside the region (placed independently of the redaction)")), + Finding::Image => detail.push("an image still carries pixels under the region".into()), + Finding::Path => detail.push("a path still paints inside the region".into()), + Finding::Shading => detail.push("a shading still paints inside the region".into()), + Finding::Missing => detail.push("the page is missing from the output".into()), + Finding::Unverifiable(why) => detail.push((*why).to_string()), + } + if !matches!(f, Finding::Unverifiable(_)) { + status = Status::Failed; + } else if status != Status::Failed { + status = Status::Unverifiable; + } + } + if sha_before.is_none() || sha_after.is_none() { + status = Status::Failed; + detail.push("the page content can't be read on one side, so it can't be compared".into()); + } else if !r.runs.is_empty() && sha_before == sha_after { + status = Status::Failed; + detail.push("glyphs were removed but the page content is unchanged".into()); + } + if r.runs.is_empty() && snap.is_some_and(|s| s.undecodable_ops > 0) && status != Status::Failed { + status = Status::Unverifiable; + detail.push("the page draws text that can't be decoded, so \"no text in the region\" can't be checked".into()); + } + let ambiguous = owned.iter().filter(|n| n.allowance > 0).count(); + if ambiguous > 0 { + detail.push(format!("{ambiguous} string(s) also remain in retained content; only a surplus over those copies counts")); + } + if matches!(status, Status::Failed | Status::Unverifiable) { + failure = Some(format!("page {} region {:?}: {}", r.page + 1, r.region, detail.join("; "))); + } + let entry = Excision { + page: r.page, + region: r.region, + mark: r.mark, + status, + detail: detail.join("; "), + runs_removed: r.runs.len(), + codes_removed: r.codes, + page_sha256_after: sha_after, + text_ops_before: snap.and_then(|s| s.text_ops), + text_ops_after: ops_after, + removed_text: opts.include_plaintext.then(|| r.runs.iter().map(|(_, t)| t.clone()).collect()), + }; + (entry, failure) + } +} + +/// Per page: the digest of its content in the output and its glyph-operator count. +type PagesAfter = BTreeMap, Option)>; + +/// Every name of the document (the values, wherever an object holds them), with how often +/// each occurs. +fn all_names(doc: &Document) -> BTreeMap, usize> { + let mut names: BTreeMap, usize> = BTreeMap::new(); + for num in doc.object_numbers() { + if let Ok(obj) = doc.try_get(num) { + walk_strings(&obj, 0, &mut |o| { + if let Object::Name(n) = o { + let seen = names.entry(n.clone()).or_default(); + *seen = seen.saturating_add(1); + } + }); + } + } + names +} + +/// How many of `names` carry `needle`, by the same search the sweep runs on a name. `None` +/// when the search work ran out: an unknown allowance must not read as "none allowed", so the +/// caller reports that (and sweeps names with no allowance). +fn count_name_matches(names: &BTreeMap, usize>, needle: &Needle, ctx: &Ctx) -> Option { + if needle.short() { + return Some(0); + } + let mut total = 0usize; + for (name, seen) in names { + if !bytes_hits(std::slice::from_ref(needle), name, MIN_NEEDLE, ctx)?.is_empty() { + total = total.saturating_add(*seen); + } + } + Some(total) +} + +/// The page's content digest and glyph-operator count in the output, and per region the glyphs +/// still placed inside it (`None` when the page's text can't be placed). The digest covers the +/// page's content as `apply` left it, the overlay streams included. +pub(super) fn page_after(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> (Option, Option, Option>) { + let sha = page_streams(doc, page, 0).ok().map(|(_, d)| sha256(&d)); + let e = extract_page(doc, page, ctx, regions).ok(); + (sha, e.as_ref().map(|e| e.glyph_ops), e.map(|e| e.shown_in)) +} + +/// One text operator's removed runs under one region, as a reader takes them: joined. +struct Line { + rect: usize, + op: usize, + at: (usize, usize), + raw: Vec, + text: String, + pieces: usize, +} + +/// The runs of `removed` joined by text operator and region, in the order they were shown. +fn joined(removed: &[Removed]) -> Vec { + let mut lines: Vec = Vec::new(); + let mut found: HashMap<(usize, usize), usize> = HashMap::new(); + for r in removed { + let line = found.get(&(r.op, r.rect)).copied().and_then(|i| lines.get_mut(i)); + match line { + Some(l) => { + l.raw.extend_from_slice(&r.raw); + l.text.push_str(&r.text); + l.pieces += 1; + } + None => { + found.insert((r.op, r.rect), lines.len()); + lines.push(Line { rect: r.rect, op: r.op, at: r.at, raw: r.raw.clone(), text: r.text.clone(), pieces: 1 }); + } + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(rect: usize, op: usize, place: usize, text: &str) -> Removed { + Removed { rect, raw: text.as_bytes().to_vec(), text: text.to_string(), codes: text.len(), op, at: (place, 0) } + } + + #[test] + fn a_word_split_across_a_tj_array_is_one_line_for_each_region() { + let lines = joined(&[run(0, 1, 0, "sec"), run(1, 1, 0, "sec"), run(0, 1, 1, "ret"), run(1, 1, 1, "ret"), run(0, 2, 0, "other")]); + let shown: Vec<(usize, usize, &str, usize)> = lines.iter().map(|l| (l.rect, l.op, l.text.as_str(), l.pieces)).collect(); + assert_eq!(shown, [(0, 1, "secret", 2), (1, 1, "secret", 2), (0, 2, "other", 1)]); + } + + #[test] + fn overlapping_marks_are_one_copy_and_two_copies_stay_two() { + // Under two marks at once: the same operator, place and bytes, so one copy was cut. + let both = joined(&[run(0, 1, 0, "word"), run(1, 1, 0, "word")]); + let mut seen = HashSet::new(); + assert_eq!(both.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 1); + // The same word twice in one array, one under each mark: two copies were cut. + let apart = joined(&[run(0, 1, 0, "word"), run(1, 1, 2, "word")]); + let mut seen = HashSet::new(); + assert_eq!(apart.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 2); + } + + #[test] + fn a_spent_decode_budget_is_told_apart() { + assert!(!Ctx::new(HashSet::new()).decode_spent()); + assert!(Ctx::with_decode_left(0).decode_spent()); + } +} diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs new file mode 100644 index 0000000000..73b44c005d --- /dev/null +++ b/crates/redact/src/verify/sweep.rs @@ -0,0 +1,759 @@ +//! Sweeping the serialized output for the removed strings, on every surface a reader could +//! recover them from. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; + +use super::extract::extract_page; +use super::util::{Ctx, decode_text, find, image_codec, spellings, squash, survivor_encodings, xml_unescape}; +use super::{SURFACES, Surface, SurfaceReport, Verdict}; +use crate::verify::limits::{MAX_NESTING, MAX_WALK, MIN_NEEDLE, MIN_STRUCTURED_NEEDLE}; + +/// A string the sweep looks for. +pub(super) struct Needle { + /// The code bytes as Latin-1 text (the "raw view" of a page) rather than decoded text. + pub(super) raw_view: bool, + /// As removed, whitespace included (for the byte encodings). + pub(super) shown: String, + /// `shown` without whitespace (for page text, which is compared whitespace-blind). + pub(super) key: String, + /// What the raw file may spell it as. + pub(super) bytes: Vec>, + pub(super) owners: BTreeSet, + /// Occurrences that legitimately remain in content that was not redacted. The surfaces that + /// hold page content (the page text, the raw file, decoded streams) count occurrences over + /// the whole document and fail on a surplus; every other surface (strings, Info, XMP, + /// attachments) is swept with no allowance. + pub(super) allowance: usize, + /// Names of the input that carry this needle (a redacted word that is also a `/Subtype`, a + /// `/Filter`, a `/BaseFont`). Names are structure redaction never edits, so the name surface + /// is swept against the input's own matches: the output may hold as many as the input did, + /// and only a surplus is a survivor. + pub(super) name_allowance: usize, +} + +impl Needle { + pub(super) fn short(&self) -> bool { + self.key.chars().count() < MIN_NEEDLE + } + + /// Long enough to be looked for in bytes (`min`: the shortest the haystack allows). + pub(super) fn at_least(&self, min: usize) -> bool { + self.key.chars().count() >= min + } +} + +pub(super) fn add_needle(set: &mut Vec, raw_view: bool, shown: String, owner: Option) { + let key = squash(&shown); + if key.is_empty() { + return; + } + if let Some(n) = set.iter_mut().find(|n| n.raw_view == raw_view && n.key == key) { + n.owners.extend(owner); + return; + } + let bytes = if raw_view { + // The code bytes themselves (Latin-1 round-trips them). + let raw: Vec = shown.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).collect(); + spellings([raw]) + } else { + survivor_encodings(&shown) + }; + set.push(Needle { raw_view, shown, key, bytes, owners: owner.into_iter().collect(), allowance: 0, name_allowance: 0 }); +} + +pub(super) struct Acc { + pub(super) reps: BTreeMap, + pub(super) failed_needles: HashSet, + pub(super) ctx: Ctx, +} + +impl Acc { + pub(super) fn new(ctx: Ctx) -> Self { + let mut a = Acc { reps: BTreeMap::new(), failed_needles: HashSet::new(), ctx }; + for s in SURFACES { + a.rep(s); + } + a + } + + pub(super) fn rep(&mut self, surface: Surface) -> &mut SurfaceReport { + self.reps.entry(surface).or_insert_with(|| SurfaceReport { + surface, + verdict: Verdict::Absent, + scanned: 0, + raw_only: 0, + survivors: 0, + locations: Vec::new(), + problems: Vec::new(), + }) + } + + pub(super) fn scanned(&mut self, s: Surface) { + self.rep(s).scanned += 1; + } + + pub(super) fn raw_only(&mut self, s: Surface) { + self.rep(s).raw_only += 1; + } + + pub(super) fn survivor(&mut self, s: Surface, loc: String, needle: Option) { + self.survivors(s, loc, needle, 1); + } + + /// `count` survivors at one place. + pub(super) fn survivors(&mut self, s: Surface, loc: String, needle: Option, count: usize) { + if let Some(n) = needle { + self.failed_needles.insert(n); + } + let r = self.rep(s); + r.survivors = r.survivors.saturating_add(count); + if !r.locations.contains(&loc) && r.locations.len() < 64 { + r.locations.push(loc); + } + } + + pub(super) fn problem(&mut self, s: Surface, msg: String) { + let r = self.rep(s); + if r.problems.len() < 64 { + r.problems.push(msg); + } + } + + /// The needles (at least `min` long) found in `hay`; running out of search work is a + /// problem of `surface`. + pub(super) fn bytes(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec { + bytes_hits(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// How often each needle (at least `min` long) occurs in `hay`, as `(needle, count)`. + pub(super) fn counts(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec<(usize, usize)> { + bytes_counts(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// Survivors and problems so far, over all surfaces. + pub(super) fn totals(&self) -> (usize, usize) { + (self.reps.values().map(|r| r.survivors).sum(), self.reps.values().map(|r| r.problems.len()).sum()) + } + + pub(super) fn finish(mut self) -> Vec { + for r in self.reps.values_mut() { + r.verdict = if r.survivors > 0 { + Verdict::Survivor + } else if !r.problems.is_empty() { + Verdict::Unswept + } else if r.scanned + r.raw_only == 0 { + Verdict::Absent + } else { + Verdict::Clean + }; + } + self.reps.into_values().collect() + } +} + +/// Occurrences of needles on one surface, per place, so that what legitimately remains +/// (a needle's allowance) is taken off the document-wide total once, not once per place. +#[derive(Default)] +pub(super) struct Tally(BTreeMap>); + +impl Tally { + pub(super) fn add(&mut self, needle: usize, loc: &str, count: usize) { + if count > 0 { + self.0.entry(needle).or_default().push((loc.to_string(), count)); + } + } + + /// Report what is left of each needle's occurrences after its allowance (none inside an + /// attachment, where nothing legitimately remains). + pub(super) fn settle(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.allowance) + } + + /// The name surface: the input's own name matches are the allowance (redaction never edits + /// names), so what is left is a surplus of matches over the input. + pub(super) fn settle_names(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.name_allowance) + } + + fn settle_with(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc, allow_of: impl Fn(&Needle) -> usize) { + for (ni, places) in self.0 { + let mut allow = if depth == 0 { needles.get(ni).map_or(0, &allow_of) } else { 0 }; + for (loc, count) in places { + let used = count.min(allow); + allow -= used; + if count > used { + acc.survivors(surface, loc, Some(ni), count - used); + } + } + } + } +} + +/// Search all encodings in one bounded pass, including overlapping matches. +fn pattern_matches<'a>( + patterns: impl IntoIterator, + hay: &[u8], + ctx: &Ctx, + mut found: impl FnMut(usize, usize, usize) -> bool, +) -> Option<()> { + const MAX_PATTERNS: usize = 100_000; + const MAX_PATTERN_BYTES: usize = 16 << 20; + const MAX_MATCHES: usize = 1 << 20; + if !ctx.charge(hay.len()) { + return None; + } + let mut map: BTreeMap, Vec> = BTreeMap::new(); + let (mut bytes, mut links) = (0usize, 0usize); + for (owner, pattern) in patterns { + if pattern.is_empty() { + continue; + } + if !ctx.charge(pattern.len().saturating_add(1)) { + return None; + } + if !map.contains_key(pattern) { + bytes = bytes.checked_add(pattern.len())?; + if bytes > MAX_PATTERN_BYTES || map.len() >= MAX_PATTERNS { + return None; + } + } + let owners = map.entry(pattern.to_vec()).or_default(); + if owners.last() != Some(&owner) { + links = links.checked_add(1)?; + if links > MAX_PATTERNS { + return None; + } + owners.push(owner); + } + } + if map.is_empty() { + return Some(()); + } + let matcher = aho_corasick::AhoCorasick::new(map.keys()).ok()?; + let owners: Vec> = map.into_values().collect(); + let mut matches = 0usize; + for hit in matcher.find_overlapping_iter(hay) { + for &owner in owners.get(hit.pattern().as_usize())? { + matches = matches.checked_add(1)?; + if matches > MAX_MATCHES || !ctx.charge(1) || !found(owner, hit.start(), hit.end()) { + return None; + } + } + } + Some(()) +} + +/// Count normalized text keys in one pass, with the same non-overlap rule as `str::matches`. +pub(super) fn key_counts(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx) -> Option> { + key_counts_from(needles, hay, raw, ctx, 0) +} + +/// [`key_counts`] of the needles at least `min` long, so a one-letter needle can't use up the match +/// limits of a search it takes no part in. +fn key_counts_from(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx, min: usize) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.raw_view == raw && n.at_least(min)).map(|(i, n)| (i, n.key.as_bytes())); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay.as_bytes(), ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut hits)| { + hits.sort_unstable(); + let (mut count, mut end) = (0, 0); + for (a, b) in hits { + if a >= end { + count += 1; + end = b; + } + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn bytes_hits(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut found = BTreeSet::new(); + pattern_matches(patterns, hay, ctx, |i, _, _| { + found.insert(i); + true + })?; + Some(found.into_iter().collect()) +} + +/// A larger count is incomplete evidence, never a silently truncated count. +pub(super) const MAX_COUNTED: usize = 1 << 16; + +pub(super) fn bytes_counts(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay, ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut spans)| { + spans.sort_unstable(); + spans.dedup(); + let (mut count, mut end) = (0usize, 0usize); + for (a, b) in spans { + if count == 0 || a >= end { + count += 1; + } + end = end.max(b); + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn string_hits(needles: &[Needle], value: &PdfString, ctx: &Ctx) -> Option> { + let text = value.to_text(); + let key = squash(&text); + if !ctx.charge(value.bytes.len().saturating_add(key.len())) { + return None; + } + let mut found: BTreeSet = bytes_hits(needles, &value.bytes, MIN_NEEDLE, ctx)?.into_iter().collect(); + let patterns = needles.iter().enumerate().filter(|(_, n)| !n.raw_view && !n.short()).map(|(i, n)| (i, n.key.as_bytes())); + pattern_matches(patterns, key.as_bytes(), ctx, |i, _, _| { + found.insert(i); + true + })?; + let tokens: BTreeSet<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|s| !s.is_empty()).collect(); + for (i, needle) in needles.iter().enumerate().filter(|(_, n)| n.short() && n.allowance == 0 && !n.raw_view) { + if !ctx.charge(needle.shown.len().saturating_add(1)) { + return None; + } + if tokens.contains(needle.shown.as_str()) { + found.insert(i); + } + } + Some(found.into_iter().collect()) +} + +/// Streams whose data is binary rather than text: images, embedded font programs, colour +/// profiles, cross-reference and object streams (the objects in the latter are swept as +/// objects). Their decoded data is swept like any other stream's; the pixels of an image codec +/// stay undecoded (text that exists only as pixels is not text), and one that won't decode (or is +/// past the cap) is searched as stored bytes only. A short needle "found" in stored bytes is +/// chance. +pub(super) fn binary_stream(d: &Dict) -> bool { + let subtype = d.name(b"Subtype"); + subtype == Some(b"Image".as_slice()) + || matches!(d.name(b"Type"), Some(b"XRef" | b"ObjStm")) + || matches!(subtype, Some(b"Type1C" | b"CIDFontType0C" | b"OpenType")) + || [b"Length1".as_slice(), b"Length2", b"Length3"].iter().any(|k| d.contains(k)) + || (d.contains(b"N") && !d.contains(b"Type") && subtype.is_none()) +} + +/// What a file identifier (`/ID` of the trailer) is spelled as in the file. +pub(super) fn id_spellings(doc: &Document) -> Vec> { + let Some(Object::Array(ids)) = doc.trailer().get(b"ID").map(|o| doc.resolve(o)).as_deref().cloned() else { return Vec::new() }; + ids.iter().filter_map(Object::as_string).flat_map(|s| spellings([s.bytes.to_vec()])).collect() +} + +/// Visit `o` and everything nested in it. False when something lay deeper than [`MAX_WALK`] +/// and was not visited: the caller reports that as unswept. +pub(super) fn walk_strings(o: &Object, depth: usize, f: &mut impl FnMut(&Object)) -> bool { + if depth > MAX_WALK { + return false; + } + f(o); + // Every child is visited even after one is too deep: nothing is skipped silently. + let mut ok = true; + match o { + Object::Array(a) => a.iter().for_each(|x| ok &= walk_strings(x, depth + 1, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + _ => {} + } + ok +} + +/// Sweep a document (the output, or an attachment) on every surface. +pub(super) fn sweep_doc(needles: &[Needle], doc: &Document, bytes: &[u8], depth: usize, tag: &str, acc: &mut Acc) { + let (embedded, overlay_raw) = sweep_objects(needles, doc, depth, tag, acc); + sweep_file(needles, doc, bytes, overlay_raw, depth, tag, acc); + sweep_pages(needles, doc, depth, tag, acc); + sweep_info(needles, doc, tag, acc); + sweep_xmp(needles, doc, tag, acc); + + // Revision structure: one revision, no earlier content kept. + acc.scanned(Surface::Revisions); + let revs = doc.revisions().len(); + if revs > 1 || doc.trailer().contains(b"Prev") { + acc.survivor(Surface::Revisions, format!("{tag}{revs} revisions: earlier ones keep the original content"), None); + } + + sweep_attachments(needles, doc, &embedded, depth, tag, acc); +} + +/// Every object: strings, names and stream data (orphans included). Returns the embedded file +/// streams found, and the raw bytes of the overlay streams (to leave out of the raw sweep). +fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) -> (BTreeSet, Vec>) { + let mut embedded: BTreeSet = BTreeSet::new(); + let mut overlay_raw: Vec> = Vec::new(); + let mut in_streams = Tally::default(); + let mut in_names = Tally::default(); + let lookup = lookup_streams(doc); + for num in doc.object_numbers() { + let loc = format!("{tag}object {num}"); + let obj = match doc.try_get(num) { + Ok(o) => o, + Err(_) => { + acc.problem(Surface::ObjectStrings, format!("{loc} can't be read")); + continue; + } + }; + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + // The file identifier is random bytes, never text: a short needle must not "read" it. + let without_id = match &*obj { + Object::Stream(s) if s.dict.name(b"Type") == Some(b"XRef") => { + let mut d = s.dict.clone(); + d.remove(b"ID"); + Some(Object::Dict(d)) + } + _ => None, + }; + let complete = walk_strings(without_id.as_ref().unwrap_or(&obj), 0, &mut |o| match o { + Object::String(s) => match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + }, + Object::Name(n) => match bytes_hits(needles, n, MIN_NEEDLE, ctx) { + // A name the input had its own matches in is structure, not a leftover copy: + // the matches are tallied and settled against the input's count. + Some(h) => h.into_iter().for_each(|ni| in_names.add(ni, &loc, 1)), + None => spent = true, + }, + _ => {} + }); + if !complete { + acc.problem(Surface::ObjectStrings, format!("{loc} is nested too deeply to sweep in full")); + } + if spent { + acc.problem(Surface::ObjectStrings, "the search work limit was reached".into()); + } + acc.scanned(Surface::ObjectStrings); + for n in hits { + acc.survivor(Surface::ObjectStrings, loc.clone(), Some(n)); + } + // File specifications, wherever they sit (a name tree keeps them as separate objects, + // annotations and collections may inline them). + walk_strings(&obj, 0, &mut |o| { + if let Object::Dict(d) = o + && let Some(Object::Dict(ef)) = d.get(b"EF") + { + embedded.extend(ef.iter().filter_map(|(_, v)| v.as_ref()).map(|r| r.num)); + } + }); + if let Object::Stream(s) = &*obj { + if s.dict.name(b"Type") == Some(b"EmbeddedFile") { + embedded.insert(num); + } + acc.scanned(Surface::DecodedStreams); + if binary_stream(&s.dict) && image_codec(&s.dict) { + // The pixels of an image codec are still encoded and are not text: their stored + // bytes are searched for whole strings only. + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } else if binary_stream(&s.dict) { + // A compressed copy of the removed text can hide in a font program or colour + // profile (with /Length1 in its dict, its stored bytes hold no plaintext): the + // stored bytes and the decoded data are both searched. Such a stream that won't + // decode (damaged, or past the cap) is searched as stored bytes only and counted + // as raw only: it holds no page content, and redaction never touches it. A + // cross-reference or object stream that won't decode is a problem of the proof. + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + match acc.ctx.decode(s) { + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + Err(()) if acc.ctx.decode_spent() || matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm")) => { + acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")); + } + Err(()) => acc.raw_only(Surface::DecodedStreams), + } + } else { + match acc.ctx.decode(s) { + // The boxes and labels this run drew are not text to look for. + Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + // Colour-space lookup data and function samples are data tables, not content. + Err(()) if !acc.ctx.decode_spent() && (lookup.contains(&num) || s.dict.contains(b"FunctionType")) => { + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } + Err(()) => acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")), + } + } + } + } + in_names.settle_names(Surface::ObjectStrings, needles, depth, acc); + in_streams.settle(Surface::DecodedStreams, needles, depth, acc); + (embedded, overlay_raw) +} + +/// The streams that hold the lookup table of an `/Indexed` colour space (object numbers). +fn lookup_streams(doc: &Document) -> BTreeSet { + let mut out = BTreeSet::new(); + for num in doc.object_numbers() { + let Ok(obj) = doc.try_get(num) else { continue }; + walk_strings(&obj, 0, &mut |o| { + if let Object::Array(a) = o + && matches!(a.first(), Some(Object::Name(n)) if n == b"Indexed" || n == b"I") + && let Some(Object::Ref(r)) = a.get(3) + { + out.insert(r.num); + } + }); + } + out +} + +/// The raw bytes of the whole file, the overlay streams (stored as they are, or not at all) and +/// the file identifier (random bytes, regenerated on every write) aside. +fn sweep_file(needles: &[Needle], doc: &Document, bytes: &[u8], overlay_raw: Vec>, depth: usize, tag: &str, acc: &mut Acc) { + acc.scanned(Surface::RawBytes); + let mut mask = overlay_raw; + mask.extend(id_spellings(doc)); + let masked = mask_all(bytes, &mask, &acc.ctx); + let mut in_file = Tally::default(); + for (n, c) in acc.counts(Surface::RawBytes, needles, masked.as_deref().unwrap_or(bytes), MIN_STRUCTURED_NEEDLE) { + in_file.add(n, &format!("{tag}file"), c); + } + in_file.settle(Surface::RawBytes, needles, depth, acc); + if masked.is_none() && !mask.is_empty() { + acc.problem(Surface::RawBytes, "the search work limit was reached".into()); + } +} + +/// Embedded files, PDF attachments swept in full. +fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet, depth: usize, tag: &str, acc: &mut Acc) { + for &num in embedded { + let loc = format!("{tag}attachment object {num}"); + acc.scanned(Surface::EmbeddedFiles); + let obj = doc.get(ObjRef::new(num, doc.generation(num))); + let Object::Stream(s) = &*obj else { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is not a stream")); + continue; + }; + let payload = match acc.ctx.decode(s) { + Ok(p) => p, + Err(()) => { + acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); + continue; + } + }; + sweep_payload(needles, &payload, depth, &loc, acc); + } +} + +/// `bytes` with every occurrence of `pats` zeroed (`None` when the work limit ran out, or +/// there is nothing to mask). +pub(super) fn mask_all(bytes: &[u8], pats: &[Vec], ctx: &Ctx) -> Option> { + if pats.iter().all(Vec::is_empty) { + return None; + } + let mut out = bytes.to_vec(); + for p in pats.iter().filter(|p| !p.is_empty()) { + if !ctx.charge(bytes.len()) { + return None; + } + let mut at = 0; + while let Some(w) = bytes.get(at..at + p.len()) { + if w == p.as_slice() { + if let Some(z) = out.get_mut(at..at + p.len()) { + z.fill(0); + } + at += p.len(); + } else { + at += 1; + } + } + } + Some(out) +} + +/// One attachment: a PDF is swept as a document, text is searched, containers are reported as +/// unswept (their content can't be seen), images are searched as raw bytes only. +pub(super) fn sweep_payload(needles: &[Needle], payload: &[u8], depth: usize, loc: &str, acc: &mut Acc) { + let head = payload.get(..1024).unwrap_or(payload); + let tag = format!("{loc} > "); + if find(head, b"%PDF-") { + if depth >= MAX_NESTING { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is nested too deep to sweep")); + return; + } + match Document::open(Arc::new(payload.to_vec())) { + Ok(inner) => { + let before = acc.totals(); + sweep_doc(needles, &inner, payload, depth + 1, &tag, acc); + // What the attachment's own sweep found counts against the attachment too. + let after = acc.totals(); + if after.0 > before.0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), None); + } + if after.1 > before.1 { + acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be swept in full")); + } + } + Err(_) => acc.problem(Surface::EmbeddedFiles, format!("{loc} is a PDF that can't be opened")), + } + return; + } + let starts = |sig: &[u8]| payload.starts_with(sig); + let container = starts(b"PK\x03\x04") + || starts(b"PK\x05\x06") + || starts(&[0x1F, 0x8B]) + || starts(b"BZh") + || starts(&[0xFD, b'7', b'z', b'X', b'Z', 0]) + || starts(&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C]) + || starts(b"Rar!") + || starts(&[0xD0, 0xCF, 0x11, 0xE0]) + || payload.get(257..262) == Some(b"ustar"); + if container { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is an archive or container that can't be opened")); + return; + } + let image = starts(&[0x89, b'P', b'N', b'G']) || starts(&[0xFF, 0xD8, 0xFF]) || starts(b"GIF8"); + if image { + acc.raw_only(Surface::EmbeddedFiles); + } + let min = if image { MIN_STRUCTURED_NEEDLE } else { MIN_NEEDLE }; + for n in acc.bytes(Surface::EmbeddedFiles, needles, payload, min) { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(n)); + } + if !image { + let key = squash(&decode_text(payload)); + match key_counts_from(needles, &key, false, &acc.ctx, MIN_NEEDLE) { + Some(counts) => { + for (i, count) in counts { + if count > 0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(i)); + } + } + } + None => acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be searched in full")), + } + } +} + +pub(super) fn sweep_pages(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) { + let mut shown = Tally::default(); + 'pages: for (i, p) in pdfcraft_model::pages(doc).iter().enumerate() { + acc.scanned(Surface::ExtractedText); + let loc = format!("{tag}page {}", i + 1); + let Ok(e) = extract_page(doc, &p.dict, &acc.ctx, &[]) else { + acc.problem(Surface::ExtractedText, format!("{loc} can't be read")); + continue; + }; + for (hay, raw) in [(&e.text, false), (&e.raw, true)] { + let Some(counts) = key_counts(needles, hay, raw, &acc.ctx) else { + acc.problem(Surface::ExtractedText, "the search work limit was reached".into()); + break 'pages; + }; + for (ni, count) in counts { + shown.add(ni, &loc, count); + } + } + } + // A surplus over what legitimately remains anywhere in the document is a survivor. The + // counts made before the work ran out are settled too: a survivor already seen is + // reported where it sits, not hidden behind the limit. + shown.settle(Surface::ExtractedText, needles, depth, acc); +} + +pub(super) fn sweep_info(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(info) = doc.trailer().get(b"Info").map(|i| doc.resolve(i)) else { return }; + let Some(d) = info.as_dict() else { + acc.problem(Surface::Info, format!("{tag}Info is not a dictionary")); + return; + }; + for (k, v) in d.iter() { + acc.scanned(Surface::Info); + let v = doc.resolve(v); + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + let complete = walk_strings(&v, 0, &mut |o| { + if let Object::String(s) = o { + match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + } + } + }); + if !complete || spent { + acc.problem(Surface::Info, format!("{tag}Info /{} can't be swept in full", String::from_utf8_lossy(k))); + } + for n in hits { + acc.survivor(Surface::Info, format!("{tag}Info /{}", String::from_utf8_lossy(k)), Some(n)); + } + } +} + +pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(root) = doc.root() else { return }; + let Some(meta) = doc.get(root).as_dict().and_then(|c| c.get(b"Metadata")).map(|m| doc.resolve(m)) else { return }; + acc.scanned(Surface::Xmp); + let loc = format!("{tag}XMP metadata"); + let Object::Stream(s) = &*meta else { + acc.problem(Surface::Xmp, format!("{loc} is not a stream")); + return; + }; + let Ok(data) = acc.ctx.decode(s) else { + acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); + return; + }; + let mut hits: BTreeSet = acc.bytes(Surface::Xmp, needles, &data, MIN_NEEDLE).into_iter().collect(); + let key = squash(&xml_unescape(&decode_text(&data))); + for (i, n) in needles.iter().enumerate() { + if n.at_least(MIN_NEEDLE) && !n.raw_view && key.contains(&n.key) { + hits.insert(i); + } + } + for n in hits { + acc.survivor(Surface::Xmp, loc.clone(), Some(n)); + } +} diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs new file mode 100644 index 0000000000..77c4309cdd --- /dev/null +++ b/crates/redact/src/verify/util.rs @@ -0,0 +1,301 @@ +//! Shared helpers of the proof: what one proof may spend, text and byte search, and the +//! spellings a string can have in a PDF. + +use std::cell::Cell; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::sync::Arc; + +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; +use sha2::{Digest, Sha256}; + +use crate::verify::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_NEEDLE}; + +/// What one proof may spend, and what it knows about the document it checks. +pub(super) struct Ctx { + /// Digests of the decoded content of the overlay streams (their boxes and labels are drawn + /// on purpose and are neither text to look for nor paint to flag). + pub(super) overlay: HashSet, + /// Decoded bytes of the proof so far: one allowance for the whole proof, failures charged + /// too (see [`Ctx::decode`]), so a stream referenced many times, or one that fails after + /// decoding part of itself, cannot decode again for free. + pub(super) decoded: Cell, + pub(super) work: Cell, +} + +impl Ctx { + pub(super) fn new(overlay: HashSet) -> Self { + Ctx { overlay, decoded: Cell::new(0), work: Cell::new(0) } + } + + /// Decode `s` strictly: a stream that is damaged, longer than [`MAX_STREAM`], or one more + /// than [`MAX_TOTAL_DECODED`] allows is an error (never silently truncated data). Every + /// attempt is charged to the cumulative allowance, failures included: a decode that fails + /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded + /// before the error — the work of finding out is not work a repeated reference gets for free. + pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); + let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); + let mut budget = DecodeBudget::new(cap, usize::MAX); + let result = budget.decode_strict(s); + // The attempt's own accounting covers a success (what it decoded), a stream cut short + // by corruption (what it decoded first) and one refused past the cap (the cap): charge + // it through, whichever way the attempt went. + let spent = cap.saturating_sub(budget.remaining()); + self.decoded.set(self.decoded.get().saturating_add(spent as u64)); + result.map_err(|_| ()) + } + + /// Is the cumulative decode allowance used up? A decode refused then says nothing about the + /// stream, so it must not pass for a stream that can't be decoded. + pub(super) fn decode_spent(&self) -> bool { + self.decoded.get() >= MAX_TOTAL_DECODED + } + + /// Spend `n` units of search work; false once [`MAX_WORK`] is used up. + pub(super) fn charge(&self, n: usize) -> bool { + let used = self.work.get().saturating_add(n as u64); + self.work.set(used); + used <= MAX_WORK + } + + /// Is this decoded content one of the overlay streams this run created? + pub(super) fn is_overlay(&self, data: &[u8]) -> bool { + !self.overlay.is_empty() && self.overlay.contains(&sha256(&[data])) + } + + /// (tests) A context whose cumulative decode allowance has `left` bytes left. + #[cfg(test)] + pub(super) fn with_decode_left(left: u64) -> Self { + Ctx { overlay: HashSet::new(), decoded: Cell::new(MAX_TOTAL_DECODED.saturating_sub(left)), work: Cell::new(0) } + } +} + +/// Follow `o` to its object; a reference that can't be loaded is an error (a missing object is +/// `Null` for [`Document::resolve`], which would read as "empty"). +pub(super) fn resolve_strict(doc: &Document, o: &Object) -> Result, ()> { + match o { + Object::Ref(r) => doc.try_get(r.num).map_err(|_| ()), + other => Ok(Arc::new(other.clone())), + } +} + +pub(super) fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +pub(super) fn sha256>(parts: &[T]) -> String { + let mut h = Sha256::new(); + for p in parts { + let p = p.as_ref(); + h.update((p.len() as u64).to_be_bytes()); + h.update(p); + } + hex(h.finalize().as_slice()) +} + +/// The text of `s` without whitespace (page text is compared whitespace-blind: a gap in a `TJ` +/// is a space in one extraction and nothing in another). +pub(super) fn squash(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}').collect() +} + +pub(super) fn latin1(b: &[u8]) -> String { + b.iter().map(|&c| char::from(c)).collect() +} + +/// The words of a run worth looking for on their own (a run that survives in pieces). +pub(super) fn words(s: &str) -> Vec<&str> { + s.split_whitespace().filter(|w| w.chars().count() >= MIN_NEEDLE).collect() +} + +pub(super) fn count(hay: &str, needle: &str) -> usize { + if needle.is_empty() { 0 } else { hay.matches(needle).count() } +} + +/// Is `pat` in `hay`? +pub(super) fn find(hay: &[u8], pat: &[u8]) -> bool { + find_from(hay, pat, 0).is_some() +} + +/// Where `pat` first occurs in `hay` at or after `from`. The pattern's prefix function is +/// built once and the haystack is walked once: the search is worst-case linear, so +/// adversarial input (a needle of one letter and a haystack full of it) can neither make it +/// quadratic nor spend more comparisons than the caller charges for. +pub(super) fn find_from(hay: &[u8], pat: &[u8], from: usize) -> Option { + let m = pat.len(); + if m == 0 { + return None; + } + // fail[i]: the longest proper prefix of `pat` that ends at `i` and is also a suffix there. + let mut fail = vec![0usize; m]; + let mut k = 0; + for (i, &c) in pat.iter().enumerate().skip(1) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + } + fail[i] = k; + } + let mut k = 0; + for (at, &c) in hay.iter().enumerate().skip(from) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + if k == m { + return Some(at + 1 - m); + } + } + } + None +} + +pub(super) fn escape_literal(b: &[u8], octal: bool) -> Vec { + let mut out = Vec::with_capacity(b.len() * 2); + for &c in b { + if octal { + out.extend(format!("\\{c:03o}").bytes()); + continue; + } + match c { + b'(' | b')' | b'\\' => out.extend([b'\\', c]), + b'\n' => out.extend(*b"\\n"), + b'\r' => out.extend(*b"\\r"), + b'\t' => out.extend(*b"\\t"), + _ => out.push(c), + } + } + out +} + +/// Every way `text` can be spelled in a PDF: UTF-8, UTF-16BE (with and without a byte order +/// mark) and UTF-16LE, each as raw bytes, as a hex string (either case) and as a backslash- +/// escaped literal string (minimal escapes, and octal for every byte). +pub(crate) fn survivor_encodings(text: &str) -> Vec> { + let utf16 = |be: bool, bom: bool| -> Vec { + let mut v = Vec::new(); + if bom { + v.extend(if be { [0xFE, 0xFF] } else { [0xFF, 0xFE] }); + } + for u in text.encode_utf16() { + v.extend(if be { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + let forms = [text.as_bytes().to_vec(), utf16(true, true), utf16(true, false), utf16(false, true), utf16(false, false)]; + spellings(forms) +} + +/// The spellings of a byte string (see [`survivor_encodings`]). +pub(super) fn spellings(forms: impl IntoIterator>) -> Vec> { + let mut out: Vec> = Vec::new(); + for f in forms { + if f.is_empty() { + continue; + } + let h = hex(&f); + out.push(h.to_uppercase().into_bytes()); + out.push(h.into_bytes()); + out.push(escape_literal(&f, false)); + out.push(escape_literal(&f, true)); + out.push(f); + } + let mut seen = HashSet::new(); + out.retain(|b| seen.insert(b.clone())); + out +} + +/// Text of a payload: UTF-16 when it has a byte order mark, else UTF-8 (lossy). +pub(super) fn decode_text(b: &[u8]) -> String { + let units = |be: bool, rest: &[u8]| -> String { + let u: Vec = rest.as_chunks::<2>().0.iter().map(|c| if be { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) }).collect(); + String::from_utf16_lossy(&u) + }; + match b { + [0xFE, 0xFF, rest @ ..] => units(true, rest), + [0xFF, 0xFE, rest @ ..] => units(false, rest), + _ => String::from_utf8_lossy(b).into_owned(), + } +} + +pub(super) fn xml_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(rest.get(..at).unwrap_or_default()); + rest = rest.get(at..).unwrap_or_default(); + match xml_entity(rest) { + Some((c, len)) => { + out.push(c); + rest = rest.get(len..).unwrap_or_default(); + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + out.push_str(rest); + out +} + +/// The entity starting at `&`: the character it reads as, and how far it runs (through the +/// closing `;`). A numeric reference may carry any number of leading zeros, so its digits +/// are padded without a bound; more significant digits than a character value can hold, or +/// a missing `;`, is not an entity and the `&` stays as it is. Walking only the digits the +/// reference itself holds keeps the decode linear in the input. +fn xml_entity(s: &str) -> Option<(char, usize)> { + for (name, c) in [("&", '&'), ("<", '<'), (">", '>'), (""", '"'), ("'", '\'')] { + if s.starts_with(name) { + return Some((c, name.len())); + } + } + let body = s.strip_prefix("&#")?; + let (radix, digits) = match body.as_bytes().first() { + Some(b'x' | b'X') => (16u32, &body[1..]), + _ => (10, body), + }; + let mut value: u32 = 0; + let mut digits_held = false; + let mut end = None; + for (i, c) in digits.char_indices() { + match c.to_digit(radix) { + Some(d) => { + digits_held = true; + // Leading zeros only pad; past a character's worth of significant digits the + // multiplication overflows and this was never an entity. + value = value.checked_mul(radix)?.checked_add(d)?; + } + None => { + if c == ';' && digits_held { + end = Some(i); + } + break; + } + } + } + let end = end?; + let ch = char::from_u32(value)?; + Some((ch, 2 + usize::from(radix == 16) + end + 1)) +} + +/// Names of the filters of a stream dictionary. +pub(super) fn filter_names(d: &Dict) -> Vec> { + match d.get(b"Filter") { + Some(Object::Name(n)) => vec![n.clone()], + Some(Object::Array(a)) => a.iter().filter_map(|o| o.as_name().map(<[u8]>::to_vec)).collect(), + _ => Vec::new(), + } +} + +/// Image codecs: their data is pixels, still encoded. +pub(super) fn image_codec(d: &Dict) -> bool { + filter_names(d).iter().any(|n| matches!(n.as_slice(), b"DCTDecode" | b"DCT" | b"JPXDecode" | b"CCITTFaxDecode" | b"CCF" | b"JBIG2Decode")) +} From 97cef90e016fc81d6683996e88c6eed13c1b8be7 Mon Sep 17 00:00:00 2001 From: Hamza Naveed <45464935+Hamza0101@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:01:42 +0500 Subject: [PATCH 08/14] Redaction: a pattern nothing paints with any more leaves the page (#3680) * Redaction: a pattern nothing paints with any more leaves the page A tiling pattern's cell is a content stream of its own and can draw anything, text included. Redacting a shape filled with such a pattern removed the fill from the page's content -- the page looked right, a black box -- but left the pattern in /Resources /Pattern, and its cell, with the text that was under the mark, was written into the saved file unchanged. The pass does not enter pattern cells, so it cannot redact inside one. What it can do is notice that nothing paints with the pattern any more. The graphics state now carries the fill and stroke pattern a `scn` or `SCN` selected (and `cs`, `g`, `rg`, `k` and their stroke forms clear it), every painting operator that survives the rewrite records the pattern it drew with, and a pattern the page selected but no surviving paint used leaves the page's resources. Measured on the report's file: before applied: 1, (SECRET) in the saved bytes after applied: 1, no /Pattern on the page, (SECRET) gone A pattern some surviving paint still uses is kept, and so is its cell: with two boxes filled from one pattern and only the first marked, patterns_removed is 0 and the resources keep the entry -- the second box still shows the word, so it is not a leak. Report gains patterns_removed. Fixes #3647. * Keep a pattern every operator that can paint with it still uses Two defects CodeRabbit found in the first version of this change, both of which could remove a pattern the page still draws with. Pattern use was recorded for path painting only. Text shows with the fill pattern too -- and with the stroke pattern in the stroking render modes -- as do image masks and anything a form draws, and a form's content is interpreted with a graphics state of its own, so a pattern it inherits is never seen inside it. Content like /Pattern cs /P0 scn 20 140 100 30 re f BT (KEEP) Tj ET with the box redacted therefore lost the fill of text still on the page. Every operator that can paint with the current pattern now records it: text (except mode 3, which paints nothing, and mode 7, which only clips), inline images, and image or form XObjects. Only path painting knows exactly what it drew and whether the paint survived, so the others are recorded as they are met. That can keep a pattern whose paint was in fact all redacted, which costs an unused resource entry, while dropping one something still draws with takes paint off the page. /Resources /Pattern may be an indirect object, like any value. Reading the reference itself as a dictionary found none and pruned nothing, so on such a file the cell -- with the text that was under the mark -- stayed in the saved bytes, which is the leak this change is for. It is resolved now, and what is written back is a dictionary of this page's own, so pruning never reaches through a /Pattern dictionary another page shares. The controls in the test: without the first fix the surviving text's pattern is removed (left: 1, right: 0), and without the second the indirect file prunes nothing (left: 0, right: 1). * M8: refuse retained unsafe tiling patterns * M8: guard shared appearances and narrow pattern scans * M8: update TLS dependencies for new advisories * Redaction: the comment on prune_patterns said more than the code does `prune_patterns` drops the `/Pattern` resource entry; the `scn` that selected it stays in the content, so the page names a pattern its resources no longer hold. The comment claimed the selector went too (noted in review). ISO 32000-1 8.6.6.2 makes that operand a key of the `/Pattern` subdictionary, so the open question is whether a reader stops reading the stream at the stale name. `a_page_whose_pruned_pattern_is_still_selected_renders` answers it with the renderer rather than with prose: the redacted page is saved, reopened and rasterized, and * it renders with no error and the same 300x300 page box, * the text written *after* the stale selector is still painted -- which is exactly what a reader would drop if it gave up at the bad operand, * the redaction overlay is black over the box the pattern filled. Control: with the drawing after the selector removed, the pixel assertion fails ("the text after the stale selector is still painted"), so it is reading that text and not a stray pixel. The selector is left in place rather than cut out: removing operators is a rewrite of the content this pass does not otherwise do, and nothing surviving paints with that colour. `pdfcraft-render` is a dev dependency of this crate only -- layer 4 on layer 3, which the layering allows. * WIP: pattern guard depth bound sized from the interpreter's * Pattern tests: keep the guard's cases reaching the guard now a hit path is removed whole * Pattern automation tests: keep the pattern in use where they test the guard, since a hit path is now removed whole * Redaction: a form without resources keeps the page pattern it still paints A form with no /Resources of its own paints with its caller's patterns, but its surviving pattern paint was recorded only in the form's own output and never reached the page. With the page's own fill under the mark removed whole, prune_patterns then dropped /P0 from the page while the form outside the mark still painted with it, so that fill lost its paint in the saved file. Pass the painted names up from a form that inherits its caller's resources. Only the painted side goes up, so this can keep a pattern, never prune one. * Redaction: a pruned pattern does not stay named by the page tree prune_patterns writes only the page's own /Resources, and the guard walked only pages' effective resources. A /Pages node that still named the pruned pattern kept its cell reachable from Root, so a full save wrote the cell, text included, while apply returned Ok. Three shapes did it: a page with no /Parent inheriting from the root, a /Parent that disagrees with the /Kids tree, and a root /Resources shadowed by a nearer holder that was retired. Patterns pruned from a page are now unlinked from page-tree holders no page inherits from, as unlink_doomed does for XObjects, when every /Parent agrees with the tree. Whenever a pattern was pruned (or a selected page had any), the guard also inspects the /Resources of every /Pages node Root reaches down /Kids and up each page's /Parent chain, so whatever stays reachable is proven safe or the redaction is refused, leaving the document unchanged. * Redaction: a Type 3 glyph without resources keeps the page pattern it paints A Type 3 font with no /Resources looks the names in its glyph descriptions up in the resources of the page or form showing it, so a glyph outside the mark can paint with the page's pattern. Text recorded only the fill and stroke pattern of the graphics state, so with the page's own fill under the mark removed whole, prune_patterns dropped the pattern and the glyph lost its paint. Showing such a font now keeps every pattern of the scope, as a form without /Resources keeps what it paints. * Redaction: say what text-recorded pattern use keeps Text, inline images and XObjects record their pattern before the hit test, so a pattern painted only by redacted text keeps its cell. The guard refuses a cell with text or images; a vector-only cell stays in the file. The comment said this cost only an unused resource entry. Pin the behaviour in a test. * Redaction: the page tree is followed up every /Parent link A full save writes every node a /Parent link reaches, but the guard followed /Parent only from pages and stopped at the first node it had already seen, and unlinking never followed it. A /Pages node reached only through the /Parent of a /Kids node -- a root /Pages with a /Parent, or an intermediate node whose /Parent leads out of the tree -- kept naming the pruned pattern, and its cell, text included, stayed in the saved file while apply returned Ok. The guard now follows /Parent from every node it visits, page or /Pages, with the same seen-set against cycles. Unlinking treats a root with a /Parent as an inconsistent tree, like a disagreeing link, and leaves the decision to the guard, which inspects every node those links reach and refuses an unsafe cell. * Redaction: a long page tree does not exhaust the pattern guard's budget The page-tree walk counted every node, pages included, against the guard's 16,384-object budget, so redacting a pattern page of a 17,000-page document was refused with a limit error where it had applied before. Only the holder nodes the guard inspects count against that budget now; the walk itself is bounded at a million nodes, as unlink_doomed bounds the same tree. * Redaction: a Type 3 font set by an ExtGState or shown in a form keeps the page patterns The resourceless Type 3 check covered only a font set by Tf in the scope that paints. A Type 3 font without /Resources set through an ExtGState /Font entry was not flagged, and a glyph shown inside a form with /Resources of its own recorded its patterns in the form's scope, which the page never saw. In both cases the page's own fill under the mark was removed whole, prune_patterns dropped the pattern, and the glyph outside the mark lost its paint. An ExtGState font is now checked the same way, and a glyph shown in any nested form marks the page as keeping every pattern it selected. This can only keep patterns; whatever is kept is checked by the guard. * Redaction: a pruned pattern a full save would still write is proven safe or refused The walks that check retained pattern cells follow the references they know -- resource categories, the page tree, appearances -- but a full save writes whatever the trailer reaches by any reference. A direct /Parent dictionary, a /Kids array on a page, or a stray resource entry (/Shading naming the pattern stream) kept a pruned cell, text included, in the saved file while apply returned Ok. pdfcraft_cos gains reachable_objects, which runs the same breadth-first walk write_full uses (factored out as full_save_order, so the two cannot drift). Before apply succeeds, every pruned pattern that walk still reaches is inspected like a retained cell: vector-only passes, anything else refuses with the existing pattern error. It runs only when a pattern was pruned and costs one walk of the reachable objects. * Redaction: a direct /Parent dictionary is part of the page tree too A /Parent given as a dictionary rather than a reference is written inline by a full save, but the page-tree walk followed /Parent only as a reference, and unlinking took such a tree for consistent. A root /Pages whose direct /Parent held /Resources naming the pruned pattern kept its cell in the saved file. The guard now walks a direct /Parent like any node (its /Resources, /Parent and /Kids), and unlinking counts any /Parent that is not the reference /Kids came from as an inconsistent tree, leaving the decision to the guard. * Redaction: page-tree walks queue each node once and borrow /Kids On a hostile tree whose /Pages nodes share one long /Kids array, the guard's page-tree walk and pattern unlinking pushed every /Kids entry of every node before checking whether it had been seen, and copied the array on each visit. Both now queue only nodes not seen yet and read /Kids in place; unlinking also stops at the first node reached twice, since its /Parent can agree with at most one of the nodes above it, and leaves that tree to the guard. * Redaction: a Type 3 font with /Resources null is resourceless too /Resources null is the same as no /Resources, but the check counted any entry, so such a font's glyph outside the mark lost the page pattern it painted with. A Type 3 glyph shown by a kept annotation appearance is a known limit, pinned as an ignored test: appearances are not interpreted for pattern use, and under ISO 32000 the glyph resolves in the appearance's own resources. * Redaction: a pruned pattern is recorded along its whole reference chain A pattern name can lead to a reference-valued object whose value is the cell (/P0 7 0 R, object 7 being 6 0 R). Pruning recorded only object 7, so the reachability backstop checked an object nothing reached while a stray reference kept the cell, object 6, in the saved file. Every object along the chain is recorded now, bounded like the other walks. * Pattern tests: pin the file identifier where two saves are compared main now gives a redacted rewrite a new identifier from fresh entropy, so an incremental and a full save after Apply differ in /ID alone. The test that checks they are the same save passes fixed entropy to both. * Redaction: replies and pop-ups removed on other pages feed the shared-appearance check Apply checks every page for retained unsafe pattern cells when a removed annotation's appearance is shared with one that stays on an unselected page. Only annotations removed under a mark fed that check. Replies and pop-ups that go with them in step 2b (removed_everywhere), often listed on another page, did not, so a text-bearing cell in an appearance a kept annotation shares with them escaped the all-pages check and Apply returned Ok. Their appearance references now go into removed_appearances too, with the same limit. Raised by FredApps' review of the rebase. --------- Co-authored-by: skylar taylor-barrick --- Cargo.lock | 1 + ROADMAP.md | 2 + crates/automation/tests/pattern_redaction.rs | 146 ++++ crates/cos/src/lib.rs | 2 +- crates/cos/src/writer.rs | 40 +- crates/redact/Cargo.toml | 3 + crates/redact/README.md | 9 + crates/redact/src/interp.rs | 127 ++- crates/redact/src/interp/prune.rs | 136 ++++ crates/redact/src/lib.rs | 137 +++- crates/redact/src/pattern_guard.rs | 496 ++++++++++++ crates/redact/src/tests.rs | 194 +++++ crates/redact/tests/pattern_page_tree.rs | 221 +++++ .../redact/tests/pattern_page_tree_shapes.rs | 275 +++++++ crates/redact/tests/pattern_reachability.rs | 304 +++++++ crates/redact/tests/pattern_redaction.rs | 758 ++++++++++++++++++ docs/roadmap.md | 2 +- 17 files changed, 2837 insertions(+), 16 deletions(-) create mode 100644 crates/automation/tests/pattern_redaction.rs create mode 100644 crates/redact/src/pattern_guard.rs create mode 100644 crates/redact/tests/pattern_page_tree.rs create mode 100644 crates/redact/tests/pattern_page_tree_shapes.rs create mode 100644 crates/redact/tests/pattern_reachability.rs create mode 100644 crates/redact/tests/pattern_redaction.rs diff --git a/Cargo.lock b/Cargo.lock index e1f271ed12..8617c45522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4628,6 +4628,7 @@ dependencies = [ "pdfcraft-fonts", "pdfcraft-forms", "pdfcraft-model", + "pdfcraft-render", "pdfcraft-testkit", "pdfcraft-xfa", "sha2 0.11.1", diff --git a/ROADMAP.md b/ROADMAP.md index 4fcdadaee9..820ceb0179 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -357,6 +357,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M5, arrow keys move comments):** With a comment selected, the arrow keys move it a screen pixel and ⌘/Ctrl or Shift and an arrow ten, instead of scrolling the view or turning the page; up is up on screen at any zoom and rotation, and each press is one undo step. Text markup and locked comments stay put. Estimates unchanged. +- **2026-10-20 (M8, #3647):** Apply prunes fully removed tiling patterns, then checks retained pattern resources for text, images and unreadable content before reporting success. Shared patterns, Form-owned and inherited patterns, and cells kept by another page cause an atomic refusal when safe removal cannot be shown. Synthetic saved-file and automation regressions cover refusal, preserved marks and undo, and vector-only controls. Estimates unchanged. + - **2026-10-20 (M1, PR #1143 review):** Repair-aware dictionary/preflight edits and snapshot restoration preserve both accepted generations of a damaged live slot. Such edits use the existing full-save fallback; classic and packed rewrites coalesce aliases so page-tree parent links retain one object identity. Explicit generation replacement remains single-generation. Materialized xref diagnostics inspect retained primary and hybrid rows before folding, retaining the 256-object cap plus summary and discarding abandoned-index notes. Synthetic red/green controls cover preflight, undo restoration, equal values, explicit narrowing, encryption packing and four save variants. Estimates unchanged. - **2026-10-19 (M14, #3373):** The main menu and its long submenus scroll in short windows, keeping their commands reachable. Estimates unchanged. diff --git a/crates/automation/tests/pattern_redaction.rs b/crates/automation/tests/pattern_redaction.rs new file mode 100644 index 0000000000..667779cef6 --- /dev/null +++ b/crates/automation/tests/pattern_redaction.rs @@ -0,0 +1,146 @@ +//! Rectangle redaction must not report a pattern-cell secret safely removed (#3647). + +use std::sync::Arc; + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_engine::DocId; +use serde_json::{Value, json}; + +fn tool(automation: &mut Automation, name: &str, args: Value) -> Value { + match automation.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +/// `retained`: paint with the pattern once more, away from the mark (x 290..340 lies right of it). +/// A path the mark hits is removed whole (F-008) and a pattern nothing surviving paints with is +/// pruned, so only a pattern that some surviving paint still uses reaches the guard. +fn fixture(cell: &[u8], partial: bool, compressed: bool, retained: bool) -> Vec { + let mut pattern = Dict::from_iter([ + (b"Type".to_vec(), Object::name("Pattern")), + (b"PatternType".to_vec(), Object::Int(1)), + (b"PaintType".to_vec(), Object::Int(1)), + (b"TilingType".to_vec(), Object::Int(1)), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"XStep".to_vec(), Object::Int(500)), + (b"YStep".to_vec(), Object::Int(300)), + ]); + pattern.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Font".to_vec(), + Object::Dict(Dict::from_iter([(b"F1".to_vec(), Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))])), + )])), + ); + let width = if partial { 400 } else { 170 }; + let survivor = if retained { " 290 115 50 30 re f" } else { "" }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET /Pattern cs /P0 scn 35 110 {width} 35 re f{survivor}"); + let mut page_stream = Vec::new(); + pdfcraft_cos::serialize(&Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes())), &mut page_stream); + let mut pattern_stream = Vec::new(); + let stream = if compressed { Stream::flate(pattern, cell) } else { Stream::from_raw(pattern, cell.to_vec()) }; + pdfcraft_cos::serialize(&Object::Stream(stream), &mut pattern_stream); + let objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >> >>" + .to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + page_stream, + pattern_stream, + ]; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend(format!("{} 0 obj\n", index + 1).as_bytes()); + bytes.extend(object); + bytes.extend(b"\nendobj\n"); + } + let xref = bytes.len(); + bytes.extend(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes()); + for offset in offsets { + bytes.extend(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes()); + bytes +} + +fn saved_has_secret(bytes: Vec) -> bool { + let document = Document::open(Arc::new(bytes)).unwrap(); + document.object_numbers().into_iter().any(|num| { + let object = document.try_get(num).unwrap(); + matches!(&*object, Object::Stream(stream) if stream.decoded().unwrap().windows(10).any(|part| part == b"CELLSECRET")) + }) +} + +#[test] +fn pattern_redaction_tools_remove_or_refuse_without_leaking_or_changing_undo() { + let dir = tempfile::tempdir().unwrap(); + for (partial, retained) in [(false, false), (true, false), (true, true)] { + for compressed in [false, true] { + let cell = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET BT /F1 20 Tf 300 120 Td (SAFE) Tj ET"; + let bytes = fixture(cell, partial, compressed, retained); + assert!(saved_has_secret(bytes.clone())); + std::fs::write(dir.path().join("input.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "input.pdf"}))["doc"].as_u64().unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let before = automation.session().get(DocId(id)).unwrap().bytes.clone(); + let pixels = automation.render_pixels(id, 1, 1.0).unwrap(); + match automation.call("redact_apply", &json!({"doc": id})) { + Ok(_) => { + assert!(!retained, "a text-bearing cell still in use must not be reported safe (partial: {partial})"); + tool(&mut automation, "doc_save", json!({"doc": id, "path": "redacted.pdf"})); + assert!( + !saved_has_secret(std::fs::read(dir.path().join("redacted.pdf")).unwrap()), + "successful apply/save left the pattern-cell text recoverable" + ); + } + Err(error) => { + assert!(retained, "a pattern nothing surviving paints with should apply (partial: {partial}): {error}"); + assert!(error.to_string().contains("tiling pattern"), "{error}"); + assert!(!error.to_string().contains("CELLSECRET")); + let document = automation.session().get(DocId(id)).unwrap(); + assert_eq!(document.bytes, before); + assert_eq!(document.redaction_marks(), 1); + assert_eq!(document.can_undo(), Some("Add redaction mark")); + assert_eq!(automation.render_pixels(id, 1, 1.0).unwrap(), pixels); + tool(&mut automation, "edit_undo", json!({"doc": id})); + assert_eq!(automation.session().get(DocId(id)).unwrap().redaction_marks(), 0); + } + } + } + } +} + +#[test] +fn pattern_redaction_vector_control_saves_reopens_and_preserves_outside_pixels() { + let dir = tempfile::tempdir().unwrap(); + // The guard's control: the cell is plain vectors and a paint outside the mark still uses it, + // so it is inspected and kept, not refused, and what that paint draws is unchanged. + let bytes = fixture(b"0.2 0.4 0.6 rg 40 120 15 15 re f 300 120 30 20 re f", true, true, true); + std::fs::write(dir.path().join("vectors.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "vectors.pdf"}))["doc"].as_u64().unwrap(); + let (width, height, original) = automation.render_pixels(id, 1, 1.0).unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let applied = tool(&mut automation, "redact_apply", json!({"doc": id})); + assert_eq!(applied["applied"], 1); + let save = tool(&mut automation, "doc_save", json!({"doc": id, "path": "vectors-redacted.pdf"})); + assert_eq!(save["incremental"], false); + let reopened = tool(&mut automation, "doc_open", json!({"path": "vectors-redacted.pdf"}))["doc"].as_u64().unwrap(); + let (new_width, new_height, actual) = automation.render_pixels(reopened, 1, 1.0).unwrap(); + assert_eq!((new_width, new_height), (width, height)); + for (index, (before, after)) in original.as_chunks::<4>().0.iter().zip(actual.as_chunks::<4>().0).enumerate() { + let (x, y) = (index % width as usize, index / width as usize); + if x >= 280 { + assert_eq!(before, after, "unmarked pattern pixels changed at {x},{y}"); + } + if (35..205).contains(&x) && (155..190).contains(&y) { + assert_eq!(after, &[0, 0, 0, 255], "marked interior must be black"); + } + } +} diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 8e051e4578..32a6f6d8b3 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::{Lexer, is_whitespace, parse_indirect}; pub use pdfcraft_crypt::{ Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, }; -pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental}; +pub use writer::{SaveOptions, pdf_date, reachable_objects, serialize, write_full, write_incremental}; #[derive(Debug, thiserror::Error, Clone, PartialEq)] pub enum CosError { diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index e0762612dc..0af74f7bab 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -273,16 +273,12 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, Ok(file) } -/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. -pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - // A full traversal must not fill the editor's shared caches (which undo snapshots also - // own). Parsed objects and decoded object streams are temporary working data here. - let mut reader = doc.object_reader(); - stamp_mod_date(&mut reader.document, opts); - let trailer_in = reader.document.trailer().clone(); +/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and +/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets +/// (aliases of one slot included) and, in visit order, whether it is a stream. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>); + +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -326,6 +322,30 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro order.push((canonical, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); } + Ok((map, order)) +} + +/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] +/// reaches from the trailer, object streams included. For a check that something removed from a +/// document cannot reach the saved file through a reference nobody looked for. +pub fn reachable_objects(doc: &Document) -> Result, CosError> { + let mut reader = doc.object_reader(); + let trailer_in = reader.document.trailer().clone(); + let (_, order) = full_save_order(&mut reader, &trailer_in)?; + Ok(order.into_iter().map(|(r, _)| r.num).collect()) +} + +/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. +pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + // A full traversal must not fill the editor's shared caches (which undo snapshots also + // own). Parsed objects and decoded object streams are temporary working data here. + let mut reader = doc.object_reader(); + stamp_mod_date(&mut reader.document, opts); + let trailer_in = reader.document.trailer().clone(); + let (map, order) = full_save_order(&mut reader, &trailer_in)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index c02335e3c4..149d8d8295 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -25,3 +25,6 @@ sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } +# The redacted page is rendered in `tests::a_page_whose_pruned_pattern_is_still_selected_renders`: +# layer 3 from layer 4, a dev dependency only. +pdfcraft-render = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index db9f46b8c9..f78f819c03 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -60,6 +60,15 @@ resume after it would be a guess. A Redact annotation with neither a usable `/Qu `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. +Tiling-pattern cells are separate streams. Applying a mark removes a pattern resource when no +surviving paint uses it. If a cell remains reachable after rewriting the selected pages and may +contain text, images, replacement descriptions, or unreadable content, Apply refuses with a +`RedactError::Pattern` and restores the document and pending marks. This includes patterns owned +by Forms and patterns inherited by Forms. Vector-only retained cells remain supported. The +bounded check is conservative: an unrelated retained pattern on a selected page can also cause +refusal when its content cannot be proven safe. If a selected page had a pattern resource, the +check covers all pages after rewriting so another page cannot keep the same cell in the saved PDF. + Before changing any page, `apply` prepares the redaction appearances with a shared 8 MiB budget for their uncompressed PDF content across the selected pages. It bounds repeated-text counts and working line buffers before allocating them, and measures and encodes each repeated diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 7a2cd26eca..b0a7257cb3 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -31,7 +31,7 @@ mod prune; use layers::strip_hidden_layers; use prune::{clip_out, extgstate_font, prune_form_resources}; -pub(crate) use prune::{prune, prune_properties, resource_ops, unlink_doomed}; +pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns}; /// How deep this pass follows form XObjects. A form past it is removed rather than walked, which /// is the fail-safe choice for a redaction -- marked content inside a form we cannot read would @@ -85,6 +85,17 @@ struct Gs { scale: f64, leading: f64, rise: f64, + /// The tiling or shading pattern selected as the fill colour (`/Name scn`), and as the stroke + /// colour (`/Name SCN`). Part of the graphics state, so `q`/`Q` save and restore them with + /// everything else. A pattern's cell is a content stream of its own and this pass does not + /// enter it, so a pattern whose only painting was removed has to leave the page's resources + /// or its drawing stays in the file (#3647). + fill_pattern: Option>, + stroke_pattern: Option>, + /// The current font is a Type 3 font without `/Resources`: its glyph descriptions look their + /// names up in this scope's resources (ISO 32000-2 §9.6.4), so a shown glyph may paint with + /// any of this scope's patterns. + type3_inherits: bool, } /// What processing a scope produced. @@ -102,6 +113,16 @@ pub(crate) struct Output { /// dictionaries may name them too (an inherited or shared /Resources, or the caller's when a /// form has none of its own): [`unlink_doomed`] removes those entries. pub doomed: Vec, + /// Pattern resource names a painting operator that survived the rewrite still uses. A name + /// this scope saw selected but never painted with afterwards is pruned from the page's + /// `/Resources /Pattern`, so a cell whose only drawing was redacted does not stay in the + /// saved file (#3647). + pub patterns_seen: Vec>, + pub patterns_painted: Vec>, + /// A Type 3 glyph without resources of its own was shown in this scope or a nested form, + /// through a font set by `Tf` or an ExtGState: which of the page's patterns it paints with is + /// not tracked, so the page keeps them all. + pub keeps_all_patterns: bool, /// Original property values retired by this scope or a nested form. Other resource holders /// may still name them; unlink only entries no surviving content uses. pub retired_properties: Vec, @@ -264,6 +285,22 @@ struct Glyph { rects: Vec, } +/// Record the patterns an operator paints with, so [`prune_patterns`] keeps them. Only path +/// painting knows exactly what it draws and whether the paint survived; text, images and forms +/// are recorded as they are met, which can keep a pattern whose paint was in fact all redacted, +/// and with it the pattern's cell. A cell that carries text or images is then refused by +/// `pattern_guard`; a vector-only cell stays in the file though nothing shows it +/// (`a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell`). Dropping a pattern something +/// still draws with would lose paint from the page instead. +fn uses_patterns(out: &mut Output, gs: &Gs, fills: bool, strokes: bool) { + if fills && let Some(n) = &gs.fill_pattern { + out.patterns_painted.push(n.clone()); + } + if strokes && let Some(n) = &gs.stroke_pattern { + out.patterns_painted.push(n.clone()); + } +} + /// Process a scope's content streams (state flows from one stream to the next). pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], resources: &Dict, ctm: Matrix) -> Output { let gs = Gs { @@ -277,6 +314,9 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< scale: 1.0, leading: 0.0, rise: 0.0, + fill_pattern: None, + stroke_pattern: None, + type3_inherits: false, }; process_from(doc, scope, streams, resources, gs) } @@ -377,6 +417,8 @@ struct Walk { fonts_res: Dict, ext_gstates: Dict, xobjects: Dict, + /// This scope's pattern names, which a Type 3 glyph without resources may paint with. + patterns: Vec>, fallback: Rc, /// The current path: its operators, its bounding box in user space, and whether it clips. path: Vec, @@ -395,6 +437,7 @@ impl Walk { fonts_res: res_dict(doc, resources, b"Font"), ext_gstates: res_dict(doc, resources, b"ExtGState"), xobjects: res_dict(doc, resources, b"XObject"), + patterns: res_dict(doc, resources, b"Pattern").iter().map(|(n, _)| n.to_vec()).collect(), fallback: Rc::new(Metrics::fallback()), path: Vec::new(), path_box: None, @@ -433,6 +476,39 @@ impl Walk { b"BI" => Self::inline_image(&self.gs, scope, &mut self.out, op, piece), b"sh" if scope.mode == Mode::Apply => Self::shading(&self.gs, scope, op, piece), b"Do" => self.draw_xobject(doc, scope, resources, op, piece), + // `/Name scn` selects a pattern as the fill colour; numbers select a plain colour + // and clear it. `cs` with a space other than /Pattern clears it too. + b"scn" | b"SCN" => { + let fill = o == b"scn"; + let picked = op.operands.last().and_then(Object::as_name).map(<[u8]>::to_vec); + if let Some(name) = &picked { + self.out.patterns_seen.push(name.clone()); + } + if fill { + self.gs.fill_pattern = picked; + } else { + self.gs.stroke_pattern = picked; + } + Some(op) + } + b"cs" | b"CS" => { + if op.name(0) != Some(b"Pattern") { + if o == b"cs" { + self.gs.fill_pattern = None; + } else { + self.gs.stroke_pattern = None; + } + } + Some(op) + } + b"g" | b"rg" | b"k" => { + self.gs.fill_pattern = None; + Some(op) + } + b"G" | b"RG" | b"K" => { + self.gs.stroke_pattern = None; + Some(op) + } _ => Some(op), }; piece.ops.extend(kept); @@ -464,6 +540,12 @@ impl Walk { let clips = std::mem::replace(&mut self.clip, false); let hit = scope.mode == Mode::Apply && op.op.as_slice() != b"n" && bbox.is_some_and(|b| scope.hits(b)); if !hit { + // The paint survives, so the pattern it draws with is still in use. `n` paints + // nothing, so it uses none. A hit path is removed whole (F-008), pattern and all. + let o = op.op.as_slice(); + let strokes = matches!(o, b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*"); + let fills = matches!(o, b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*"); + uses_patterns(&mut self.out, &self.gs, fills, strokes); piece.ops.extend(body); piece.ops.push(op); return; @@ -497,6 +579,8 @@ impl Walk { _ => { if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); + // As for `Tf`: a Type 3 font without /Resources paints through this scope's. + self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } } } @@ -507,6 +591,7 @@ impl Walk { let Some(name) = op.name(0) else { return }; let font_dict = self.fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); self.gs.vertical = font_dict.as_ref().is_some_and(|d| is_vertical(doc, d)); + self.gs.type3_inherits = font_dict.as_ref().is_some_and(|d| type3_without_resources(doc, d)); self.gs.font = match scope.fonts.get(name) { Some(f) => f.clone(), None => { @@ -555,6 +640,16 @@ impl Walk { /// `Tj`, `TJ`, `'` and `"`: glyphs under a region are cut out (or counted, when verifying). fn show_text(&mut self, scope: &mut Scope<'_>, op: Op, piece: &mut Piece) -> Option { + // Text paints with the fill pattern, and with the stroke pattern in the stroking render + // modes. Mode 3 paints nothing and mode 7 only clips. + let mode = self.gs.render_mode; + uses_patterns(&mut self.out, &self.gs, matches!(mode, 0 | 2 | 4 | 6), matches!(mode, 1 | 2 | 5 | 6)); + // A Type 3 glyph without resources of its own paints through this scope's: keep every + // pattern it could select, as with a form that has no /Resources. + if self.gs.type3_inherits { + self.out.patterns_painted.extend(self.patterns.iter().cloned()); + self.out.keeps_all_patterns = true; + } let o = op.op.as_slice(); if o == b"\"" { self.gs.word_spacing = op.num(0).unwrap_or(self.gs.word_spacing); @@ -608,6 +703,8 @@ impl Walk { /// An inline image under a region is removed (or counted, when verifying). fn inline_image(gs: &Gs, scope: &mut Scope<'_>, out: &mut Output, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, pattern included. + uses_patterns(out, gs, true, false); if !scope.hits(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { return Some(op); } @@ -632,6 +729,9 @@ impl Walk { } fn draw_xobject(&mut self, doc: &mut Document, scope: &mut Scope<'_>, resources: &Dict, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, and a form's content is interpreted with a + // graphics state of its own, so a pattern it inherits from here is never recorded inside it. + uses_patterns(&mut self.out, &self.gs, true, true); let Some(new) = xobject(doc, scope, &op, &self.gs, resources, &mut self.xobjects, &mut self.out) else { return Some(op) }; piece.changed = true; piece.ops.extend(new); @@ -924,8 +1024,18 @@ fn xobject( out.ambiguous |= inner.ambiguous; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); + // A Type 3 glyph without resources shown inside the form may paint with the page's + // patterns (the renderer resolves its names there), whether or not the form has + // /Resources of its own. + out.keeps_all_patterns |= inner.keeps_all_patterns; out.tag_changes.extend(inner.tag_changes); out.error = inner.error.or(out.error.take()); + // A form without /Resources of its own paints with its caller's patterns, so what it + // still paints keeps the caller's entry. Only the painted side is passed up: a name + // the form selected but no longer paints with is left for the caller to keep. + if own.is_none() { + out.patterns_painted.extend(inner.patterns_painted); + } if scope.mode == Mode::Verify { return None; } @@ -992,3 +1102,18 @@ fn xobject( _ => None, } } + +/// Whether the ExtGState `name` sets a Type 3 font that has no `/Resources` of its own. +fn extgstate_type3_inherits(doc: &Document, ext_gstates: &Dict, name: &[u8]) -> bool { + let Some(state) = ext_gstates.get(name).map(|s| doc.resolve(s)) else { return false }; + let Some(font) = state.as_dict().and_then(|d| d.get(b"Font")).map(|f| doc.resolve(f)) else { return false }; + let Some(reference) = font.as_array().and_then(|a| a.first()) else { return false }; + let dict = doc.resolve(reference); + dict.as_dict().is_some_and(|d| type3_without_resources(doc, d)) +} + +/// A Type 3 font whose glyph descriptions use the resources of whatever shows them: no +/// `/Resources`, or `/Resources null`, which is the same thing. +fn type3_without_resources(doc: &Document, font: &Dict) -> bool { + font.name(b"Subtype") == Some(b"Type3") && font.get(b"Resources").is_none_or(|r| matches!(&*doc.resolve(r), Object::Null)) +} diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 6ca5ea2803..188af03abe 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -355,3 +355,139 @@ pub(super) fn extgstate_font( }; Some((metrics, vertical, size)) } + +/// Drop every `/Pattern` entry this scope saw selected but no surviving painting operator drew +/// with. The `scn` that selected it stays in the content, so the page names a pattern its +/// resources no longer hold; nothing surviving paints with that colour, and +/// `a_page_whose_pruned_pattern_is_still_selected_renders` pins that the page still renders and +/// that operators after the stale selector still draw. A pattern's cell is a content stream of its own which +/// this pass does not enter, so a pattern whose only painting was redacted would otherwise keep +/// its drawing -- text included -- in the saved file (#3647). A pattern some surviving paint +/// still uses, or one the scope never saw, is left alone. +/// `/Pattern` may be an indirect object like any other value, so it is resolved: reading the +/// reference itself as a dictionary found none and pruned nothing, leaving the cell in the file. +/// What is written back is a dictionary of this page's own, so pruning never reaches through a +/// `/Pattern` dictionary another page shares. The pattern objects dropped are added to `pruned`, +/// for [`unlink_pruned_patterns`]. +pub(crate) fn prune_patterns(doc: &Document, res: &mut Dict, seen: &[Vec], painted: &[Vec], pruned: &mut Vec) -> usize { + let Some(mut pat) = res.get(b"Pattern").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()) else { return 0 }; + let mut dropped = 0; + for n in seen { + if !painted.contains(n) + && let Some(value) = pat.get(n) + { + // Every object along the reference chain: a name can lead to a reference-valued + // object whose value is the cell, and the cell is what must not stay reachable. + let mut next = value.as_ref(); + for _ in 0..MAX_DEPTH { + let Some(reference) = next else { break }; + pruned.push(reference); + next = match &*doc.get(reference) { + Object::Ref(r) => Some(*r), + _ => None, + }; + } + pat.remove(n); + dropped += 1; + } + } + if dropped > 0 { + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + } + dropped +} + +/// Unlink the patterns a page's rewrite pruned (`pruned`) from page-tree nodes whose +/// `/Resources` no page inherits any more: a full save keeps whatever Root still reaches, so a +/// `/Pages` node above the page that still names the cell would keep it in the file. That happens +/// when the page's nearest holder was retired but a holder further up, which it shadowed, still +/// names the pattern. A holder some page still inherits from is left alone; whatever stays +/// reachable is checked by `pattern_guard`, which refuses a retained cell it cannot prove safe. +/// When any node's `/Parent` disagrees with where the `/Kids` tree puts it (or the root has +/// one), which holder a page inherits from is ambiguous and a node may be reached only through +/// `/Parent`, so nothing is unlinked and the guard, which follows `/Parent` too, decides. +pub(crate) fn unlink_pruned_patterns(doc: &mut Document, pruned: &[ObjRef]) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let pruned: HashSet = pruned.iter().copied().collect(); + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(()) }; + // Every node with a /Resources of its own, and whether any page inherits from it (it is the + // nearest holder above a page, or the page itself). Cycles are cut as in `unlink_doomed`. + let mut holders: Vec<(ObjRef, bool)> = Vec::new(); + let mut seen = HashSet::new(); + let mut consistent = true; + let mut stack: Vec<(ObjRef, Option, Option)> = vec![(top, None, None)]; + seen.insert(top); + while let Some((node, parent, mut holder)) = stack.pop() { + let obj = doc.get(node); + let Some(d) = obj.as_dict() else { continue }; + // Every node's /Parent must be the node /Kids reached it from, and the root has none. + // A node reached only through a /Parent link is then impossible here; when the links + // disagree, the guard inspects every node they reach instead. A /Parent that is not a + // reference (a direct dictionary, written inline by a full save) never agrees. + let agrees = match d.get(b"Parent") { + None => parent.is_none(), + Some(Object::Ref(r)) => Some(*r) == parent, + Some(_) => false, + }; + if !agrees { + consistent = false; + break; + } + if d.contains(b"Resources") { + holders.push((node, false)); + holder = Some(holders.len() - 1); + } + let is_pages = d.name(b"Type") == Some(b"Pages") || (d.contains(b"Kids") && d.name(b"Type") != Some(b"Page")); + if is_pages { + // Borrowed, not copied: a /Kids array can be shared by many hostile nodes. + let kids_object = d.get(b"Kids").map(|k| doc.resolve(k)); + for kid in kids_object.as_deref().and_then(Object::as_array).into_iter().flatten().filter_map(Object::as_ref) { + // A node reached twice (a shared /Kids entry or a cycle) has a /Parent that agrees + // with at most one of the nodes above it. + if !seen.insert(kid) || seen.len() > 1_000_000 { + consistent = false; + break; + } + stack.push((kid, Some(node), holder)); + } + if !consistent { + break; + } + } else if let Some(h) = holder.and_then(|i| holders.get_mut(i)) { + h.1 = true; + } + } + if !consistent { + return Ok(()); + } + for (node, inherited) in holders { + if inherited { + continue; + } + let Some(mut res) = doc.get(node).as_dict().and_then(|d| d.get(b"Resources")).and_then(|r| doc.resolve(r).as_dict().cloned()) else { + continue; + }; + let mut pat = res_dict(doc, &res, b"Pattern"); + let names: Vec> = pat.iter().filter(|(_, v)| v.as_ref().is_some_and(|r| pruned.contains(&r))).map(|(n, _)| n.to_vec()).collect(); + if names.is_empty() { + continue; + } + for n in &names { + pat.remove(n); + } + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + // A private copy on this node: the dictionary object may be shared with other nodes. + doc.update_dict(node, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + } + Ok(()) +} diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 6bb8e5cc06..6d323004ad 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -22,6 +22,7 @@ pub mod codes; mod image; mod interp; mod overlay; +mod pattern_guard; pub mod patterns; pub mod sanitize; mod tags; @@ -34,6 +35,7 @@ pub(crate) mod type3_tables; pub mod verify; use interp::{Mode, Scope, process}; +pub use pattern_guard::PatternProblem; pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -53,6 +55,8 @@ pub enum RedactError { #[error("redaction could not be verified: {0} item(s) were still found under the marks, so nothing was changed")] Residue(usize), /// A mark's overlay text has a character Helvetica can't draw; nothing was redacted. + #[error("page {page} cannot be redacted safely: {reason}; nothing was changed")] + Pattern { page: usize, reason: PatternProblem }, #[error("{0}")] Undrawable(String), #[error( @@ -146,6 +150,8 @@ fn overlay_look(doc: &Document, d: &Dict) -> pdfcraft_annot::OverlayLook { /// What [`apply`] removed. #[derive(Clone, Debug, Default, PartialEq)] pub struct Report { + /// Patterns dropped from a page's resources because nothing painted with them any more. + pub patterns_removed: usize, pub marks: usize, pub pages: usize, pub glyphs: usize, @@ -397,6 +403,38 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result Option> { + const MAX_REFERENCES: usize = 16_384; + let mut references = HashSet::new(); + let Some(ap) = annotation.get(b"AP") else { return Some(references) }; + if let Some(reference) = ap.as_ref() { + references.insert(reference); + } + let resolved_ap = doc.resolve(ap); + let Some(appearances) = resolved_ap.as_dict() else { return Some(references) }; + for key in [b"N".as_slice(), b"R", b"D"] { + let Some(value) = appearances.get(key) else { continue }; + if let Some(reference) = value.as_ref() { + references.insert(reference); + } + let resolved = doc.resolve(value); + if let Object::Dict(states) = &*resolved { + for (_, state) in states.iter() { + if let Some(reference) = state.as_ref() { + references.insert(reference); + if references.len() > MAX_REFERENCES { + return None; + } + } + } + } + } + Some(references) +} + /// [`apply`] without the copy: an error can leave `doc` part redacted. fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result { let images = pdfcraft_cos::ImageBudget::new(doc); @@ -429,6 +467,16 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // content rewriting and verification instead of scanning all marks for every page. let groups: Vec<_> = chosen.chunk_by(|a, b| a.page == b.page).filter_map(|group| group.first().map(|mark| (mark.page, group))).collect(); let by_page: Vec<_> = groups.iter().map(|(page, _)| *page).collect(); + // A pattern pruned from a selected page can remain reachable through another page's + // resources. When a selected page had patterns, inspect every page after the rewrite. + let source_pages = pdfcraft_model::pages(doc); + let shared_pattern_possible = by_page.iter().any(|&pi| { + source_pages + .get(pi) + .and_then(|page| page.dict.get(b"Resources")) + .map(|resources| doc.resolve(resources)) + .is_some_and(|resources| resources.as_dict().is_some_and(|dict| dict.get(b"Pattern").is_some())) + }); // Budget failures must leave even a direct library caller's document untouched. let mut remaining = overlay::MAX_BYTES; let overlays = groups @@ -448,6 +496,9 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co widget_owner, removed_everywhere: HashSet::new(), images, + removed_appearances: HashSet::new(), + appearance_scan_limit: false, + pruned_patterns: Vec::new(), }; for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -458,7 +509,18 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if applying.images.refused() > 0 { return Err(RedactError::ImagesPastAllowance); } - let Applying { mut report, doomed_fields, doomed_xobjects, retired_properties, mut removed_everywhere, mut images, .. } = applying; + let Applying { + mut report, + doomed_fields, + doomed_xobjects, + retired_properties, + mut removed_everywhere, + mut images, + mut removed_appearances, + mut appearance_scan_limit, + pruned_patterns, + .. + } = applying; // 2b. Replies and pop-ups of a removed annotation can be listed on another page than their // parent: their `/IRT` or `/Parent` would keep the removed comment reachable. @@ -481,6 +543,19 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); if removed_everywhere.contains(&r) || is_reply || is_popup { + // A reply or pop-up removed here can share its appearance with an annotation + // that stays on an unselected page, like one removed under a mark: the + // all-pages pattern check must see it. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } if removed_everywhere.insert(r) { grew = true; if is_reply { @@ -541,6 +616,8 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // A removed image or form must not stay reachable through another resource dictionary (an // inherited or shared /Resources): a full save keeps whatever is still referenced. interp::unlink_doomed(doc, &doomed_xobjects, &retired_properties)?; + // Likewise a pruned pattern: a /Pages node above the page can still name its cell. + interp::unlink_pruned_patterns(doc, &pruned_patterns)?; report.images_restored = restore_shared_jbig2(doc, &doomed_xobjects, &mut images)?; // 5. Verify: nothing readable may remain under a region. @@ -563,6 +640,29 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if residue > 0 { return Err(RedactError::Residue(residue)); } + // The interpreter can remove a pattern's paint while another use keeps the cell reachable. + // Inspect the finished selected pages, including rewritten Forms, and all pages if a removed + // pattern or appearance may still be shared. Retained unsafe cells refuse the redaction; + // `apply` runs on a copy, so the refusal leaves the caller's document as it was. + let shared_appearance_possible = appearance_scan_limit + || !removed_appearances.is_empty() + && pdfcraft_model::pages(doc).iter().enumerate().any(|(index, page)| { + !by_page.contains(&index) + && annots_of(doc, &page.dict).iter().any(|annotation| { + doc.resolve(annotation).as_dict().is_some_and(|dict| { + appearance_refs(doc, dict).is_none_or(|refs| refs.iter().any(|reference| removed_appearances.contains(reference))) + }) + }) + }); + let first_page = by_page.first().map_or(1, |p| p + 1); + let check_pages = + if shared_pattern_possible || shared_appearance_possible { (0..pdfcraft_model::pages(doc).len()).collect::>() } else { by_page }; + // A pruned pattern may also stay named by a /Pages node no page inherits from, or by one a + // page with a broken /Parent link still reaches: then every page-tree node is inspected too. + pattern_guard::check(doc, &check_pages, shared_pattern_possible || !pruned_patterns.is_empty())?; + // Whatever reference keeps a pruned pattern in the file -- one no walk above follows -- the + // full save would write it: prove it safe or refuse. + pattern_guard::check_pruned_unreachable(doc, &pruned_patterns, first_page)?; // The previous revision still holds the removed content: the next save must rewrite, and the // rewritten file is a different file from its source, so it gets a new identifier. doc.require_full_save_with_new_id(); @@ -650,6 +750,13 @@ struct Applying { removed_everywhere: HashSet, /// What clearing images may spend, across the run's pages. images: pdfcraft_cos::ImageBudget, + /// Appearance streams of the annotations removed, on any page: one shared with a kept + /// annotation keeps a pruned pattern's cell reachable (see `pattern_guard`). + removed_appearances: HashSet, + /// Too many appearance references to track: the pattern guard inspects every page. + appearance_scan_limit: bool, + /// Pattern objects pruned from a page's resources, which page-tree nodes may still name. + pruned_patterns: Vec, } /// Step 1 for one page: rewrites its content so nothing readable stays under a mark, and cleans the tags it no longer shows. @@ -660,7 +767,7 @@ fn redact_page_content( page: &pdfcraft_model::Page, rects: &[[f64; 4]], ) -> Result<(), RedactError> { - let Applying { report, doomed_xobjects, retired_properties, images, .. } = applying; + let Applying { report, doomed_xobjects, retired_properties, images, pruned_patterns, .. } = applying; // 1. Content. let mut budget = pdfcraft_cos::DecodeBudget::page(); let (list, data) = page_streams_within(doc, &page.dict, pi, &mut budget)?; @@ -733,6 +840,11 @@ fn redact_page_content( }, pi, )?; + // A pattern this page selected but no surviving paint draws leaves its resources, so + // its cell -- a content stream this pass does not enter -- cannot stay in the file. + // A Type 3 glyph without resources paints with patterns nobody tracked: keep them all. + let painted = if out.keeps_all_patterns { &out.patterns_seen } else { &out.patterns_painted }; + report.patterns_removed += interp::prune_patterns(doc, &mut res, &out.patterns_seen, painted, pruned_patterns); retired_properties.extend(interp::prune_properties(doc, &mut res, &cleaned_data)?); type3::prune_type3_from_resources(doc, &mut res, &cleaned_data)?; doc.update_dict(page.obj, |d| { @@ -761,7 +873,7 @@ fn remove_page_annotations( rects: &[[f64; 4]], page_marks: &[&Mark], ) -> Result<(), RedactError> { - let Applying { report, doomed_fields, widget_owner, removed_everywhere, .. } = applying; + let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); let mut removed: Vec = Vec::new(); @@ -819,6 +931,25 @@ fn remove_page_annotations( } } removed_everywhere.extend(removed.iter().copied()); + // A removed annotation's appearance streams can be shared with a kept one on another page, + // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. + for reference in &removed { + if *appearance_scan_limit { + break; + } + if let Some(annotation) = doc.get(*reference).as_dict() { + match appearance_refs(doc, annotation) { + Some(references) => { + removed_appearances.extend(references); + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } + } + None => *appearance_scan_limit = true, + } + } + } // Pop-ups dropped with their parent below count too: a reply on another page may answer one. for a in &annots { let Some(r) = a.as_ref() else { continue }; diff --git a/crates/redact/src/pattern_guard.rs b/crates/redact/src/pattern_guard.rs new file mode 100644 index 0000000000..96b865e3da --- /dev/null +++ b/crates/redact/src/pattern_guard.rs @@ -0,0 +1,496 @@ +//! Refuse pattern content that remains reachable after redaction (#3647). +//! +//! Pattern cells paint through a path, not `Do`, so clipping/removing the path does not remove +//! their bytes. Inspect the rewritten pages' resource graphs before committing the +//! working copy. This is deliberately conservative: an unused text/image-bearing pattern is +//! refused too. A plain vector pattern remains supported. The bounded read-only pass follows +//! the fail-closed approach proposed in PR #162. + +use std::collections::HashSet; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use crate::RedactError; + +/// How deep the walk may recurse. Every step down (a reference, an array, a dictionary, a stream, +/// a resources map) is one frame, and a nested form costs about four of them: the 50-deep chain +/// in `a_deeply_nested_form_keeps_the_text_nobody_marked` peaks at 200. The interpreter follows +/// forms as deep as the renderer and Edit text read (`interp::MAX_DEPTH`, #3645), so this must +/// reach at least that far, or the guard would refuse a document the redaction itself handles. +/// Total work is bounded separately by `MAX_OBJECTS`, and 512 frames stay far inside a thread's +/// stack. +const MAX_DEPTH: usize = 512; +const _: () = assert!(MAX_DEPTH >= 8 * crate::interp::MAX_DEPTH, "the guard must walk past the deepest form the interpreter follows"); +const MAX_OBJECTS: usize = 16_384; +const MAX_STREAM_BYTES: usize = 1 << 20; +const MAX_TOTAL_BYTES: usize = 8 << 20; +const MAX_FILTERS: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum PatternProblem { + #[error("a tiling pattern carries text or images that cannot be removed safely")] + Content, + #[error("pattern resources or content cannot be read completely")] + Unreadable, + #[error("pattern inspection exceeded its resource, depth or decoded-content budget")] + Limit, +} + +struct Guard { + left: usize, + bytes_left: usize, + seen: HashSet<(ObjRef, bool)>, + active: HashSet<(ObjRef, bool)>, +} + +impl Guard { + fn new() -> Self { + Self { left: MAX_OBJECTS, bytes_left: MAX_TOTAL_BYTES, seen: HashSet::new(), active: HashSet::new() } + } + + fn charge(&mut self, depth: usize) -> Result<(), PatternProblem> { + self.left = self.left.checked_sub(1).ok_or(PatternProblem::Limit)?; + if depth > MAX_DEPTH { + return Err(PatternProblem::Limit); + } + Ok(()) + } + + fn walk(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + match object { + Object::Ref(reference) => { + let key = (*reference, inside); + if inside && self.active.contains(&key) { + return Err(PatternProblem::Unreadable); + } + if !self.seen.insert(key) { + return Ok(()); + } + let value = doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?; + self.active.insert(key); + let result = self.walk(doc, &value, inside, depth + 1); + self.active.remove(&key); + result + } + Object::Array(values) => { + for value in values { + self.walk(doc, value, inside, depth + 1)?; + } + Ok(()) + } + Object::Dict(dict) => self.dictionary(doc, dict, inside, false, depth + 1), + Object::Stream(stream) => { + if inside { + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, stream.dict.get(b"PatternType"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + if subtype.as_name() == Some(b"Form") { + if resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null { + return Err(PatternProblem::Unreadable); + } + self.content(doc, stream)?; + } else if pattern_type.as_int() == Some(1) { + self.content(doc, stream)?; + } + } + self.dictionary(doc, &stream.dict, inside, true, depth + 1) + } + _ => Ok(()), + } + } + + fn dictionary(&mut self, doc: &Document, dict: &Dict, inside: bool, stream: bool, depth: usize) -> Result<(), PatternProblem> { + // Stream dictionaries include appearance and soft-mask streams whose consumers + // may infer Form semantics without /Subtype. Resource-map names, in contrast, + // are arbitrary and must not be mistaken for dictionary field names. + let subtype = resolved(doc, dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, dict.get(b"PatternType"))?; + let owns_resources = stream || matches!(subtype.as_name(), Some(b"Form" | b"Type3")) || pattern_type.as_int() == Some(1); + for (key, value) in dict.iter() { + if owns_resources && key.as_slice() == b"Resources" { + self.resources(doc, value, inside, depth + 1)?; + } else if inside || !owns_resources { + self.walk(doc, value, inside, depth + 1)?; + } + } + Ok(()) + } + + fn resources(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + let object = resolved(doc, Some(object))?; + // A Form's null Resources is equivalent to an absent entry: it uses the already + // inspected caller resources. Tiling-pattern resources are required separately. + if object == Object::Null { + return Ok(()); + } + let Some(resources) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (category, value) in resources.iter() { + // Resource dictionaries may include large embedded fonts and colour tables. Only + // structures that can own another pattern cell need recursive inspection. + if !matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Font" | b"Properties" | b"ExtGState") { + continue; + } + if category.as_slice() == b"Font" { + let fonts = resolved(doc, Some(value))?; + let Some(fonts) = fonts.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, font) in fonts.iter() { + self.charge(depth)?; + let font = resolved(doc, Some(font))?; + let Some(dict) = font.as_dict() else { return Err(PatternProblem::Unreadable) }; + if resolved(doc, dict.get(b"Subtype"))?.as_name() == Some(b"Type3") + && let Some(resources) = dict.get(b"Resources") + { + self.resources(doc, resources, inside, depth + 1)?; + } + } + continue; + } + if matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Properties" | b"ExtGState") && resolved(doc, Some(value))? == Object::Null { + continue; + } + if category.as_slice() == b"Pattern" { + let object = resolved(doc, Some(value))?; + let Some(patterns) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, pattern) in patterns.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(pattern))?; + if object == Object::Null { + continue; + } + let Some(dict) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + let kind = resolved(doc, dict.get(b"PatternType"))?; + match (kind.as_int(), &object) { + (Some(1), Object::Stream(_)) => { + // Do not guess a fallback for a malformed cell, which different + // readers may resolve differently. Pattern resources are mandatory. + if resolved(doc, dict.get(b"Resources"))?.as_dict().is_none() { + return Err(PatternProblem::Unreadable); + } + } + (Some(2), Object::Dict(_)) => {} + _ => return Err(PatternProblem::Unreadable), + } + self.walk(doc, pattern, true, depth + 1)?; + } + } else { + if (inside && matches!(category.as_slice(), b"XObject" | b"Properties")) || category.as_slice() == b"ExtGState" { + let object = resolved(doc, Some(value))?; + let Some(entries) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if object == Object::Null { + continue; + } + if category.as_slice() == b"Properties" { + let Some(properties) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + replacement_text(doc, properties)?; + } else if category.as_slice() == b"ExtGState" { + let Some(state) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + match resolved(doc, state.get(b"SMask"))? { + Object::Null => {} + Object::Name(name) if name == b"None" => {} + Object::Dict(mask) => { + let group = resolved(doc, mask.get(b"G"))?; + let Object::Stream(group) = group else { return Err(PatternProblem::Unreadable) }; + readable_form(doc, &group)?; + if let Some(value) = mask.get(b"G") { + self.walk(doc, value, inside, depth + 1)?; + } + } + _ => return Err(PatternProblem::Unreadable), + } + } else { + let Object::Stream(stream) = object else { return Err(PatternProblem::Unreadable) }; + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + readable_form(doc, &stream)?; + } + } + } + if category.as_slice() == b"XObject" { + let entries = resolved(doc, Some(value))?; + let Some(entries) = entries.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if let Object::Stream(stream) = object + && resolved(doc, stream.dict.get(b"Subtype"))?.as_name() == Some(b"Form") + { + self.walk(doc, entry, inside, depth + 1)?; + } + } + } + } + } + Ok(()) + } + + fn content(&mut self, doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // Strict decoding with a small bound, unlike the tolerant general reader. Normalize + // indirect filter declarations before using Stream::filters, which accepts direct data. + let normalized = normalized_stream(doc, stream)?; + let chain = normalized.filters(); + let max = MAX_STREAM_BYTES.min(self.bytes_left); + if chain.is_empty() && normalized.raw.len() > max { + return Err(PatternProblem::Limit); + } + let data = pdfcraft_filters::decode(&chain, &normalized.raw, max).map_err(|error| match error { + pdfcraft_filters::FilterError::LimitExceeded(_) => PatternProblem::Limit, + _ => PatternProblem::Unreadable, + })?; + self.bytes_left = self.bytes_left.checked_sub(data.len()).ok_or(PatternProblem::Limit)?; + let parsed = pdfcraft_content::parse(&data); + if parsed.skipped != 0 { + return Err(PatternProblem::Unreadable); + } + for op in parsed.ops { + self.charge(0)?; + if matches!(op.op.as_slice(), b"BDC" | b"DP") { + for operand in &op.operands { + if let Some(properties) = resolved(doc, Some(operand))?.as_dict() { + replacement_text(doc, properties)?; + } + } + } + match op.op.as_slice() { + b"Tj" | b"TJ" | b"'" | b"\"" | b"BI" => return Err(PatternProblem::Content), + // Unknown operators cannot establish that no hidden content was skipped. + b"q" | b"Q" | b"cm" | b"w" | b"J" | b"j" | b"M" | b"d" | b"ri" | b"i" | b"gs" | b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" + | b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n" | b"W" | b"W*" | b"BT" | b"ET" | b"Tc" | b"Tw" | b"Tz" + | b"TL" | b"Tf" | b"Tr" | b"Ts" | b"Td" | b"TD" | b"Tm" | b"T*" | b"d0" | b"d1" | b"CS" | b"cs" | b"SC" | b"SCN" | b"sc" | b"scn" + | b"G" | b"g" | b"RG" | b"rg" | b"K" | b"k" | b"sh" | b"Do" | b"MP" | b"DP" | b"BMC" | b"BDC" | b"EMC" | b"BX" | b"EX" => {} + _ => return Err(PatternProblem::Unreadable), + } + } + Ok(()) + } +} + +fn readable_form(doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // PS, Form/Subtype2 PS, reference XObjects and unknown/missing subtypes cannot be + // proven free of recoverable pattern content by the PDF content parser. + if resolved(doc, stream.dict.get(b"Subtype"))?.as_name() != Some(b"Form") + || resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null + || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null + { + return Err(PatternProblem::Unreadable); + } + Ok(()) +} + +fn replacement_text(doc: &Document, properties: &Dict) -> Result<(), PatternProblem> { + for key in [b"ActualText".as_slice(), b"Alt", b"E"] { + if resolved(doc, properties.get(key))? != Object::Null { + return Err(PatternProblem::Content); + } + } + Ok(()) +} + +fn resolved(doc: &Document, value: Option<&Object>) -> Result { + let mut value = value.cloned().unwrap_or(Object::Null); + for _ in 0..MAX_DEPTH { + let Object::Ref(reference) = value else { return Ok(value) }; + value = (*doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?).clone(); + } + Err(PatternProblem::Limit) +} + +fn normalized_stream(doc: &Document, stream: &Stream) -> Result { + if stream.dict.contains(b"F") || stream.dict.contains(b"FFilter") || stream.dict.contains(b"FDecodeParms") || stream.dict.contains(b"DP") { + return Err(PatternProblem::Unreadable); + } + let mut normalized = stream.clone(); + let filter = resolved(doc, stream.dict.get(b"Filter"))?; + let names = match &filter { + Object::Null => Vec::new(), + Object::Name(_) => vec![filter], + Object::Array(values) if values.len() <= MAX_FILTERS => values.iter().map(|v| resolved(doc, Some(v))).collect::, _>>()?, + _ => return Err(PatternProblem::Unreadable), + }; + if names.iter().any(|name| name.as_name().is_none()) { + return Err(PatternProblem::Unreadable); + } + let parms = resolved(doc, stream.dict.get(b"DecodeParms"))?; + let parms = match parms { + Object::Null => vec![Object::Null; names.len()], + Object::Dict(_) if names.len() == 1 => vec![parms], + Object::Array(values) if values.len() == names.len() => values, + _ => return Err(PatternProblem::Unreadable), + }; + let parms = parms + .iter() + .map(|value| { + let value = resolved(doc, Some(value))?; + let Object::Dict(mut dict) = value else { + return if value == Object::Null { Ok(value) } else { Err(PatternProblem::Unreadable) }; + }; + for key in [b"Predictor".as_slice(), b"Colors", b"BitsPerComponent", b"Columns", b"EarlyChange"] { + if let Some(value) = dict.get(key) { + let value = resolved(doc, Some(value))?; + if value.as_int().is_none() { + return Err(PatternProblem::Unreadable); + } + dict.set(key.to_vec(), value); + } + } + if !matches!(dict.int(b"Predictor").unwrap_or(1), 1 | 2 | 10..=15) + || dict.int(b"Colors").unwrap_or(1) <= 0 + || dict.int(b"Columns").unwrap_or(1) <= 0 + || !matches!(dict.int(b"BitsPerComponent").unwrap_or(8), 1 | 2 | 4 | 8 | 16) + || !matches!(dict.int(b"EarlyChange").unwrap_or(1), 0 | 1) + { + return Err(PatternProblem::Unreadable); + } + Ok(Object::Dict(dict)) + }) + .collect::, _>>()?; + normalized.dict.set(b"Filter".to_vec(), Object::Array(names)); + normalized.dict.set(b"DecodeParms".to_vec(), Object::Array(parms)); + Ok(normalized) +} + +/// Page-tree nodes a walk of the tree may visit, pages included, before it refuses: the page tree +/// is walked whole elsewhere too (`pdfcraft_model::pages`, `unlink_doomed`), so a long document +/// must not refuse here. +const MAX_TREE_NODES: usize = 1_000_000; + +/// Every node a full save reaches through the page tree -- down `/Kids` and up the `/Parent` of +/// every node visited, page or `/Pages`, so a node reached only through a `/Parent` link counts +/// too -- that has `/Resources` of its own and is not one of the document's pages: those are +/// walked with their effective resources. Cycles are cut by the seen-set. Holders are bounded by +/// `MAX_OBJECTS`, pages are not. +fn tree_holders(doc: &Document) -> Result, PatternProblem> { + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(Vec::new()) }; + let pages: HashSet = pdfcraft_model::pages(doc).iter().map(|p| p.obj).collect(); + // The `/Resources` values to inspect. A direct `/Parent` dictionary is written inline by a + // full save, so it is a node like any other: its resources, `/Parent` and `/Kids` count. + let mut holders = Vec::new(); + let mut seen = HashSet::new(); + seen.insert(top); + let mut stack = vec![top]; + let mut inline: Vec = Vec::new(); + // One node: queue what it links to that was not seen yet (so a /Kids array shared by many + // nodes costs its length once per node, never a copy), and keep its resources. + let mut visit = + |dict: &Dict, is_page: bool, stack: &mut Vec, inline: &mut Vec, holders: &mut Vec| -> Result<(), PatternProblem> { + match dict.get(b"Parent") { + Some(Object::Ref(parent)) => { + if seen.insert(*parent) { + stack.push(*parent); + } + } + Some(Object::Dict(parent)) => { + if inline.len() >= MAX_DEPTH { + return Err(PatternProblem::Limit); + } + inline.push(parent.clone()); + } + _ => {} + } + if let Some(resources) = dict.get(b"Resources") + && !is_page + { + holders.push(resources.clone()); + if holders.len() > MAX_OBJECTS { + return Err(PatternProblem::Limit); + } + } + let is_pages = dict.name(b"Type") == Some(b"Pages") || (dict.contains(b"Kids") && dict.name(b"Type") != Some(b"Page")); + if is_pages { + let kids_object = match dict.get(b"Kids") { + Some(Object::Ref(reference)) => Some(doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?), + _ => None, + }; + let kids = match &kids_object { + Some(object) => object.as_array(), + None => dict.get(b"Kids").and_then(Object::as_array), + }; + for kid in kids.into_iter().flatten().filter_map(Object::as_ref) { + if seen.insert(kid) { + stack.push(kid); + } + } + } + if seen.len() > MAX_TREE_NODES { + return Err(PatternProblem::Limit); + } + Ok(()) + }; + loop { + if let Some(node) = stack.pop() { + let object = doc.try_get(node.num).map_err(|_| PatternProblem::Unreadable)?; + if let Some(dict) = object.as_dict() { + visit(dict, pages.contains(&node), &mut stack, &mut inline, &mut holders)?; + } + } else if let Some(dict) = inline.pop() { + visit(&dict, false, &mut stack, &mut inline, &mut holders)?; + } else { + break; + } + } + Ok(holders) +} + +/// Validate the requested pages before `apply` commits its working copy. With `tree`, the +/// `/Resources` of every page-tree node above the pages is inspected as well: a node no page +/// inherits from, or one a page with a broken `/Parent` link still reaches, can keep a pruned +/// cell in the saved file. +pub(crate) fn check(doc: &Document, selected: &[usize], tree: bool) -> Result<(), RedactError> { + if tree && let Some(&first) = selected.first() { + let page = first.saturating_add(1); + let holders = tree_holders(doc).map_err(|reason| RedactError::Pattern { page, reason })?; + for resources in holders { + let mut guard = Guard::new(); + guard.resources(doc, &resources, false, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + let pages = pdfcraft_model::pages(doc); + for &index in selected { + let Some(page) = pages.get(index) else { return Err(RedactError::Unreadable(index.saturating_add(1))) }; + let mut guard = Guard::new(); + let checked = (|| { + if let Some(resources) = page.dict.get(b"Resources") { + guard.resources(doc, resources, false, 0)?; + } + // Kept appearance streams may own their own pattern resources. + for annotation in crate::annots_of(doc, &page.dict) { + let object = resolved(doc, Some(&annotation))?; + if let Some(appearance) = object.as_dict().and_then(|dict| dict.get(b"AP")) { + guard.walk(doc, appearance, false, 0)?; + } + } + Ok(()) + })(); + checked.map_err(|reason| RedactError::Pattern { page: index.saturating_add(1), reason })?; + } + Ok(()) +} + +/// The backstop after every other check: a full save writes whatever the trailer reaches, by any +/// reference at all -- a stray resource category, a key on a page, a direct /Parent, `/AcroForm +/// /DR`, `/PieceInfo` -- and the walks above follow only the references they know. A pattern +/// pruned from a page that a full save would still write must therefore be proven safe like any +/// retained cell (vector-only), or the redaction is refused. Costs one walk of the reachable +/// objects, and only when a pattern was pruned. +pub(crate) fn check_pruned_unreachable(doc: &Document, pruned: &[ObjRef], page: usize) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let reachable = pdfcraft_cos::reachable_objects(doc)?; + let mut guard = Guard::new(); + let mut checked = HashSet::new(); + for &pattern in pruned { + if reachable.contains(&pattern.num) && checked.insert(pattern.num) { + guard.walk(doc, &Object::Ref(pattern), true, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + Ok(()) +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 78cff889d3..8978e0e7d6 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -290,6 +290,200 @@ fn vectors_are_removed_or_clipped_and_inline_images_go() { assert!(c.contains("150 150 m"), "the line elsewhere stays"); } +/// A tiling pattern's cell is a content stream of its own and can draw text. Redacting a shape +/// filled with such a pattern removed the fill from the page but left the pattern in the page's +/// resources, so its cell -- with the text that was under the mark -- stayed in the saved file +/// (#3647). A pattern nothing paints with any more now leaves the resources; one some surviving +/// paint still uses stays. +#[test] +fn a_pattern_whose_only_painting_was_redacted_leaves_the_page() { + fn page_with(fills: &str) -> Document { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let content = format!("BT /F1 10 Tf 10 280 Td (PUBLIC) Tj ET {fills}"); + one_page(content.as_bytes(), "/Pattern << /P0 7 0 R >>", vec![cell]) + } + let has_pattern = |doc: &Document| { + pdfcraft_model::pages(doc)[0] + .dict + .get(b"Resources") + .and_then(|r| doc.resolve(r).as_dict().cloned()) + .is_some_and(|res| res.get(b"Pattern").is_some()) + }; + + // One box, marked: the pattern is no longer painted, so it goes with the fill. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "the pattern left the page"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + + // Two boxes share a text-bearing cell. The second still needs its pattern, but it is not + // safe to infer that it visibly displays every byte of the cell. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f /Pattern cs /P0 scn 20 40 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal keeps the pending mark and original bytes"); + + // Paint is not only path painting: text shows with the fill pattern too, and the text here + // survives the mark. Dropping the pattern would take the fill away from text still on the + // page, so a pattern is kept for every operator that can paint with it. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert!(has_pattern(&doc), "the refused operation keeps the original pattern"); + assert!(content(&doc, 0).contains("(KEEP)"), "the text is still there: {}", content(&doc, 0)); + + // Text in render mode 3 paints nothing, so it keeps no pattern: the unpainted pattern goes. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT 3 Tr /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "invisible text paints no pattern"); + + // `/Pattern` may be an indirect object, like any value. Reading the reference itself as a + // dictionary found none, so nothing was pruned and the cell stayed in the saved file. + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Pattern cs /P0 scn 20 140 100 30 re f", "/Pattern 8 0 R", vec![cell, b"<< /P0 7 0 R >>".to_vec()]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "an indirect /Pattern is resolved before it is pruned"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); +} + +/// Pruning takes the pattern out of `/Resources` but leaves the `/P0 scn` that selected it, so +/// the saved page names a pattern that is no longer there. ISO 32000-1 (8.6.6.2) says that +/// operand is a key of the `/Pattern` subdictionary, so this is what a reader is handed: the +/// question is whether it stops reading the stream there. Asked in the review of #3647. +#[test] +fn a_page_whose_pruned_pattern_is_still_selected_renders() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + // `0 g` after the fill puts the text back on a plain colour: with the pattern still selected + // the text would paint with it, and a pattern something still paints with is never pruned. + // The text is *after* the selector on purpose -- that is what a reader would lose if it gave + // up on the stale `/P0`. + let content_stream = b"/Pattern cs /P0 scn 20 140 100 30 re f 0 g BT /F1 24 Tf 10 40 Td (PUBLIC) Tj ET"; + let mut doc = one_page(content_stream, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "the pattern is pruned"); + + // The state this test is about: the selector outlived the resource it names. + let after = content(&doc, 0); + assert!(after.contains("/P0 scn"), "the selector is still in the content: {after}"); + assert!(!after.contains("20 140 100 30 re"), "the fill that used it is gone: {after}"); + + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + let mut r = pdfcraft_render::PageRenderer::new(std::sync::Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(out.error.is_none(), "the page still renders: {:?}, warnings {:?}", out.error, out.warnings); + assert_eq!((out.width, out.height), (300, 300), "the page box is unchanged"); + + // The text after the stale selector is still drawn: 24 pt at y 40, so the top-left origin + // puts it in the rows around 240..265. Without it the reader gave up at the bad operand. + let painted = |rows: std::ops::Range, cols: std::ops::Range| { + rows.flat_map(|y| cols.clone().map(move |x| (y, x))).any(|(y, x)| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] != [255, 255, 255] + }) + }; + assert!(painted(236..268, 8..120), "the text after the stale selector is still painted"); + // The mark itself is drawn over the area the pattern filled -- `apply` writes a black + // overlay there (`q 0 0 0 rg 15 135 115 40 re f Q`), so the cell cannot show through it + // even if a reader kept the stale selector. + let black = |y: u32, x: u32| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] == [0, 0, 0] + }; + assert!(black(145, 70), "the redaction overlay is painted over the pattern's box"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_when_pattern_is_used_elsewhere() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"0.9 g 0 0 300 300 re f 0 g BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let content = b"/Pattern cs /P0 scn 20 140 100 30 re f 20 40 100 30 re f"; + let mut doc = one_page(content, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must be atomic"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_on_an_unselected_page() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 8 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 140 100 30 re f"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + cell, + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 9 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 40 100 30 re f"), + ]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, Some(&[0])) { + Ok(_) => { + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the unselected page kept the hidden cell text"); + } + Err(RedactError::Pattern { reason: PatternProblem::Content, .. }) => { + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must keep both pages and the mark"); + } + Err(error) => panic!("unexpected refusal: {error}"), + } +} + +#[test] +fn redacted_form_must_not_retain_its_own_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Pattern << /P0 7 0 R >> >>", + b"/Pattern cs /P0 scn 20 140 100 30 re f", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "form-owned pattern refusal must be atomic"); +} + +#[test] +fn redacted_form_must_not_retain_inherited_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"20 140 100 30 re f"); + let mut doc = one_page(b"/Pattern cs /P0 scn /Fm Do", "/Pattern << /P0 7 0 R >> /XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "inherited pattern refusal must be atomic"); +} + #[test] fn partially_covered_vector_paths_do_not_leak_drawing_commands() { // A vector path with detailed drawing commands (e.g. outlined text or confidential geometry) diff --git a/crates/redact/tests/pattern_page_tree.rs b/crates/redact/tests/pattern_page_tree.rs new file mode 100644 index 0000000000..c2b68a4485 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree.rs @@ -0,0 +1,221 @@ +//! Pattern redaction through the page tree and through glyphs: synthetic files whose pattern +//! cells could stay reachable, or whose surviving paint could lose its pattern, after Apply. +//! Each test reads the saved bytes back (full and incremental, with and without object +//! streams) and searches every decoded stream. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +/// Public text outside the mark, and a pattern fill wholly under it. +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +/// The needle in any decoded stream, with and without object streams, and in an incremental save. +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// A successful apply must leave no trace of the cell; a refusal must leave the document as it was. +fn assert_confidential_or_refused(mut doc: Document) { + mark(&mut doc, 0); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The page inherits its resources, but its `/Parent` does not lead back to the node it inherits +/// from (missing here). Readers resolve inheritance down the `/Kids` tree, so the page draws with +/// the root's `/Pattern`; that root keeps its `/Resources` after the page's copy is pruned. +#[test] +fn a_page_whose_parent_link_is_missing_does_not_keep_the_pruned_cell_in_the_page_tree() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + assert_confidential_or_refused(doc); +} + +/// As above with a `/Parent` that names a different, resource-less `/Pages` node. +#[test] +fn a_page_whose_parent_link_disagrees_with_the_tree_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + ]); + assert_confidential_or_refused(doc); +} + +/// Two ancestors both hold `/Resources` naming the pattern; the page inherits from the nearer one, +/// which is retired once the page gets its own pruned copy. The root's dictionary, which no page +/// inherits, still names the cell. +#[test] +fn a_shadowed_ancestor_resource_dictionary_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + // The tree is consistent and no page inherits the root's dictionary any more, so the pruned + // pattern is unlinked from it and the redaction applies rather than refusing. + let mut doc = doc; + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "the root's shadowed /Resources still names the cell ({report:?})"); +} + +/// Control: the same shape with only the nearer ancestor holding resources applies cleanly. +#[test] +fn control_a_single_inheriting_ancestor_is_retired_with_the_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert_confidential_or_refused(doc); +} + +/// A hostile chain of tiling patterns, each cell selecting the next, read from a file: the guard +/// must stop with an error (or apply), never overflow the stack. +#[test] +fn a_hostile_chain_of_nested_patterns_is_bounded() { + const N: usize = 3000; + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + // The fill outside the mark keeps /P0 in use, so the guard must inspect the chain. + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f /P0 scn 300 200 100 40 re f\n"), + ]; + for i in 0..N { + let num = 6 + i; + let (res, body) = if i + 1 < N { + (format!("/Resources << /Pattern << /Q {} 0 R >> >>", num + 1), b"/Pattern cs /Q scn 0 0 10 10 re f\n".to_vec()) + } else { + ("/Resources << /Font << /F1 4 0 R >> >>".to_string(), TEXT_CELL.to_vec()) + }; + objs.push(stream_obj(&format!("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 {res}"), body)); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Limit | PatternProblem::Content, .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A Type 3 font without `/Resources` draws its glyph with the page's `/P0` (ISO 32000-1 9.6.5: +/// names are then looked up in the page's resources). The glyph is outside the mark; the page's +/// own fill with `/P0` is under it. Pruning `/P0` loses the glyph's paint. +#[test] +fn a_type3_glyph_without_resources_keeps_the_page_pattern_it_paints_with() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, b"0 0 1 rg 0 0 500 300 re f\n"), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>".to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]); + let mut doc = doc; + let blue_glyph = |doc: &Document| { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + // The glyph covers x 300..340, y 230..270 (rows 30..70 from the top). + let i = ((50 * out.width + 320) * 4) as usize; + out.rgba[i..i + 3].to_vec() + }; + let before = blue_glyph(&doc); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(blue_glyph(&doc), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} + +/// Text, inline images and XObjects record their fill pattern as they are met, before the hit +/// test, so a pattern whose only paint was text wholly under the mark is kept, and so is its cell. +/// That is the documented direction of error (keeping paint over losing it): a cell that carries +/// text or images is then refused by the guard, so what can stay is vector drawing nothing shows. +#[test] +fn a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell() { + const DRAWING: &[u8] = b"0.1 0.2 0.3 rg 111.11 22.22 m 133.33 44.44 l 155.55 22.22 l f\n"; + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nBT /F1 20 Tf /Pattern cs /P0 scn 40 120 Td (HIDDEN) Tj ET\n"), + stream_obj(PATTERN, DRAWING), + ]); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert!(report.glyphs >= 6, "{report:?}"); + assert_eq!(report.patterns_removed, 0, "text paint keeps its pattern"); + assert!(!saved_contains(&doc, b"(HIDDEN)"), "the glyphs are gone"); + assert!(saved_contains(&doc, b"133.33 44.44 l"), "the vector cell is kept, as documented"); +} diff --git a/crates/redact/tests/pattern_page_tree_shapes.rs b/crates/redact/tests/pattern_page_tree_shapes.rs new file mode 100644 index 0000000000..dafedd56b4 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree_shapes.rs @@ -0,0 +1,275 @@ +//! Pattern redaction over unusual page trees -- cyclic, deep, wide, inconsistent, packed in +//! object streams, or reached only through `/Parent` -- and over Type 3 fonts reached other than +//! by `Tf` in the scope that paints. Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Round-trip through a full save with object streams, so every dictionary lives in one. +fn packed(doc: &Document) -> Document { + Document::open(Arc::new(write_full(doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap() +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The root `/Pages` node carries a `/Parent` to a node outside the `/Kids` tree that names the +/// pattern. A full save follows that `/Parent`, so the node and the cell are written. +#[test] +fn a_node_reached_only_through_the_roots_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 7 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// As above, one level down: an intermediate node in `/Kids` whose `/Parent` leads elsewhere. +#[test] +fn a_node_reached_only_through_an_intermediate_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [8 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 8 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Pages /Parent 7 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// `/Parent` cycles: the root's parent is itself, and the page's chain loops through it. +#[test] +fn a_parent_cycle_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 2 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A page 5,000 levels below the root that holds the pattern (past `resource_source`'s 4,096). +#[test] +fn a_very_deep_tree_is_confidential_or_refused() { + const DEPTH: usize = 5000; + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Page /Parent {} 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>", 7 + DEPTH - 1).into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for i in 0..DEPTH { + let num = 7 + i; + let parent = if i == 0 { 2 } else { num - 1 }; + let kid = if i + 1 == DEPTH { 3 } else { num + 1 }; + objs.push(format!("<< /Type /Pages /Parent {parent} 0 R /Kids [{kid} 0 R] /Count 1 >>").into_bytes()); + } + confidential_or_refused(doc_or(objs), 0); +} +fn doc_or(objs: Vec>) -> Document { + open(objs) +} +/// The shadowed-root and missing-parent shapes again, with every dictionary in object streams. +#[test] +fn page_tree_shapes_in_object_streams_are_confidential_or_refused() { + let shadowed = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert!(confidential_or_refused(packed(&shadowed), 0), "the shadowed root applies"); + let missing = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(packed(&missing), 0); +} +/// The shadowed root's `/Resources` object is also a form's resources, drawn outside the mark on +/// the redacted page: unlinking from the root must not reach through to the form. +#[test] +fn a_root_resource_dictionary_shared_with_a_form_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources 9 0 R >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\nq 1 0 0 1 300 200 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm 8 0 R >> >> >>".to_vec(), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 50] /Resources 9 0 R", b"0 0 1 rg 0 0 50 20 re f\n"), + RES.as_bytes().to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// A document of 17,000 pages whose first page names a pattern: only that page has patterns. +/// Redacting it should apply as it does in a short document. +#[test] +fn a_long_document_with_a_pattern_on_one_page_still_applies() { + const PAGES: usize = 17_000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let result = apply(&mut doc, None); + assert!(result.is_ok(), "a long document refuses: {result:?}"); + assert!(!saved_contains(&doc, SECRET)); +} +/// Control for the above: the same with 1,000 pages. +#[test] +fn control_a_shorter_document_with_a_pattern_on_one_page_applies() { + const PAGES: usize = 1000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + assert!(apply(&mut doc, None).is_ok()); + assert!(!saved_contains(&doc, SECRET)); +} +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The Type 3 font without `/Resources` is selected through an ExtGState `/Font` entry, not `Tf`. +#[test] +fn a_type3_font_set_by_extgstate_keeps_the_page_pattern_it_paints_with() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /ExtGState << /G 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/G gs BT 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + b"<< /Type /ExtGState /Font [7 0 R 1] >>".to_vec(), + ]); + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font without `/Resources` is shown inside a form that has its own `/Resources` +/// (with no `/Pattern`); the page paints `/P0` under the mark. +#[test] +fn a_type3_glyph_in_a_form_with_resources_renders_the_same_after_the_redaction() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph renders differently ({report:?}, before {before:?})"); +} diff --git a/crates/redact/tests/pattern_reachability.rs b/crates/redact/tests/pattern_reachability.rs new file mode 100644 index 0000000000..9ca4be0c27 --- /dev/null +++ b/crates/redact/tests/pattern_reachability.rs @@ -0,0 +1,304 @@ +//! Pattern redaction against references no walk follows -- a direct `/Parent`, `/Kids` on a +//! page, a stray resource entry -- Type 3 fonts reached in other ways, and hostile wide trees. +//! Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} + +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The root's `/Parent` is a direct dictionary, not a reference: a full save writes it inline +/// with the root, and the pattern it names with it. +#[test] +fn a_direct_parent_dictionary_on_the_root_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent << /Type /Pages /Kids [] /Count 0 /Resources {RES} >> /Resources {RES} >>") + .into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A leaf (`/Type /Page`) that also carries `/Kids`: readers stop at the leaf, a full save does +/// not, and the hidden node names the pattern. +#[test] +fn a_node_hidden_under_a_leaf_s_kids_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} /Kids [7 0 R] >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 3 0 R /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// A stray reference to the cell from a resource category the guard does not inspect +/// (`/Shading`): the class #3999 tracks, shown here for patterns. +#[test] +fn a_stray_resource_reference_to_the_cell_does_not_keep_it() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +fn type3_page(page_res_extra: &str, content: &[u8], extra: Vec>) -> Document { + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> {page_res_extra} >> >>").into_bytes(), + FONT.to_vec(), + stream_obj("", content), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]; + objs.extend(extra); + open(objs) +} +fn assert_glyph_kept(mut doc: Document) { + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font is shown in a form without `/Resources`, which draws through the page's. +#[test] +fn a_type3_glyph_in_a_resourceless_form_keeps_the_page_pattern() { + assert_glyph_kept(type3_page( + "/XObject << /Fm 9 0 R >>", + b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n")], + )); +} +/// The Type 3 font is shown inside a pattern cell the page paints outside the mark: the cell +/// carries text, so the guard refuses rather than dropping anything. +#[test] +fn a_type3_glyph_inside_a_pattern_cell_is_refused() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R /P1 9 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Pattern cs /P1 scn 300 200 100 40 re f\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A hostile wide tree: `K` `/Pages` nodes that all share one `/Kids` array of `M` references +/// back to themselves, beside the real page. Time and memory are measured from outside +/// (`PDF_WIDE_K`, `PDF_WIDE_M`). Ignored by default. +#[test] +#[ignore] +fn hostile_wide_tree_probe() { + let k: usize = std::env::var("PDF_WIDE_K").ok().and_then(|v| v.parse().ok()).unwrap_or(200); + let m: usize = std::env::var("PDF_WIDE_M").ok().and_then(|v| v.parse().ok()).unwrap_or(500_000); + let mut array = String::with_capacity(m * 9); + for i in 0..m { + array.push_str(&format!("{} 0 R ", 10 + i % k)); + } + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R 10 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("[{array}]").into_bytes(), + b"null".to_vec(), + b"null".to_vec(), + ]; + for _ in 0..k { + objs.push(b"<< /Type /Pages /Kids 7 0 R /Count 0 >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let start = std::time::Instant::now(); + let result = apply(&mut doc, None); + eprintln!("WIDE k={k} m={m} apply={:?} in {:?}", result.as_ref().map(|r| r.patterns_removed), start.elapsed()); +} +/// The Type 3 font has `/Resources null`, which readers treat as absent. +#[test] +fn a_type3_font_with_null_resources_keeps_the_page_pattern() { + let mut doc = type3_page("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n", vec![]); + doc.update_dict(pdfcraft_cos::ObjRef::new(7, 0), |d| d.set(b"Resources".to_vec(), Object::Null)).unwrap(); + assert_glyph_kept(doc); +} +/// Known limit (fidelity only, never a leak: a missed use can only prune more). Kept annotation +/// appearances are not interpreted for pattern use, so a Type 3 font without resources shown in +/// one does not keep the page's patterns. ISO 32000 resolves the glyph's names in the +/// appearance's own resources, where `/P0` does not exist, so only a lenient renderer draws it. +/// The Type 3 font is shown by a kept annotation's appearance stream (outside the mark). +#[test] +#[ignore = "known limit: kept annotation appearances are not interpreted, so a resourceless Type 3 glyph in one does not keep the page pattern; ISO 32000 resolves it in the appearance's own resources, which pdfcraft's renderer does not"] +fn a_type3_glyph_in_an_annotation_appearance_renders_the_same() { + let mut doc = type3_page( + "", + b"/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![ + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 40 40] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 0 0 Td (a) Tj ET\n"), + b"<< /Type /Annot /Subtype /Stamp /Rect [300 230 340 270] /F 4 /AP << /N 9 0 R >> >>".to_vec(), + ], + ); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |d| { + d.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(pdfcraft_cos::ObjRef::new(10, 0))])) + }) + .unwrap(); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the annotation renders differently ({report:?}, before {before:?})"); +} +/// A form wholly under the mark is removed; its own `/Pattern` cell carries text. Nothing prunes +/// that pattern, so the backstop never looks, and a stray reference (`/Shading`) keeps the cell. +#[test] +fn a_removed_form_s_pattern_does_not_survive_through_a_stray_reference() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nq 1 0 0 1 35 110 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 170 35] /Resources << /Pattern << /P0 6 0 R >> >>", b"/Pattern cs /P0 scn 0 0 170 35 re f\n"), + ]); + let mut doc = doc; + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the removed form's cell is in the saved file ({report:?})"), + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + } + } +} +/// `/P0` names an indirect object whose value is a reference to the cell; the cell is also named +/// by a stray reference. The backstop checks the number it pruned, not what it points at. +#[test] +fn a_pruned_pattern_named_through_a_reference_object_does_not_dodge_the_backstop() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"6 0 R".to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// The backstop's reachability is a full save's: after Apply an incremental save must be that +/// full save, byte for byte, never an appended revision over the old bytes. +#[test] +fn an_incremental_save_after_apply_is_the_full_save() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + let mut doc = doc; + mark(&mut doc, 0); + apply(&mut doc, None).unwrap(); + for object_streams in [true, false] { + // A redacted rewrite gets a new identifier from fresh entropy; pin it so the two saves + // can be compared byte for byte. + let opts = SaveOptions { object_streams, id_entropy: Some([7; 16]), ..SaveOptions::default() }; + assert_eq!(write_incremental(&doc, &opts).unwrap(), write_full(&doc, &opts).unwrap()); + } + // A later edit after a save-and-reopen starts from the clean file. + let reopened = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!reopened.bytes().windows(SECRET.len()).any(|w| w == SECRET)); +} diff --git a/crates/redact/tests/pattern_redaction.rs b/crates/redact/tests/pattern_redaction.rs new file mode 100644 index 0000000000..4fd898c9ff --- /dev/null +++ b/crates/redact/tests/pattern_redaction.rs @@ -0,0 +1,758 @@ +//! Saved-output confidentiality regressions for #3647, using only synthetic PDFs. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"CELLSECRET"; +const CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; + +fn fixture(cell: &[u8], compressed: bool, partial: bool, inherited: bool) -> Document { + let resources = "/Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; + let parent_res = if inherited { resources } else { "" }; + let own_res = if inherited { "" } else { resources }; + let width = if partial { 400 } else { 170 }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 {width} 35 re f\n"); + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 {parent_res} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R {own_res} >>").into_bytes(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", content.as_bytes()), + stream_obj(PATTERN, cell), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + if compressed { + let pattern = doc.get(ObjRef::new(6, 0)); + let Object::Stream(pattern) = &*pattern else { panic!("fixture pattern must be a stream") }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(pattern.dict.clone(), cell))); + } + doc +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn saved(doc: &Document) -> Vec { + write_full(doc, &SaveOptions::default()).unwrap() +} + +fn has_decoded_secret(bytes: Vec) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap().windows(SECRET.len()).any(|w| w == SECRET)) + }) +} + +/// Success must remove the payload from the saved object graph. Refusal must be atomic even +/// for the direct library caller, not just the engine's working clone. +fn assert_confidential_or_refused(mut doc: Document, expect_refusal: bool) { + mark(&mut doc, 0); + let before = saved(&doc); + assert!(has_decoded_secret(before.clone()), "the fixture must carry the secret"); + match apply(&mut doc, None) { + Ok(report) => { + assert!(!expect_refusal, "a retained text-bearing cell must refuse"); + assert_eq!(report.patterns_removed, 1); + assert!(!has_decoded_secret(saved(&doc)), "successful redaction retained the pattern cell's secret in the saved PDF"); + } + Err(error) => { + assert!(expect_refusal, "a fully removed pattern should apply: {error}"); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Content, .. }), "{error}"); + assert_eq!(saved(&doc), before, "a refused pattern must leave the entire document unchanged"); + assert_eq!(marks(&doc).len(), 1, "the pending mark must survive refusal"); + } + } +} + +#[test] +fn pattern_whole_fill_saved_output_is_confidential_or_refused() { + for compressed in [false, true] { + assert_confidential_or_refused(fixture(CELL, compressed, false, false), false); + } +} + +#[test] +fn pattern_partial_fill_saved_output_is_confidential_or_refused() { + // The mark hits part of the patterned path, which is removed whole (F-008), so nothing paints + // with the pattern any more: it is pruned with its cell, and the redaction is confidential + // rather than refused. Once a hit path was clipped instead, its paint kept the cell in use. + let cell = [CELL, b"BT /F1 20 Tf 300 120 Td (SAFE) Tj ET\n"].concat(); + for compressed in [false, true] { + assert_confidential_or_refused(fixture(&cell, compressed, true, false), false); + } +} + +#[test] +fn pattern_inherited_resources_saved_output_is_confidential_or_refused() { + assert_confidential_or_refused(fixture(CELL, true, false, true), false); +} + +#[test] +fn pattern_vector_only_control_still_applies_and_preserves_the_cell() { + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let mut doc = fixture(cell, true, true, false); + // The guard's control: a plain vector cell some surviving paint still uses is inspected and + // kept, not refused. The hit path is removed whole (F-008), so keep another paint in use. + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!((report.paths_removed, report.paths_clipped, report.patterns_removed), (1, 0, 0)); + assert!(marks(&doc).is_empty()); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + let pattern = reopened.resolve(patterns.get(b"P0").unwrap()); + let Object::Stream(pattern) = &*pattern else { panic!("pattern must be preserved") }; + assert_eq!(pattern.decoded().unwrap(), cell); +} + +#[test] +fn unrelated_font_widths_do_not_exhaust_pattern_budget() { + let mut doc = fixture(b"40 120 15 15 re f\n", false, true, false); + doc.update_dict(ObjRef::new(4, 0), |font| { + font.set(b"W".to_vec(), Object::Array(vec![Object::Int(500); 17_000])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn shared_annotation_appearance_on_unselected_page_is_refused() { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + for page in pages { + let annotation = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + } + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_on_second_page_does_not_change_the_first_page() { + let mut doc = two_pages(); + // The second page's pattern must stay in use for it to refuse: its hit path alone would be + // removed whole (F-008) and the pattern pruned. + keep_pattern_in_use(&mut doc, 1); + mark(&mut doc, 0); + mark(&mut doc, 1); + let before = saved(&doc); + let result = apply(&mut doc, None); + assert!(result.is_err(), "text-carrying pattern must refuse the whole apply"); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 2); +} + +fn two_pages() -> Document { + let mut doc = fixture(CELL, false, true, false); + let source_page = pdfcraft_model::pages(&doc)[0].dict.clone(); + let mut clean = source_page.clone(); + let mut clean_resources = doc.dict(clean.get(b"Resources").unwrap()).unwrap(); + clean_resources.remove(b"Pattern"); + clean.set(b"Resources".to_vec(), Object::Dict(clean_resources)); + let content = doc.add(Object::Stream(Stream::from_raw(Default::default(), b"BT /F1 20 Tf 40 120 Td (DIRECTSECRET) Tj ET".to_vec()))); + clean.set(b"Contents".to_vec(), Object::Ref(content)); + let first = doc.add(Object::Dict(clean)); + doc.update_dict(ObjRef::new(2, 0), |pages| { + pages.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(3, 0))])); + pages.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + doc +} + +fn pattern_stream(doc: &Document) -> Stream { + let object = doc.get(ObjRef::new(6, 0)); + let Object::Stream(stream) = &*object else { panic!("fixture pattern must be a stream") }; + stream.clone() +} + +/// Paint with the pattern once more, well away from the mark. A path the mark hits is removed +/// whole (F-008), and a pattern nothing surviving paints with is pruned along with its cell, so +/// the guard only ever inspects a pattern that some surviving paint still uses. This keeps one, +/// so the guard has to inspect this cell rather than the pattern simply leaving the file. +fn keep_pattern_in_use(doc: &mut Document, page: usize) { + let page = pdfcraft_model::pages(doc).swap_remove(page); + let extra = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"/Pattern cs /P0 scn 300 200 100 40 re f".to_vec()))); + let mut list = match page.dict.get(b"Contents") { + Some(Object::Array(items)) => items.clone(), + Some(other) => vec![other.clone()], + None => Vec::new(), + }; + list.push(Object::Ref(extra)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(list))).unwrap(); +} + +fn assert_refused(mut doc: Document, reason: PatternProblem) { + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, None).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 1, reason }); + assert!(!error.to_string().contains("CELLSECRET"), "errors must not disclose page text"); + assert_eq!(saved(&doc), before, "even direct callers must be unchanged"); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_does_not_block_redacting_other_selected_pages() { + let mut doc = two_pages(); + mark(&mut doc, 0); + mark(&mut doc, 1); + let report = apply(&mut doc, Some(&[0])).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!(marks(&doc).len(), 1); + assert_eq!(marks(&doc)[0].page, 1); + assert!(has_decoded_secret(saved(&doc)), "the unselected page must remain intact"); +} + +#[test] +fn pattern_nested_form_and_nested_pattern_text_are_refused() { + for nested_pattern in [false, true] { + let mut doc = fixture(CELL, false, true, false); + let mut inner = pattern_stream(&doc); + if !nested_pattern { + inner.dict.remove(b"PatternType"); + inner.dict.set(b"Type".to_vec(), Object::name("XObject")); + inner.dict.set(b"Subtype".to_vec(), Object::name("Form")); + } + let reference = doc.add(Object::Stream(inner)); + let resources = Dict::from_iter([( + if nested_pattern { b"Pattern".to_vec() } else { b"XObject".to_vec() }, + Object::Dict(Dict::from_iter([(b"Inner".to_vec(), Object::Ref(reference))])), + )]); + let mut outer = pattern_stream(&doc); + outer.dict.set(b"Resources".to_vec(), Object::Dict(resources)); + let content = if nested_pattern { b"/Pattern cs /Inner scn 35 110 400 35 re f".as_slice() } else { b"/Inner Do" }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(outer.dict, content))); + assert_refused(doc, PatternProblem::Content); + } +} + +#[test] +fn pattern_inside_form_resources_is_refused() { + let mut doc = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&doc)[0].clone(); + let resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(resources)), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let resources = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Fm".to_vec(), Object::Ref(form))])))]); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"/Fm Do".to_vec()))); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_images_and_uncolored_text_are_refused() { + let inline = b"q 10 0 0 10 40 120 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q"; + assert_refused(fixture(inline, true, true, false), PatternProblem::Content); + let mut uncolored = fixture(CELL, false, true, false); + uncolored.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"PaintType".to_vec(), Object::Int(2))).unwrap(); + assert_refused(uncolored, PatternProblem::Content); +} + +#[test] +fn pattern_indirect_filter_and_parameters_are_decoded_not_skipped() { + let mut doc = fixture(CELL, true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let predictor = doc.add(Object::Int(1)); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Ref(predictor))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Ref(filter)); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_damaged_filters_and_unparsed_content_fail_closed() { + for bad in [b"BT (unterminated".as_slice(), b"] BT /F1 20 Tf (CELLSECRET) Tj ET", b"BX UnknownPaint EX"] { + assert_refused(fixture(bad, false, true, false), PatternProblem::Unreadable); + } + let mut damaged = fixture(CELL, true, true, false); + let stream = pattern_stream(&damaged); + let cut = stream.raw.get(..stream.raw.len() / 2).unwrap().to_vec(); + damaged.set(ObjRef::new(6, 0), Object::Stream(Stream::from_raw(stream.dict, cut))); + assert_refused(damaged, PatternProblem::Unreadable); + let mut parameters = fixture(CELL, true, true, false); + parameters + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"DecodeParms".to_vec(), Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(99))]))); + }) + .unwrap(); + assert_refused(parameters, PatternProblem::Unreadable); +} + +#[test] +fn pattern_oversized_decoding_and_resource_cycles_fail_closed() { + let large = vec![b' '; (1 << 20) + 1]; + assert_refused(fixture(&large, true, true, false), PatternProblem::Limit); + let mut cycle = fixture(b"0 0 20 20 re f", false, true, false); + cycle + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"Self".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ); + }) + .unwrap(); + assert_refused(cycle, PatternProblem::Unreadable); +} + +#[test] +fn pattern_plain_nested_form_control_still_applies() { + let mut doc = fixture(b"/Plain Do", false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"40 120 15 15 re f".to_vec(), + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Plain".to_vec(), Object::Ref(form))])))])), + ); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + assert!(marks(&doc).is_empty()); +} + +#[test] +fn pattern_unused_and_annotation_appearance_resources_are_checked() { + let mut unused = fixture(CELL, false, false, false); + unused.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + assert_refused(unused, PatternProblem::Content); + + let mut appearance = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&appearance)[0].clone(); + let mut resources = appearance.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = resources.remove(b"Pattern").unwrap(); + appearance.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + appearance.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + let ap = appearance.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), patterns)]))), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let annotation = appearance.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 250.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + appearance.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + let mut implicit = appearance.clone(); + implicit + .update_dict(ap, |dict| { + dict.remove(b"Subtype"); + }) + .unwrap(); + assert_refused(implicit, PatternProblem::Content); + assert_refused(appearance, PatternProblem::Content); +} + +#[test] +fn pattern_image_xobjects_and_missing_resources_are_refused() { + let mut doc = fixture(b"/Picture Do", false, true, false); + let image = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Image")), + (b"Width".to_vec(), Object::Int(1)), + (b"Height".to_vec(), Object::Int(1)), + (b"ColorSpace".to_vec(), Object::name("DeviceGray")), + (b"BitsPerComponent".to_vec(), Object::Int(8)), + ]), + vec![0], + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Picture".to_vec(), Object::Ref(image))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); + let mut missing = fixture(b"0 0 20 20 re f", false, true, false); + missing + .update_dict(ObjRef::new(6, 0), |dict| { + dict.remove(b"Resources"); + }) + .unwrap(); + assert_refused(missing, PatternProblem::Unreadable); +} + +#[test] +fn pattern_indirect_vector_filter_control_still_applies() { + let mut doc = fixture(b"40 120 15 15 re f", true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(1))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::Ref(filter)])); + dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Ref(parms)])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn pattern_depth_and_total_decoded_budgets_fail_closed() { + let mut deep = fixture(b"0 0 20 20 re f", false, true, false); + // Two frames a level (the array, then the reference): 300 levels run past the 512-frame + // depth budget long before the object budget, so it is the depth budget that refuses, and it + // does so without overflowing the stack. + let mut nested = Object::Int(1); + for _ in 0..300 { + nested = Object::Array(vec![Object::Ref(deep.add(nested))]); + } + deep.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Extra".to_vec(), nested)).unwrap(); + assert_refused(deep, PatternProblem::Limit); + + let mut total = fixture(b"0 0 20 20 re f", false, true, false); + let blank = vec![b' '; 1 << 20]; + let template = pattern_stream(&total).dict; + let mut patterns = Dict::new(); + for index in 0..9 { + let cell = total.add(Object::Stream(Stream::flate(template.clone(), &blank))); + patterns.set(format!("P{index}").into_bytes(), Object::Ref(cell)); + } + total + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Dict(patterns))]))); + }) + .unwrap(); + assert_refused(total, PatternProblem::Limit); +} + +#[test] +fn pattern_replacement_text_without_text_showing_is_refused() { + assert_refused(fixture(b"/Span << /ActualText (CELLSECRET) >> BDC 40 120 20 20 re f EMC", false, true, false), PatternProblem::Content); + let mut named = fixture(b"/Span /Description BDC 40 120 20 20 re f EMC", true, true, false); + named + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Properties".to_vec(), + Object::Dict(Dict::from_iter([( + b"Description".to_vec(), + Object::Dict(Dict::from_iter([(b"Alt".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"CELLSECRET".to_vec())))])), + )])), + )])), + ); + }) + .unwrap(); + assert_refused(named, PatternProblem::Content); +} + +#[test] +fn pattern_resource_names_are_not_dictionary_categories_or_descriptions() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for has_pattern in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let font = Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(ObjRef::new(4, 0)))])); + if has_pattern { + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + } else { + doc.update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"35 110 400 35 re f".to_vec()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "resource named {name}, pattern: {has_pattern}"); + } + } +} + +#[test] +fn pattern_opaque_and_external_xobjects_are_refused() { + for kind in ["PS", "Bogus", "missing", "Form-PS", "Form-Ref"] { + let mut doc = fixture(b"/X Do", false, true, false); + let mut dict = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject"))]); + match kind { + "missing" => {} + "Form-PS" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set(b"Subtype2".to_vec(), Object::name("PS")); + } + "Form-Ref" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set( + b"Ref".to_vec(), + Object::Dict(Dict::from_iter([(b"F".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"external.pdf".to_vec())))])), + ); + } + subtype => { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + } + // Reference XObjects can have an empty fallback: that cannot establish safety of + // the external representation. No external file is read by this test or the guard. + let data = if kind == "Form-Ref" { b"".as_slice() } else { b"/Helvetica findfont 20 scalefont setfont (CELLSECRET) show" }; + let object = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"X".to_vec(), Object::Ref(object))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Unreadable); + } +} + +#[test] +fn pattern_named_form_resources_still_apply() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for inside in [false, true] { + let content = format!("/{name} Do"); + let mut doc = fixture(content.as_bytes(), false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"35 110 400 35 re f".to_vec(), + ))); + let resources = Object::Dict(Dict::from_iter([( + b"XObject".to_vec(), + Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(form))])), + )])); + doc.update_dict(ObjRef::new(if inside { 6 } else { 3 }, 0), |dict| dict.set(b"Resources".to_vec(), resources)).unwrap(); + if !inside { + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "form named {name}, inside: {inside}"); + } + } +} + +#[test] +fn pattern_soft_mask_groups_are_checked_by_resource_context() { + for subtype in [None, Some("Bogus"), Some("Form")] { + for text in [false, true] { + let mut doc = fixture(b"q /GS gs 40 120 20 20 re f Q", false, true, false); + let mut dict = Dict::from_iter([ + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Group".to_vec(), Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Transparency"))]))), + (b"Resources".to_vec(), Object::Dict(Dict::new())), + ]); + if let Some(subtype) = subtype { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + let group = doc.add(Object::Stream(Stream::from_raw(dict, if text { CELL.to_vec() } else { b"40 120 20 20 re f".to_vec() }))); + let state = Object::Dict(Dict::from_iter([( + b"SMask".to_vec(), + Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Luminosity")), (b"G".to_vec(), Object::Ref(group))])), + )])); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"ExtGState".to_vec(), Object::Dict(Dict::from_iter([(b"GS".to_vec(), state)])))])), + ); + }) + .unwrap(); + match (subtype, text) { + (Some("Form"), false) => { + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + (Some("Form"), true) => assert_refused(doc, PatternProblem::Content), + _ => assert_refused(doc, PatternProblem::Unreadable), + } + } + } +} + +#[test] +fn pattern_null_resource_categories_and_entries_are_absent() { + for null_entries in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let mut resources = Dict::new(); + for category in ["Pattern", "XObject", "Properties", "ExtGState"] { + let value = if null_entries { Object::Dict(Dict::from_iter([(b"Unused".to_vec(), Object::Null)])) } else { Object::Null }; + resources.set(category.as_bytes().to_vec(), value); + } + doc.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + let mut plain = fixture(b"", false, false, false); + plain + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Null)]))); + }) + .unwrap(); + plain.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"40 120 20 20 re f".to_vec()))); + mark(&mut plain, 0); + assert_eq!(apply(&mut plain, None).unwrap().marks, 1); +} + +#[test] +fn a_pattern_whose_only_paint_was_redacted_leaves_the_file_with_its_cell() { + // The mark hits the patterned path, which is removed whole (F-008). Nothing paints with the + // pattern any more, so it leaves the page's resources and its cell -- text included -- leaves + // the saved file: the redaction applies rather than refusing. + for compressed in [false, true] { + let mut doc = fixture(CELL, compressed, true, false); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "compressed: {compressed}"); + assert!(!has_decoded_secret(saved(&doc)), "the cell's text is not in the saved file (compressed: {compressed})"); + } +} + +#[test] +fn a_form_without_resources_keeps_the_page_pattern_it_still_paints() { + // The page fills a box under the mark with /P0, then draws a form that has no /Resources of + // its own, so it paints with the page's: its /P0 fill lies outside the mark and survives. + // The page's own fill is removed whole, but the pattern must stay for the form's. + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm0 7 0 R >> >> >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f\n0 g\n/Fm0 Do\n"), + stream_obj(PATTERN, cell), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"/Pattern cs /P0 scn 300 200 100 40 re f\n"), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!((report.paths_removed, report.patterns_removed), (1, 0)); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + assert!(patterns.get(b"P0").is_some(), "the form still paints with /P0 outside the mark"); +} + +#[test] +fn a_shared_appearance_of_a_reply_or_pop_up_removed_on_another_page_is_refused() { + // The marked comment on page 1 has no appearance; its reply (or pop-up) is listed on page 2, + // goes with it, and shares its appearance with a stamp that stays on page 2. That stamp keeps + // the text-bearing cell reachable, so the all-pages check must run and refuse. + for link in ["IRT", "Parent"] { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let comment = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Text")), + (b"Rect".to_vec(), Object::Array(vec![40.into(), 110.into(), 60.into(), 130.into()])), + ]))); + let follower = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name(if link == "IRT" { "Text" } else { "Popup" })), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 260.into()])), + (link.as_bytes().to_vec(), Object::Ref(comment)), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + let stamp = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(pages[0].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(comment)]))).unwrap(); + doc.update_dict(pages[1].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(follower), Object::Ref(stamp)]))).unwrap(); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }, "/{link}"); + assert_eq!(saved(&doc), before, "/{link}"); + assert_eq!(marks(&doc).len(), 1, "/{link}"); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 023688ddc4..736d53bb7a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -245,7 +245,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. Renaming a field renames it in the calculations that read it (field lists, notation, literal `getField` names), in one undo step, and says how many changed. Prepare a form ▸ Calculation order, a page of its own: each calculated field's formula and value, what is wrong (fields that calculate too early, loops, fields missing from `/CO`) and Fix order, Up/Down, Alt+↑/↓ and drag, and on the page each field's step and arrows to and from the field in focus (`form_calculation_order` headlessly). | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). Retained text/image-bearing or unreadable tiling-pattern cells now make Apply refuse atomically after a bounded check of rewritten resources; fully removed cells still apply, and cross-page retention is checked. The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178); black-and-white images stored losslessly as CCITT Group 4 when smaller, averaged down to 300 ppi above 450 (halftones aside), Group 3 stored again as Group 4, and black-and-white RGB and indexed images that use only palette entries 0 and 1 packed to one bit. Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | From af76b5ca89ed0902c30f4da97677cea38b58709d Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:10:16 +0200 Subject: [PATCH 09/14] Delete key removes selected content object in edit mode (#6384) * Delete key removes selected content object in edit mode The reporter says Delete does nothing for a selected textbox or image in edit mode; the only Delete/Backspace handlers in the UI are for organize pages and comments, so a selected content object has no delete path. Add Delete/Backspace handling to the object selection input in object_ui.rs that deletes the currently selected object(s), and cover it with a unit test that selects an object and presses Delete. Test: delete_key_removes_selected_object failed before the change and passes after. * Tests: the Delete key test takes PageToolContext and imports PagePointer Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/ui-egui/src/tools/object_ui.rs | 72 +++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/crates/ui-egui/src/tools/object_ui.rs b/crates/ui-egui/src/tools/object_ui.rs index 67a077bc7b..a35bf795e1 100644 --- a/crates/ui-egui/src/tools/object_ui.rs +++ b/crates/ui-egui/src/tools/object_ui.rs @@ -218,6 +218,22 @@ pub(crate) fn page_input(cx: PageToolContext<'_>, view: &mut DocView) -> bool { } return false; } + // Delete or Backspace removes the selected objects on this page as one edit, + // like Edit::DeletePages removes pages. The key is consumed even when the pointer + // is off the page, because canvas keys belong to the selection while it exists. + if view.objects.page == Some(page) + && !view.objects.selected.is_empty() + && view.objects.gesture.is_none() + && view.pending_edit.is_none() + && !ui.ctx().egui_wants_keyboard_input() + { + let deleted = + ui.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Delete) || i.consume_key(egui::Modifiers::NONE, egui::Key::Backspace)); + if deleted { + view.pending_edit = delete_edit(page, view.objects.selected.iter().map(|o| o.target).collect()); + return true; + } + } let mut selection = std::mem::take(&mut view.objects); let result = input(&mut selection, cx, view); view.objects = selection; @@ -527,4 +543,60 @@ mod tests { assert!(matches!(delete_edit(2, vec![t(ObjectKind::Image, 4)]), Some(Edit::EditPageImage { page: 2, index: 4, .. }))); assert!(delete_edit(0, Vec::new()).is_none()); } + + /// Selecting content objects on a page and pressing Delete must queue one edit that removes + /// them: `page_input` selects and moves content objects but never consumes Delete or + /// Backspace, so pressing Delete currently does nothing (issue #3780). + #[test] + fn delete_key_removes_selected_object() { + use crate::canvas::page_pointer::PagePointer; + use crate::{PdfCraftApp, canvas::ViewDefaults}; + use pdfcraft_engine::{DocId, ImageEdit}; + use pdfcraft_render::PageInfo; + + let mut app = PdfCraftApp::new(); + let bytes = app.session.create_blank(300.0, 400.0, 1).unwrap(); + app.open_bytes("blank.pdf", None, bytes.as_ref().clone()).unwrap(); + let doc = app.session.get(app.views[0].id).unwrap(); + + let info = pdfcraft_render::DocInfo { + pages: vec![PageInfo { width: 300.0, height: 400.0, label: String::new(), crop: [0.0, 0.0, 300.0, 400.0], rotation: 0 }], + ..Default::default() + }; + let mut view = DocView::new(DocId(1), &info, ViewDefaults::default()); + view.objects.page = Some(0); + view.objects.selected.push(EditableObject { + target: ObjectTarget { kind: ObjectKind::Image, index: 0 }, + rect: [50.0, 60.0, 100.0, 110.0], + content_type: "image", + text: None, + }); + + let ctx = egui::Context::default(); + let xf = PageXform { rect: Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0)), rot: 0, pw: 800.0, ph: 600.0 }; + let ptr = PagePointer { pos: None, ..Default::default() }; + let mut output = ctx.run_ui( + egui::RawInput { + screen_rect: Some(Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0))), + events: vec![egui::Event::Key { + key: egui::Key::Delete, + physical_key: None, + pressed: true, + repeat: false, + modifiers: egui::Modifiers::NONE, + }], + ..Default::default() + }, + |ui| { + let resp = ui.interact(ui.available_rect_before_wrap(), egui::Id::unique("page"), egui::Sense::click()); + page_input(PageToolContext { ui, resp: &resp, xf: &xf, ptr: &ptr, page: 0, doc, info: &info }, &mut view); + }, + ); + output.textures_delta.clear(); + + assert!( + matches!(&view.pending_edit, Some(Edit::EditPageImage { page: 0, index: 0, change: ImageEdit::Delete })), + "pressing Delete with one object selected must queue that object's deletion as one edit" + ); + } } From a0811650dec177caf06a1b143a6cc7590c85dff6 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:29:00 +0200 Subject: [PATCH 10/14] Add Pages range field to Compare Files dialog (#6336) * Add Pages range field to Compare Files dialog Users cannot limit a comparison to selected pages, so the dialog needs a validated Pages field that blocks Compare on invalid ranges. Add a Pages text field under New file, keep its text in pages_text, parse comma-separated numbers and ranges to 0-based pages, show an error and disable Compare for zero, reversed, or out-of-range pages, and leave the comparison itself unchanged. Test: compare_dialog_pages_field_rejects_out_of_range failed before the change and passes after. * Compare Files: the Pages field limits the comparison, and says what is wrong with a bad range Co-Authored-By: Claude Opus 5.5 * Compare: a change across a page break counts on both pages; a bad range is refused, not read as all pages Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/compare/src/lib.rs | 14 ++++++++ crates/engine/src/compare.rs | 23 ++++++++++++-- crates/engine/src/tests.rs | 4 +++ crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/lib.rs | 2 ++ crates/ui-egui/src/workflows/compare_ui.rs | 35 ++++++++++++++++---- crates/ui-egui/tests/it/compare.rs | 37 ++++++++++++++++++++++ 7 files changed, 107 insertions(+), 9 deletions(-) diff --git a/crates/compare/src/lib.rs b/crates/compare/src/lib.rs index 91245cbc21..26a50eaf71 100644 --- a/crates/compare/src/lib.rs +++ b/crates/compare/src/lib.rs @@ -41,6 +41,8 @@ pub struct Side { /// The page the change is on (for a pure insertion or deletion, where the other side's /// words would have been). pub page: usize, + /// The page its last word is on: words changed across a page break are one change. + pub last_page: usize, pub rects: Vec<[f64; 4]>, /// For an empty side (the other side's words were inserted or deleted): the box of the /// word next to where they would be. @@ -245,6 +247,7 @@ fn side(words: &[&Word], neighbour: Option<&Word>) -> Side { Side { text: words.iter().map(|w| w.text.as_str()).collect::>().join(" "), page: words.first().or(neighbour.as_ref()).map_or(0, |w| w.page), + last_page: words.last().or(neighbour.as_ref()).map_or(0, |w| w.page), rects: rects(words), near: if words.is_empty() { neighbour.map(|w| w.rect) } else { None }, } @@ -349,6 +352,17 @@ mod tests { assert_eq!(c.count(Kind::Inserted), 2); } + #[test] + fn a_change_across_a_page_break_knows_both_pages() { + let mut old = words("alpha beta", 0); + old.extend(words("gamma delta", 1)); + let mut new = words("alpha one", 0); + new.extend(words("two delta", 1)); + let c = compare(&old, &new); + assert_eq!(c.changes.len(), 1); + assert_eq!((c.changes[0].new.page, c.changes[0].new.last_page), (0, 1)); + } + #[test] fn deletions_across_pages_keep_their_pages() { let mut old = words("alpha beta gamma", 0); diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..ec5cf3faf3 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -63,6 +63,15 @@ impl Session { Ok(pdfcraft_compare::compare(&a.words(), &b.words())) } + /// [`Session::compare`], keeping the changes on `pages` of `new` (0-based; empty: every page). + pub fn compare_pages(&self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let mut c = self.compare(old, new)?; + if !pages.is_empty() { + c.changes.retain(|ch| pages.iter().any(|page| (ch.new.page..=ch.new.last_page).contains(page))); + } + Ok(c) + } + /// Visual compare: regions where page n of `new` looks different from page n of `old` /// (rendered at `dpi`), as (page, user-space box in `new`). pub fn compare_visual(&self, old: DocId, new: DocId, dpi: f32) -> Result, EditError> { @@ -95,7 +104,12 @@ impl Session { /// The compare report as a new PDF (not opened). pub fn compare_report(&self, old: DocId, new: DocId) -> Result>, EditError> { - let c = self.compare(old, new)?; + self.compare_report_of(old, new, &[]) + } + + /// [`Session::compare_report`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn compare_report_of(&self, old: DocId, new: DocId, pages: &[usize]) -> Result>, EditError> { + let c = self.compare_pages(old, new, pages)?; let name = |id| self.get(id).map(|d| d.name.clone()).unwrap_or_default(); self.create_report("Compare Report", &pdfcraft_compare::report(&c, &name(old), &name(new))) } @@ -104,7 +118,12 @@ impl Session { /// (blue, green) and a note where text was deleted (red), authored "Compare". One undoable /// step; returns how many comments were added. pub fn mark_differences(&mut self, old: DocId, new: DocId) -> Result { - let c = self.compare(old, new)?; + self.mark_differences_of(old, new, &[]) + } + + /// [`Session::mark_differences`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn mark_differences_of(&mut self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let c = self.compare_pages(old, new, pages)?; let mut edits = Vec::new(); for ch in &c.changes { let color = colour(ch.kind); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 688dcec049..5b6fb01b9e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5841,6 +5841,10 @@ fn comparing_two_versions_of_a_document() { let got: Vec<(compare::Kind, &str, &str)> = c.changes.iter().map(|x| (x.kind, x.old.text.as_str(), x.new.text.as_str())).collect(); assert_eq!(got, [(compare::Kind::Replaced, "Monday.", "Tuesday."), (compare::Kind::Replaced, "days.", "days of invoice.")], "{got:?}"); assert!(s.compare(old, old).unwrap().identical()); + // limited to pages of the new file: both changes are on the first + assert_eq!(s.compare_pages(old, new, &[]).unwrap().changes.len(), 2); + assert_eq!(s.compare_pages(old, new, &[0]).unwrap().changes.len(), 2); + assert!(s.compare_pages(old, new, &[1]).unwrap().identical()); let report = s.compare_report(old, new).unwrap(); let rid = s.open("report.pdf", None, report, None).unwrap(); diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 3b3d5dd036..04dbd19447 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -210,6 +210,7 @@ impl PdfCraftApp { action_files_override: None, pdfa: Default::default(), compare_old: None, + compare_pages: String::new(), compare: None, js_console: Default::default(), doc_js: Default::default(), diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2f393c61d1..380f84a644 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -526,6 +526,8 @@ pub struct PdfCraftApp { pub pdfa: standards_ui::PdfaState, /// Compare files: the chosen older document and the last result. pub compare_old: Option, + /// The Compare Files dialog's Pages field: the new file's pages to compare, empty for all. + pub compare_pages: String, pub compare: Option, /// The JavaScript console and the Document JavaScripts draft. pub js_console: js_ui::JsConsole, diff --git a/crates/ui-egui/src/workflows/compare_ui.rs b/crates/ui-egui/src/workflows/compare_ui.rs index 939d5fed3e..7bf4c9f24b 100644 --- a/crates/ui-egui/src/workflows/compare_ui.rs +++ b/crates/ui-egui/src/workflows/compare_ui.rs @@ -14,6 +14,8 @@ pub struct CompareState { pub old: DocId, pub new: DocId, pub result: Comparison, + /// The pages of the new file this comparison covers (0-based; empty: every page). + pub pages: Vec, /// Regions that look different (page, user-space box), page n against page n. pub visual: Vec<(usize, [f64; 4])>, pub selected: Option, @@ -52,13 +54,14 @@ pub fn colour(kind: Kind) -> Color32 { pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { ui.label(egui::RichText::new(tl!("Compare Files")).font(theme::semibold(18.0))); ui.add_space(8.0); - let Some((_, new)) = app.active_ids() else { return (false, true) }; + let Some((active_i, new)) = app.active_ids() else { return (false, true) }; let others: Vec<(DocId, String)> = app.views.iter().filter(|v| v.id != new).filter_map(|v| app.session.get(v.id).map(|d| (d.id, d.name.clone()))).collect(); if app.compare_old.is_none_or(|o| !others.iter().any(|(id, _)| *id == o)) { app.compare_old = others.first().map(|(id, _)| *id); } let new_name = app.session.get(new).map(|d| d.name.clone()).unwrap_or_default(); + let page_count = app.views.get(active_i).map_or(0, |v| v.page_count()); egui::Grid::new("compare-files").num_columns(2).spacing([12.0, 10.0]).show(ui, |ui| { ui.label(tl!("Old file:")); let shown = @@ -72,14 +75,23 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.label(tl!("New file:")); ui.label(egui::RichText::new(new_name).strong()); ui.end_row(); + ui.label(tl!("Pages:")); + let field = ui.add(egui::TextEdit::singleline(&mut app.compare_pages).desired_width(320.0).hint_text(format!("1-{page_count}"))); + field.widget_info(|| egui::WidgetInfo::labeled(egui::Role::TextInput, ui.is_enabled(), tl!("Pages"))); + ui.end_row(); }); + let pages_error = crate::page_selection_ui::parse(&app.compare_pages, page_count).err(); + if let Some(e) = &pages_error { + ui.label(e.said(&|s| crate::i18n::t(s).to_string())); + } ui.add_space(4.0); ui.label(egui::RichText::new(tl!("Text is compared word by word; the differences are shown on the new file.")).small().color(t.text_muted)); ui.add_space(10.0); let (mut go, mut cancel) = (false, false); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(app.compare_old.is_some(), |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { + let ready = app.compare_old.is_some() && pages_error.is_none(); + if ui.add_enabled_ui(ready, |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -191,9 +203,17 @@ impl PdfCraftApp { pub fn run_compare(&mut self) { let Some((i, new)) = self.active_ids() else { return }; let Some(old) = self.compare_old else { return }; - match self.session.compare(old, new) { + let count = self.views.get(i).map_or(0, |v| v.page_count()); + let pages = match crate::page_selection_ui::parse(&self.compare_pages, count) { + Ok(pages) => pages, + Err(e) => return self.notify_error(e.said(&|s| crate::i18n::t(s).to_string())), + }; + match self.session.compare_pages(old, new, &pages) { Ok(result) => { - let visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + let mut visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + if !pages.is_empty() { + visual.retain(|(page, _)| pages.contains(page)); + } let n = result.changes.len(); // Side by side when the old version is open in a tab and the window has the room. let side_by_side = self.views.iter().any(|v| v.id == old) && self.ctx.as_ref().is_none_or(|c| c.content_rect().width() >= 1000.0); @@ -201,6 +221,7 @@ impl PdfCraftApp { old, new, result, + pages, visual, selected: None, side_by_side, @@ -225,7 +246,7 @@ impl PdfCraftApp { pub(crate) fn compare_action(&mut self, index: usize, a: PanelAction) { let Some(s) = self.compare.as_mut() else { return }; - let (old, new) = (s.old, s.new); + let (old, new, pages) = (s.old, s.new, s.pages.clone()); match a { PanelAction::Select(i) => { s.selected = Some(i); @@ -252,7 +273,7 @@ impl PdfCraftApp { s.show_visual = visual; self.shade_compare(index); } - PanelAction::Mark => match self.session.mark_differences(old, new) { + PanelAction::Mark => match self.session.mark_differences_of(old, new, &pages) { Ok(n) => { if let Some(info) = self.session.get(new).map(|d| d.info.clone()) { self.views[index].document_changed(&info); @@ -265,7 +286,7 @@ impl PdfCraftApp { } Err(e) => self.notify_error(e), }, - PanelAction::Report => match self.session.compare_report(old, new) { + PanelAction::Report => match self.session.compare_report_of(old, new, &pages) { Ok(bytes) => { if let Err(e) = self.open_bytes("Compare Report.pdf", None, bytes.to_vec()) { self.notify_error(e); diff --git a/crates/ui-egui/tests/it/compare.rs b/crates/ui-egui/tests/it/compare.rs index 52acae545f..25a4616727 100644 --- a/crates/ui-egui/tests/it/compare.rs +++ b/crates/ui-egui/tests/it/compare.rs @@ -160,6 +160,43 @@ fn compare_filters_hide_text_or_visual_changes() { assert_eq!(kinds(&h, old_i), (text_old, 0)); } +/// The Compare Files dialog's Pages field rejects an out-of-range page: the +/// comparison does not run and the dialog stays open with the typed range kept. +#[test] +fn compare_dialog_pages_field_rejects_out_of_range() { + let s = Session::new(); + let v1 = s.create_from_text("t", "Delivery within five days. Returns accepted.").unwrap().to_vec(); + let v2 = s.create_from_text("t", "Delivery within three days. Returns accepted for a week.").unwrap().to_vec(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(move |_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("v1.pdf", None, v1.clone()).unwrap(); + app.open_bytes("v2.pdf", None, v2.clone()).unwrap(); + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("doc.compare")); + h.run_steps(2); + h.get_by_label("Compare Files"); + let pages = h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages"); + assert!(pages.is_some(), "the dialog has a Pages text input"); + assert!(h.query_by_label("Pages:").is_some(), "the dialog has a Pages field"); + let Some(pages) = pages else { return }; + pages.focus(); + pages.type_text("99"); + h.run_steps(2); + assert_eq!( + h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages").and_then(|n| n.value()), + Some("99".to_string()), + "the typed range is kept" + ); + h.get_by_label("Compare").click(); + h.run_steps(3); + assert!(h.query_by_label("Compare Files").is_some(), "the dialog stays open for an out-of-range page"); + assert!(h.state().compare.is_none(), "the comparison did not run"); + assert_ne!(h.state().right, Some(RightPanel::Compare)); +} + /// Writes a screenshot of the Compare panel when PDFCRAFT_SHOT is set (for review). #[test] fn compare_panel_screenshot() { From 81b0833b245fee444e4703b4298aea949995a9aa Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:50:02 +0100 Subject: [PATCH 11/14] Copy the shared house rules into AGENTS.md. The blocks come from craft-contrib at dde09ea31e2e457f5b4653a6e76f7cdf55944bdf. CLAUDE.md imports AGENTS.md. The house-rules workflow checks the copy. --- .github/workflows/house-rules.yml | 28 ++++++++++++ AGENTS.md | 71 ++++++++++++++++++++++++++++--- CLAUDE.md | 66 +--------------------------- 3 files changed, 95 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/house-rules.yml diff --git a/.github/workflows/house-rules.yml b/.github/workflows/house-rules.yml new file mode 100644 index 0000000000..de64ad3ee4 --- /dev/null +++ b/.github/workflows/house-rules.yml @@ -0,0 +1,28 @@ +name: House rules + +on: + push: + branches: [main] + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + pull_request: + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: house-rules-${{ github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + uses: storytold/craft-common-actions/.github/workflows/house-rules.yml@main + with: + rust: true diff --git a/AGENTS.md b/AGENTS.md index 3ab900aa95..f55ab534a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,56 @@ + +## House rules + +These apply in every Craft repository. A later rule may be stricter, not weaker. The source is `standards/house-rules-core.md` in `storytold/craft-contrib`. Propose changes there, not in a copy. + +1. **A person is accountable.** Whoever runs an agent answers for what it opens and for its review comments. +2. **Check, then claim.** Before writing code, read the issue comments, search pull requests for `#N` and the keywords, and check recent commits on `main`. Comment on the issue first. Release a claim you will not finish. +3. **Verify, don't guess.** Open a definition before using it. Confirm the cause in the code, even when someone names a location. +4. **Keep fixes narrow and tested.** Fix what the issue describes, where the bug is. Add a test that fails without the fix. +5. **Review the diff before opening a pull request.** Identifiers exist as used, the test fails without the fix, nothing unrelated changed, and the description is true. +6. **Look for a better pull request.** If another fixes the same thing, close yours when it is better or equal, and say why. +7. **Report only what ran.** Say what you tested and what you could not run. Without a local build or test, say so and rely on CI before marking ready. +8. **`main` changes through a pull request.** Do not push to it. Branch protection enforces this. There is no exception. +9. **Stop and ask only about** licensing outside the allowlist, publishing, releases, a new remote, secrets, spending money, destructive git, and anything else hard to undo. Otherwise keep working. `plan/` is a gitignored notebook, not rules. Without it, do not ask for it and do not stop. +10. **Keep secrets and personal details out** of the repository, logs and requests. +11. **CI stays shared.** Workflows call `storytold/craft-common-actions`. Do not change CI in a feature or fix pull request. If CI cannot cover a change, open an issue there. +12. **Guards say why.** A check that looks removable says what it prevents. Read that before changing it. +13. **Instructions must be readable by everyone.** No notes about one machine, and no links to files a reader cannot open. `craftrules` is retired. Do not link to it. +14. **Keep pull requests moving.** Concurrent pull requests are fine; unanswered actionable feedback is not. Respond in 7 days, a reminder at 14, closure eligible at 30, from when an actionable request waits on the author. Suggestions and unconfirmed bot findings do not start the clock. A fix, explanation or agreed plan resets it; unrelated activity does not. Waiting on a reviewer or maintainer pauses it and does not count against the author. A maintainer decides closure; it is never automatic. Reuse search results, do not poll, and stop at a rate limit. +15. **No agent attribution.** A pull request does not say an agent wrote it. A repository rule may not require that. + + +## Rust Craft repositories + +These apply to native Craft apps and to shared Rust libraries. Select them by repository type. A repository that merely contains some Rust does not take this block. Rules later in a repository's file may be stricter, but not weaker. The source is `standards/house-rules-rust.md` in [storytold/craft-contrib](https://github.com/storytold/craft-contrib). + +1. **Never crash.** Non-test code does not panic and has no `unsafe`, except where the repository file names an exception. Return errors, treat input as hostile, and bound recursion and allocations. New production crate roots copy the `deny` attribute from an existing crate: `clippy::unwrap_used`, `clippy::expect_used`, `clippy::panic`, `clippy::unimplemented`, `clippy::todo`, `clippy::unreachable`. Every crash fix comes with a test that panicked before the fix. +2. **Clean room.** Learn from observed behaviour and public specifications. Do not copy another product's code, assets, strings or artwork. Check a dependency's licence before adding it. +3. **Assets and fonts.** Every asset is original or suitably licensed, and is recorded in `ATTRIBUTION.md` in the same change. Font files live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), not in the app repository. +4. **Rust only for native Craft apps.** No handwritten JavaScript or TypeScript, and no Tauri, Electron or webview shell, in those apps. Shared tooling follows its documented technical scope and maintainer-approved exceptions. Craft PR Review's existing Tauri and TypeScript UI is one such exception. It does not relax this rule for native apps. Preserve any existing web build. +5. **Time goes through `chrono`.** Do not hand-roll calendar conversion. For dates beyond what `chrono` can represent, follow [standards/time.md](https://github.com/storytold/craft-contrib/blob/main/standards/time.md). +6. **Contributor credits.** Add or edit only your own person's entry in `contributors/people.toml`, and only after asking them. Never guess a name or hand-edit `contributors.json`. + +## Licence allowlist + +Dependencies and assets use SPDX identifiers. An expression is allowed when every identifier is in this list. A repository may name a shorter list. It may not add an identifier here by editing its own copy of these rules. + +- `ISC` +- `MIT` +- `BSD-2-Clause` +- `BSD-3-Clause` +- `Apache-2.0` +- `OFL-1.1` +- `Ubuntu-font-1.0` +- `Bitstream-Vera` +- `CC0-1.0` +- `CC-BY-4.0` +- `CC-BY-SA-4.0` +- `LicenseRef-Public-Domain` + +`LicenseRef-Storyteller-Trademark` is not in this list. It covers Storyteller brand marks only, where a repository already says so. + + # AGENTS.md — rules for every agent and contributor These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** @@ -85,7 +138,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi --- -## 2. Clean-room rules (summary; details in `CLAUDE.md` and `plan/adr/0001`) +## 2. Clean-room rules (summary; `plan/adr/0001` is an optional local note) - Never read, disassemble or copy anything inside the Acrobat bundle. Only names and listings are allowed, and **never** open `Contents/Resources/JavaScripts/`. - Behaviour comes only from public specifications (ISO 32000-2, the Arlington model), public documentation and black-box observation. @@ -106,7 +159,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi PdfCraft opens files from strangers, and people trust it with their work. A malformed PDF, a bad tool argument, a corrupt settings file or a full disk must produce an error the user (or agent) can act on, never a crash and never lost work. **This outranks feature work:** don't ship a feature by adding a panic path, and fix a crash before building on top of it. The rules below are the ones that bind this repository. -- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity in `CLAUDE.md`). +- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity below). - **No panicking shortcuts.** No `unwrap()`, `expect()`, `panic!`, `unreachable!`, `todo!` or `unimplemented!` outside tests, unless the call is provably infallible (e.g. writing to a `Vec`), with a comment saying why. `assert!` only for internal invariants that input cannot reach. - **No `unsafe`.** The workspace sets `unsafe_code = "forbid"`. If a platform API needs it, use a maintained safe wrapper crate; a real exception needs the owner's sign-off. - **Treat every input-derived number as hostile.** Use `get()` instead of `[i]` or `[a..b]`. Slice strings only at char boundaries. Use checked/saturating arithmetic for lengths, offsets and counts. Guard against division by zero and NaN/inf casts, and never cast a negative value to `usize`. Cap allocations sized by the file. @@ -115,10 +168,18 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Prove it.** Every crash fix comes with a small synthetic regression test that panicked before the fix. Turn fuzzer findings (`cargo xtask fuzz`) into such tests before fixing them. - **Enforced by clippy.** Root `clippy.toml` lets tests use `unwrap`/`expect`/`panic!`. A crate that is clean carries `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`; new crates start with it. -## 5. Everything else +## 5. Page interaction, fidelity and gates - -Follow `CLAUDE.md`. It covers quality gates, layering, commit rules and how to run and look at the app. +- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd. `cos`, `filters`, `crypt` and `arlington` stay standalone. +- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state. `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. +- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim`, `crates/ui-egui/src/canvas/page_cursor.rs`) and never set it themselves. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. +- **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. +- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. +- **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. +- `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. +- `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. ## Contributor credits (About window) diff --git a/CLAUDE.md b/CLAUDE.md index 5c7034b7a4..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,65 +1 @@ -# PdfCraft — instructions for agents - -PdfCraft is a clean-room, open-source, Rust-native PDF application targeting Adobe Acrobat Pro parity. It runs natively on macOS, Windows, Linux and FreeBSD, and on the web via WASM. It is the sibling of `../photocraft` (a Photoshop-class editor) and follows the same conventions. - -## Start every session here -1. Read `plan/STATUS.md`: the current milestone, the next unchecked task and any blockers. `ROADMAP.md` is the one-page summary (stage, numbers, progress log) and `docs/roadmap.md` the milestone table; at the end of every session update the milestone row and add a line to the ROADMAP.md progress log. - - Read `docs/gaps.md` (the ranked gaps: Acrobat interop, the open-issue backlog, the renderer and a fidelity harness, editing existing content, Pro workflows, 1.0 polish) and `docs/target-app-parity.md` (the numbers and how they are measured) before choosing work. Prefer the gaps over new P2/P3 features, and keep both documents true when things change (`docs/` progress docs carry a status line and a revision history). - - "Shipped" in `parity/` means "exists and tested", not "as good as Acrobat". Don't mark a feature shipped on a generic test, and say in its notes what is still missing. -2. Read that task in `plan/execution-plan.md` §3, the relevant section of `plan/architecture.md`, and the README of the crate you're touching. -3. Follow the **autonomous operation protocol** in `plan/execution-plan.md` §7 (orient → plan → implement + test → verify → record → commit). Don't stop to ask unless §7 lists the decision as the user's. - -`plan/` is gitignored (local-only, like PhotoCraft). The machine-readable parity checklist lives in `parity/` (committed). - -## Non-negotiables -- **Assets: read `AGENTS.md` §1 before adding or showing any icon, image, font or document.** No assets from Adobe products, ever. Only openly licensed or contributor-original assets are allowed, each with an entry in `ATTRIBUTION.toml`. `cargo xtask assets` enforces this. `AGENTS.md` overrides this file. -- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts).** Never commit font files here (`AGENTS.md` §1.4). It is the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts && CRAFT_FONTS_DIR=../craft-fonts cargo test --workspace` embeds the Japanese fonts (UI fallback, Japanese text in edited PDFs) and runs their tests, which otherwise skip. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Clean-room.** - - Never read, disassemble or copy anything inside the Acrobat bundle (names and listings only). **Never open `Contents/Resources/JavaScripts/`.** - - Behaviour comes from public docs, specs (ISO 32000-2, the Arlington model) and black-box observation (`plan/acrobat/`). - - Never copy GPL/AGPL code. MuPDF, Ghostscript, Poppler, veraPDF and DSS run only as external oracle processes. - - See `plan/README.md` §Clean-room and `plan/adr/0001`. -- **Privacy.** When observing Acrobat, use synthetic fixtures only. Never capture the Home view, recent files or account info. Capture by window id (`plan/acrobat/tools/`). Never commit Acrobat outputs, corpus files or personal data. -- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd (`plan/architecture.md` §3). `cos`/`filters`/`crypt`/`arlington` stay standalone. -- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state (`hover_pos`, `interact_pos`, `latest_pos`, `press_origin`, button reads); `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. -- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim(Layer::…, icon)`, `crates/ui-egui/src/canvas/page_cursor.rs`), never set it (`set_cursor_icon`, `cursor_icon =`, `on_hover_cursor`); `document_area` sets it once, from the claim on the highest layer the active tool reaches. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. -- **Page presses.** A page module decides what of its own is under a point in one hit test, which returns a `Hit` (owner, layer, cursor, tip; `crates/ui-egui/src/canvas/page_router.rs`) and gives its cursor, its tip and its press. It acts on a press only when the press is its own or, for the active tool, nobody's: the page view tells it which (`PagePress`: its own, nobody's, or another page's), and it never tests the press point again or asks another module whether it wants it. Escape cancels any press the pages hold (`cancel_on_escape`), and that press is ignored until the button is up. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. -- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. -- **Never panic.** Every PDF, script, MCP call, settings file and keystroke is untrusted input, and none of it may crash the app or lose the user's work. This outranks feature work. Return `Result` (the crate's error enum) and propagate with `?`, or fall back leniently and record it. In non-test code: - - No `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` unless it is provably infallible, with a comment saying why. - - No indexing or slicing with input-derived positions (use `get`, and slice strings only at char boundaries). - - Use checked or saturating arithmetic on input-derived numbers, and guard against division by zero and NaN/inf casts. - - Cap allocations sized by input, and bound recursion with depth limits or seen-sets. - - Handle lock poisoning. - - No `unsafe` (`unsafe_code = "forbid"`). - - Every crash fix gets a synthetic regression test. See `AGENTS.md` §4. -- **Rust only** in the product and build (`xtask`). No handwritten JS/TS. -- **Time goes through `chrono`** (workspace dependency). Never hand-roll UTC/calendar conversion (days-to-date algorithms, leap-year math); a crate owns only its output format string. The one exception is range extension: past the years chrono can represent, shift the date by whole 400-year Gregorian cycles (146,097 days, after which the calendar repeats), let chrono compute the month and day, and add the cycles back to the year (splitting seconds into hours, minutes and seconds is plain arithmetic). -- **Review every update as you make it, at four zooms.** Not an end-of-task pass — a per-update habit, so a defect never gets built on. **Clause:** checked arithmetic, `get()` not `[i]`, char-boundary slicing, the right error through `?`, `<` against `<=`, the 1-based/0-based seam, capped allocations. **Function:** is it total — name an input that panics; walk empty, zero, max, malformed, cyclic; is the return type honest. **File/module:** two sources of truth, dead or duplicated code, is the public surface still smallest and the doc comment still true. **Project:** layering, wasm, undo/redo, save fidelity, encryption and permissions, a matching `automation` tool or registry entry, and whether a `parity/` status or note just became untrue. Scale the zoom to the change, and fix what a level turns up before widening. - - Tests mirror the zooms: property → unit → crate integration → end-to-end. - - **Prove a test fails before the fix by actually reverting the fix.** A test nobody has seen fail is not evidence. This has caught two worthless tests in this repo: one confinement test passed with path resolution switched off entirely, because the tools failed on a nonexistent document id before reaching the path. - - **Rebasing is a review step, not a mechanical one.** Read every hunk you resolve, and the auto-merged hunks in the files you touched too — `main` moves fast enough that a file can change by hundreds of lines under a branch. -- **External PRs are held to safety, security and completeness before style.** Provenance first (no Adobe-derived asset, no committed font, no copied GPL/AGPL code, behaviour from the spec and not from reading another implementation), then no new panic path or `unsafe`, then `--root` confinement on every path including ones reached by nested dispatch, opt-in-only MCP and control channel, no new outbound network, then a headless `automation` tool, tests at the right zoom and an honest `parity/` entry. Provenance is the part a contributor cannot be expected to know, and the only part that cannot be undone after merge. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check once `xtask ci` exists (M0). - - New workspace crates must include `[lints] workspace = true` in `Cargo.toml` to inherit `[workspace.lints]`. Production crate roots must declare the required crate-level lint attributes before modules or other crate items; crate-level `//!` documentation may precede them. - - **Scope builds and tests to the crates you touched** — `cargo clippy -p -j 2 --all-targets` and `cargo test -p -j 2`, under a `timeout`. A full `--workspace` build has locked up a development machine's CPU; it is CI's job and CI runs it on every PR. Run the workspace sweep locally only when you have a reason to and the machine is free. -- **Check for existing work before starting a fix or feature.** Search the issues and PRs, open and closed, with `gh issue list -R storytold/pdfcraft --state all --search ""` and the same with `gh pr list` (try a few phrasings), and check whether a merged fix is already on `main` (`git merge-base --is-ancestor origin/main`). If one is, the report is likely from an older build: say so rather than fixing it again. If an open PR covers it, review or build on that PR instead of opening a competing one. If an issue exists, reference it in the PR (`Fixes #123`). -- **Commits and PRs:** `main` is the only integration branch and is **never pushed to directly**. Branch from an up-to-date `main` as `/` (`feat/`, `fix/`, `docs/`, `chore/`, `refactor/`), one task id per commit (e.g. `M1.4: xref stream reader`), commit only green states, and open a PR into `main` using `.github/pull_request_template.md`. Merge only when CI is green. - - **`main` moves fast enough that PRs rot.** It took 396 commits in the three days to 2026-10-12, and most open PRs are conflicting at any given time. Rebase your own branch the same day `main` moves under it, and ask the author to rebase theirs — never rewrite a contributor's branch. Conflict risk tracks how *contested* the files are, not how large the change is: tooling, tests and docs are nearly free, while `crates/fonts`, `ui-egui` and the i18n catalogues are where several people collide at once. - -## Running and looking at the app -- `cargo run -p pdfcraft -- ` opens the desktop app. -- `cargo run -p pdfcraft-cli -- run --script steps.json --root DIR` drives the engine headlessly through the automation tools (`pdfcraft-cli tools` lists them). Use it, together with `page_render`, to check engine changes. `pdfcraft-cli mcp` is the opt-in MCP server (AGENTS.md §3). -- `cargo xtask fuzz --time 300` mutation-fuzzes open/render/edit/save in child processes. Findings land in `fuzz-out/findings/` (git-ignored; never commit corpus-derived files). Turn every real finding into a small synthetic regression test before fixing it. -- `cargo xtask parity [--partial]` reports Acrobat-parity progress from `parity/acrobat-features.toml`. Update the entry when a feature ships. -- `cargo xtask demo-pdf` builds `dist/demo/pdfcraft-showcase.pdf` (needs Chrome) for visual checks. -- For UI work, **look at the result**. Either launch `pdfcraft --control FILE doc.pdf` and use `pdfcraft-cli ui --control FILE screenshot --out x.png` (plus `inspect`, `click`, `key`, `type`, `command`, `set`), or take a headless shot with `cargo run -p pdfcraft-ui-egui --example shot`. Compare against `plan/acrobat/02-ui-ux.md`. Control-channel tests use kittest (`crates/ui-egui/tests/control.rs`). -- Parallel agents: use a separate `CARGO_TARGET_DIR` per agent and separate git worktrees. - -## Current bootstrap debt (tracked in STATUS.md) -- `pdfcraft-render` renders through the `hayro` crate directly and inspects documents through `lopdf`. Both get replaced by `cos` / `model` / the DisplayList device (M1–M2, ADR-0004). - -## CI workflows -- Workflows in `.github/workflows/` call the shared workflows in [`storytold/craft-common-actions`](https://github.com/storytold/craft-common-actions) at `@main`, with their inputs and secrets. No other jobs: if no shared workflow does what's needed, it doesn't go here. -- Don't add or change CI in a feature or fix PR. If the existing workflows can't build or test a change, link an existing craft-common-actions issue or PR, or open a new issue, or a PR only for a small fix that is safe for every app and does not increase CI time. -- Some workflow files here still run their own jobs. Don't add to them. Moving a job into craft-common-actions is its own task, never part of another PR, and the job stays here until a shared workflow runs it. +@AGENTS.md From 465783fc47a3cb98ceaabaa1cc851c0c0a1b4ea3 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:37:25 +0100 Subject: [PATCH 12/14] docs: preserve local guidance in house-rules rollout --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f55ab534a8..c228c4185b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Dependencies and assets use SPDX identifiers. An expression is allowed when ever # AGENTS.md — rules for every agent and contributor -These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** +These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. Day-to-day working instructions are in this file; `CLAUDE.md` imports it. --- @@ -176,7 +176,7 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. - **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. - **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `workspace = true` under `[lints]`. Scope `cargo clippy -p --all-targets -- -D warnings` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. - **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. - `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. - `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. From 1d71d6039f43a645ec9a1b711d6f080f3e5b6cca Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:16:36 +0200 Subject: [PATCH 13/14] Add PCX image support to Create PDF conversion (#6396) * Add PCX image support to Create PDF conversion PCX is listed as an accepted Create source elsewhere but PdfCraft rejects it, so users cannot convert PCX images to PDF. Detect PCX by its header in is_image and decode it in embed, adding pcx to CONVERTIBLE so existing from_images and source_kind paths handle it. Test: pcx_images_become_pages failed before the change and passes after. * Create: PCX scanlines hold every plane's row, 8-bit indexed images, older versions, standard test files Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/create/src/lib.rs | 134 ++++++++++++++++++++++++++++++++++++- crates/create/src/tests.rs | 76 +++++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index 53a587728f..149bd99d97 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -516,6 +516,129 @@ fn decoded(name: &str, bytes: &[u8], format: image::ImageFormat) -> Result Option { + b.get(at..at + 2).map(|x| u16::from_le_bytes([x[0], x[1]])) +} + +/// How a PCX image holds its pixels: bits per plane and planes. +#[derive(Clone, Copy, PartialEq)] +enum PcxLayout { + /// One byte per pixel per colour: each scanline has a red, a green and a blue row. + Rgb, + /// One byte per pixel, an index into the palette after the image (grey without one). + Indexed, + /// One bit per pixel in each of 1 to 4 planes, an index into the header's palette. + Bits(usize), +} + +impl PcxLayout { + fn of(bits: u8, planes: usize) -> Option { + match (bits, planes) { + (8, 3) => Some(PcxLayout::Rgb), + (8, 1) => Some(PcxLayout::Indexed), + (1, 1..=4) => Some(PcxLayout::Bits(planes)), + _ => None, + } + } +} + +/// The palette of a PCX image, and where its run-length coded scanlines end. An 8-bit image ends +/// with a 0x0C byte and 256 colours; a one-plane bit image is black and white whatever its header +/// holds; other bit images use the 16 colours at byte 16. +fn pcx_palette(bytes: &[u8], layout: PcxLayout) -> (Vec, usize) { + match layout { + PcxLayout::Rgb => (Vec::new(), bytes.len()), + PcxLayout::Indexed => match bytes.len().checked_sub(769).filter(|at| *at >= 128 && bytes.get(*at) == Some(&0x0C)) { + Some(at) => (bytes.get(at + 1..).unwrap_or_default().to_vec(), at), + None => ((0..=255u8).flat_map(|v| [v, v, v]).collect(), bytes.len()), + }, + PcxLayout::Bits(1) => (vec![0, 0, 0, 255, 255, 255], bytes.len()), + PcxLayout::Bits(_) => (bytes.get(16..64).unwrap_or_default().to_vec(), bytes.len()), + } +} + +/// The `rows` scanlines of `line` bytes each, decoded. A run may cross from one plane's row into the +/// next, but not out of its scanline. +fn pcx_scanlines(mut body: &[u8], rows: usize, line: usize) -> Option> { + let mut out = Vec::new(); + out.try_reserve_exact(rows.checked_mul(line)?).ok()?; + for row in 1..=rows { + let end = row * line; + while out.len() < end { + let (&c, rest) = body.split_first()?; + body = rest; + if c < 0xC0 { + out.push(c); + continue; + } + let (&value, rest) = body.split_first()?; + body = rest; + let count = usize::from(c & 0x3F); + if count > end - out.len() { + return None; + } + out.resize(out.len() + count, value); + } + } + Some(out) +} + +/// The RGB samples of the `w` pixels of scanline `row` (planes `stride` bytes apart). +fn pcx_row(out: &mut Vec, row: &[u8], w: usize, stride: usize, layout: PcxLayout, palette: &[u8]) { + let at = |i: usize| row.get(i).copied().unwrap_or(0); + for x in 0..w { + let index = match layout { + PcxLayout::Rgb => { + out.extend([at(x), at(stride + x), at(2 * stride + x)]); + continue; + } + PcxLayout::Indexed => usize::from(at(x)), + PcxLayout::Bits(planes) => (0..planes).fold(0, |index, k| index | (usize::from(at(k * stride + x / 8) >> (7 - x % 8) & 1) << k)), + }; + out.extend((0..3).map(|c| palette.get(index * 3 + c).copied().unwrap_or(0))); + } +} + +/// A PCX image: a 128-byte header (size at 4-11, resolution at 12 and 14, plane count at 65, bytes +/// per plane row at 66), then run-length coded scanlines, each holding one row of every plane. +fn pcx(name: &str, bytes: &[u8]) -> Result { + let bad = |m: &str| CreateError::Image(name.into(), m.into()); + let header = bytes.get(..128).ok_or_else(|| bad("a damaged PCX header"))?; + let word = |at: usize| le16(header, at).unwrap_or(0); + let byte = |at: usize| header.get(at).copied().unwrap_or(0); + let (bits, planes, stride) = (byte(3), usize::from(byte(65)), usize::from(word(66))); + let (Some(w), Some(h)) = (word(8).checked_sub(word(4)), word(10).checked_sub(word(6))) else { + return Err(bad("a damaged PCX header")); + }; + let (w, h) = (usize::from(w) + 1, usize::from(h) + 1); + if planes == 0 || stride == 0 { + return Err(bad("a damaged PCX header")); + } + let layout = PcxLayout::of(bits, planes).ok_or_else(|| bad("this PCX plane layout is not supported"))?; + if stride < (w * usize::from(bits)).div_ceil(8) { + return Err(bad("a damaged PCX header")); + } + let line = planes * stride; + if line.saturating_mul(h) > MAX_DECODED_BYTES || w.saturating_mul(h).saturating_mul(3) > MAX_DECODED_BYTES { + return Err(bad("unusual PCX image size")); + } + let (palette, end) = pcx_palette(bytes, layout); + let body = bytes.get(128..end).unwrap_or_default(); + let scanlines = pcx_scanlines(body, h, line).ok_or_else(|| bad("a damaged PCX body"))?; + let mut data = Vec::new(); + data.try_reserve_exact(w * h * 3).map_err(|_| bad("not enough memory for this image"))?; + for row in scanlines.chunks_exact(line) { + pcx_row(&mut data, row, w, stride, layout, &palette); + } + let dpi = (f64::from(word(12)), f64::from(word(14))); + let dpi = if dpi.0 > 0.0 && dpi.1 > 0.0 { dpi } else { (72.0, 72.0) }; + let mut d = Dict::new(); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let px = (u32::try_from(w).unwrap_or(0), u32::try_from(h).unwrap_or(0)); + Ok(Embedded { dict: d, data, filtered: false, smask: None, icc: None, px, dpi }) +} + /// Largest GIF logical screen, in pixels (4096×4096). const MAX_GIF_PIXELS: u64 = 1 << 24; @@ -623,7 +746,13 @@ pub enum SourceKind { } /// File extensions Create converts to PDF (images and plain text). -pub const CONVERTIBLE: [&str; 12] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "txt", "text"]; +pub const CONVERTIBLE: [&str; 13] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx", "txt", "text"]; + +/// A PCX file: its manufacturer byte, a version (1 was never used), run-length coding and 1 or 8 +/// bits per plane. `pcx` says when the rest of the header isn't one it can read. +fn is_pcx(bytes: &[u8]) -> bool { + bytes.first() == Some(&0x0A) && matches!(bytes.get(1), Some(0 | 2..=5)) && bytes.get(2) == Some(&1) && matches!(bytes.get(3), Some(1 | 8)) +} /// A classic TIFF or a BigTIFF (version 43), in either byte order. fn is_tiff(bytes: &[u8]) -> bool { @@ -640,6 +769,7 @@ fn is_image(bytes: &[u8]) -> bool { || bytes.starts_with(b"\x89PNG\r\n\x1a\n") || is_tiff(bytes) || is_gif(bytes) + || is_pcx(bytes) // JPEG 2000: a JP2 file or a raw codestream. || bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) @@ -675,6 +805,8 @@ fn embed(name: &str, bytes: &[u8], wanted: usize) -> Result, Creat Ok(vec![decoded(name, &pdfcraft_xfa::image::bmp_with_pixel_offset(bytes), image::ImageFormat::Bmp)?]) } else if is_gif(bytes) { Ok(vec![decoded(name, bytes, image::ImageFormat::Gif)?]) + } else if is_pcx(bytes) { + Ok(vec![pcx(name, bytes)?]) } else if bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) { Ok(vec![jpx(name, bytes)?]) } else { diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 9e620c19c3..bea96c887d 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -485,6 +485,82 @@ fn bigtiff_images_become_pages() { assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 1.0], "4×2 px at 144 dpi"); } +/// A PCX header for a `w` by `h` image at 72 dpi. +fn pcx_header(version: u8, bits: u8, planes: u8, stride: u16, w: u16, h: u16) -> Vec { + let mut pcx = vec![0u8; 128]; + pcx[..4].copy_from_slice(&[0x0A, version, 1, bits]); + pcx[8..10].copy_from_slice(&(w - 1).to_le_bytes()); + pcx[10..12].copy_from_slice(&(h - 1).to_le_bytes()); + pcx[12..14].copy_from_slice(&72u16.to_le_bytes()); + pcx[14..16].copy_from_slice(&72u16.to_le_bytes()); + pcx[65] = planes; + pcx[66..68].copy_from_slice(&stride.to_le_bytes()); + pcx +} + +/// A 24-bit PCX: each scanline holds a red, a green and a blue row. +#[test] +fn pcx_images_become_pages() { + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[10, 11, 20, 21, 30, 31]); + pcx.extend_from_slice(&[40, 41, 50, 51, 60, 61]); + let embedded = super::pcx("a.pcx", &pcx).unwrap(); + assert_eq!(embedded.data, [10, 20, 30, 11, 21, 31, 40, 50, 60, 41, 51, 61]); + let doc = reopen(&from_images(&[("a.pcx".into(), pcx)]).unwrap()); + assert_eq!(pages(&doc).len(), 1); + assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 2.0]); + assert_eq!(image_of(&doc, 0).name(b"ColorSpace"), Some(&b"DeviceRGB"[..])); +} + +#[test] +fn a_pcx_run_may_cross_planes_but_not_scanlines() { + // 0xC4 repeats 7 four times: the red row and half of the green one + let mut pcx = pcx_header(5, 8, 3, 2, 2, 1); + pcx.extend_from_slice(&[0xC4, 7, 8, 9]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [7, 7, 8, 7, 7, 9]); + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[0xC8, 7, 1, 2, 3, 4]); + assert!(super::pcx("a.pcx", &pcx).is_err(), "eight bytes in a six-byte scanline"); +} + +#[test] +fn an_8_bit_pcx_uses_its_palette_and_is_grey_without_one() { + let mut pcx = pcx_header(5, 8, 1, 2, 2, 1); + pcx.extend_from_slice(&[1, 2]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 1, 1, 2, 2, 2], "no palette: grey"); + pcx.push(0x0C); + let mut palette = vec![0u8; 768]; + palette[3..9].copy_from_slice(&[9, 8, 7, 6, 5, 4]); + pcx.extend_from_slice(&palette); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [9, 8, 7, 6, 5, 4]); +} + +#[test] +fn bit_plane_pcx_images_read_their_palette() { + // version 2, one plane: black and white + let mut pcx = pcx_header(2, 1, 1, 2, 2, 1); + pcx.extend_from_slice(&[0b1000_0000, 0]); + assert!(from_images(&[("a.pcx".into(), pcx.clone())]).is_ok(), "an old version is still a PCX"); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [255, 255, 255, 0, 0, 0]); + // two planes: colour 3 is at byte 16 + 9 + let mut pcx = pcx_header(5, 1, 2, 1, 1, 1); + pcx[25..28].copy_from_slice(&[1, 2, 3]); + pcx.extend_from_slice(&[0x80, 0x80]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 2, 3]); +} + +#[test] +fn damaged_pcx_headers_and_bodies_are_refused() { + let no_planes = pcx_header(5, 8, 0, 2, 2, 1); + let no_stride = pcx_header(5, 8, 3, 0, 2, 1); + let short_stride = pcx_header(5, 8, 3, 1, 2, 1); + let short_body = pcx_header(5, 8, 3, 2, 2, 1); + for bytes in [no_planes, no_stride, short_stride, short_body] { + assert!(super::pcx("a.pcx", &bytes).is_err()); + } + assert!(super::pcx("a.pcx", &[0x0A, 5, 1, 8]).is_err(), "no header"); +} + /// A synthetic ICC profile: a 128-byte header (size, device class, colour space, `acsp`) and /// some payload. Only the header is read. fn icc_profile(space: &[u8; 4]) -> Vec { From dc5cb122db4e79e795e92073340262f5c40d1344 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:23:18 +0200 Subject: [PATCH 14/14] trust certificate key type when CNG algorithm group is unreadable (#6391) * trust certificate key type when CNG algorithm group is unreadable When the algorithm group can't be read from the key store due to a provider bug, the certificate should not be rejected: its SubjectPublicKeyInfo already states the key type, and a real mismatch will surface at signing time instead of at enumeration. Change matches_key to fall back to Ok(()) when key.algorithm_group() returns an error, trusting the certificate's key type; extract the group comparison into key_group_matches(group: Option<&AlgorithmGroup>, want: StoreKey) so the fallback and mismatch cases can be unit tested without a live CNG key. Test: key_group_matches_trusts_when_group_unreadable failed before the change and passes after. * Sign: back to LF line endings, shorter test comment Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/sign/src/windows.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/sign/src/windows.rs b/crates/sign/src/windows.rs index 120fea3dab..61620ad384 100644 --- a/crates/sign/src/windows.rs +++ b/crates/sign/src/windows.rs @@ -178,7 +178,27 @@ fn entries() -> Result { /// Whether the store's key is what the certificate needs (RSA; ECDSA on its curve). fn matches_key(key: &NCryptKey, want: StoreKey) -> Result<(), String> { - let group = key.algorithm_group().map_err(|e| format!("its private key's algorithm couldn't be read: {}", describe(&e)))?; + // Some KSPs (Certisign's desktopID, for one) report NCRYPT_ALGORITHM_GROUP_PROPERTY + // as a character count instead of a byte count, so the group can't be read even though + // the key works. The certificate already names the key type, so an unreadable group is + // trusted: a working key is accepted instead of wrongly rejected. The key size, when it + // can be read, still has to agree with the certificate. + matches_key_from_group(key.algorithm_group().ok(), key, want) +} + +/// Core check, decoupled from the NCryptKey handle so the algorithm group can be injected +/// (e.g. forced to `None` to simulate a KSP that reports the property as a character count +/// instead of a byte count, as Certisign's desktopID KSP does). +fn matches_key_from_group(group: Option, key: &NCryptKey, want: StoreKey) -> Result<(), String> { + let Some(group) = group else { + return match want { + StoreKey::Rsa => Ok(()), + StoreKey::Ecdsa(bits) => match key.bits() { + Ok(actual) if actual != bits => Err(format!("its certificate holds a P-{bits} key but the store's private key has {actual} bits")), + _ => Ok(()), + }, + }; + }; let name = |g: &AlgorithmGroup| match g { AlgorithmGroup::Rsa => "an RSA".to_string(), AlgorithmGroup::Ecdsa => "an ECDSA".to_string(), @@ -272,4 +292,13 @@ mod tests { assert_eq!(describe(&CngError::WindowsError(0x57)), "Windows error 0x00000057"); assert_eq!(describe(&CngError::InvalidHashLength), "Invalid hash length"); } + + #[test] + fn key_group_matches_trusts_when_group_unreadable() { + // Certisign's desktopID KSP reports the algorithm-group property as a character + // count (3 for "RSA") instead of a byte count (8), so `algorithm_group` fails. An + // unreadable group must not make a usable key look unusable. + let key = NCryptKey::new_owned(0); + assert_eq!(matches_key(&key, StoreKey::Rsa), Ok(())); + } }