diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94a4c545fe..38ee53caf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,10 @@ jobs: permissions: contents: read steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index b7b1d409c8..78dc1b614b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -24,13 +24,17 @@ jobs: name: fuzz (mutation, 15 min) runs-on: ubuntu-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 with: toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: fuzz - name: Fetch the pdf.js corpus (seeds) diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index ea61d84303..1771a35fea 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -27,6 +27,10 @@ jobs: name: Windows installation policy (no app build) runs-on: windows-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b44d79b68..d214d6738b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,10 @@ name: Release on: + # A nightly dry run on main: every release build except macOS, nothing signed or released + # (craft-common-actions release.yml `dry-run`). Finds release-only breakage before release day. + schedule: + - cron: "20 2 * * *" push: branches: [release] workflow_dispatch: @@ -30,6 +34,7 @@ jobs: # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: + dry-run: ${{ github.event_name == 'schedule' }} app-name: pdfcraft app-title: PdfCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 diff --git a/Cargo.lock b/Cargo.lock index 9cf008f02e..e0968ff2c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1184,6 +1184,14 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-cms" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99" +dependencies = [ + "rayon", +] + [[package]] name = "craft-docpath" version = "0.1.0" @@ -1289,7 +1297,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "chrono", "dunce", @@ -1299,6 +1307,7 @@ dependencies = [ "image", "log", "plist", + "rustix 1.1.6", "semver", "serde", "serde_json", @@ -4263,7 +4272,7 @@ dependencies = [ [[package]] name = "pdfcraft" -version = "0.18.1" +version = "0.21.1" dependencies = [ "chrono", "craft-update", @@ -4288,7 +4297,7 @@ dependencies = [ [[package]] name = "pdfcraft-a11y" -version = "0.18.1" +version = "0.21.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4299,7 +4308,7 @@ dependencies = [ [[package]] name = "pdfcraft-annot" -version = "0.18.1" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4311,7 +4320,7 @@ dependencies = [ [[package]] name = "pdfcraft-audit" -version = "0.18.1" +version = "0.21.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4322,9 +4331,10 @@ dependencies = [ [[package]] name = "pdfcraft-automation" -version = "0.18.1" +version = "0.21.1" dependencies = [ "base64 0.23.1", + "craft-cms", "flate2", "image", "pdfcraft-cos", @@ -4346,7 +4356,7 @@ dependencies = [ [[package]] name = "pdfcraft-cli" -version = "0.18.1" +version = "0.21.1" dependencies = [ "base64 0.23.1", "pdfcraft-audit", @@ -4364,11 +4374,11 @@ dependencies = [ [[package]] name = "pdfcraft-compare" -version = "0.18.1" +version = "0.21.1" [[package]] name = "pdfcraft-content" -version = "0.18.1" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4376,7 +4386,7 @@ dependencies = [ [[package]] name = "pdfcraft-cos" -version = "0.18.1" +version = "0.21.1" dependencies = [ "chrono", "hayro-syntax", @@ -4393,7 +4403,7 @@ dependencies = [ [[package]] name = "pdfcraft-create" -version = "0.18.1" +version = "0.21.1" dependencies = [ "encoding_rs", "hayro-ccitt", @@ -4413,7 +4423,7 @@ dependencies = [ [[package]] name = "pdfcraft-crypt" -version = "0.18.1" +version = "0.21.1" dependencies = [ "aes", "md-5", @@ -4426,7 +4436,7 @@ dependencies = [ [[package]] name = "pdfcraft-edit" -version = "0.18.1" +version = "0.21.1" dependencies = [ "hayro-syntax", "icu_segmenter", @@ -4445,9 +4455,10 @@ dependencies = [ [[package]] name = "pdfcraft-engine" -version = "0.18.1" +version = "0.21.1" dependencies = [ "chrono", + "craft-cms", "craft-docpath", "getrandom 0.4.3", "icu_properties", @@ -4489,14 +4500,14 @@ dependencies = [ [[package]] name = "pdfcraft-export" -version = "0.18.1" +version = "0.21.1" dependencies = [ "flate2", ] [[package]] name = "pdfcraft-filters" -version = "0.18.1" +version = "0.21.1" dependencies = [ "fax", "flate2", @@ -4509,7 +4520,7 @@ dependencies = [ [[package]] name = "pdfcraft-fonts" -version = "0.18.1" +version = "0.21.1" dependencies = [ "harfrust", "pdfcraft-cos", @@ -4519,7 +4530,7 @@ dependencies = [ [[package]] name = "pdfcraft-forms" -version = "0.18.1" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4532,11 +4543,11 @@ dependencies = [ [[package]] name = "pdfcraft-geom" -version = "0.18.1" +version = "0.21.1" [[package]] name = "pdfcraft-js" -version = "0.18.1" +version = "0.21.1" dependencies = [ "boa_engine", "chrono", @@ -4544,7 +4555,7 @@ dependencies = [ [[package]] name = "pdfcraft-measure" -version = "0.18.1" +version = "0.21.1" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4557,14 +4568,14 @@ dependencies = [ [[package]] name = "pdfcraft-model" -version = "0.18.1" +version = "0.21.1" dependencies = [ "pdfcraft-cos", ] [[package]] name = "pdfcraft-ocr" -version = "0.18.1" +version = "0.21.1" dependencies = [ "ocrs", "pdfcraft-fonts", @@ -4579,7 +4590,7 @@ dependencies = [ [[package]] name = "pdfcraft-ocr-worker" -version = "0.18.1" +version = "0.21.1" dependencies = [ "js-sys", "pdfcraft-ocr", @@ -4589,7 +4600,7 @@ dependencies = [ [[package]] name = "pdfcraft-optimize" -version = "0.18.1" +version = "0.21.1" dependencies = [ "image", "pdfcraft-annot", @@ -4606,11 +4617,11 @@ dependencies = [ [[package]] name = "pdfcraft-oracle" -version = "0.18.1" +version = "0.21.1" [[package]] name = "pdfcraft-organize" -version = "0.18.1" +version = "0.21.1" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4621,7 +4632,7 @@ dependencies = [ [[package]] name = "pdfcraft-platform" -version = "0.18.1" +version = "0.21.1" dependencies = [ "async-channel", "futures-lite", @@ -4632,7 +4643,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc" -version = "0.18.1" +version = "0.21.1" dependencies = [ "flate2", "proptest", @@ -4641,7 +4652,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc-render" -version = "0.18.1" +version = "0.21.1" dependencies = [ "flate2", "pdfcraft-prc", @@ -4650,7 +4661,7 @@ dependencies = [ [[package]] name = "pdfcraft-preflight" -version = "0.18.1" +version = "0.21.1" dependencies = [ "flate2", "pdfcraft-content", @@ -4661,8 +4672,9 @@ dependencies = [ [[package]] name = "pdfcraft-print" -version = "0.18.1" +version = "0.21.1" dependencies = [ + "craft-cms", "craft-print", "log", "pdfcraft-content", @@ -4676,7 +4688,7 @@ dependencies = [ [[package]] name = "pdfcraft-redact" -version = "0.18.1" +version = "0.21.1" dependencies = [ "aho-corasick", "pdfcraft-annot", @@ -4700,7 +4712,7 @@ dependencies = [ [[package]] name = "pdfcraft-render" -version = "0.18.1" +version = "0.21.1" dependencies = [ "bytemuck", "flate2", @@ -4724,7 +4736,7 @@ dependencies = [ [[package]] name = "pdfcraft-sign" -version = "0.18.1" +version = "0.21.1" dependencies = [ "aes", "aws-lc-rs", @@ -4763,11 +4775,11 @@ dependencies = [ [[package]] name = "pdfcraft-testkit" -version = "0.18.1" +version = "0.21.1" [[package]] name = "pdfcraft-ui-egui" -version = "0.18.1" +version = "0.21.1" dependencies = [ "arboard", "base64 0.23.1", @@ -4826,7 +4838,7 @@ dependencies = [ [[package]] name = "pdfcraft-web" -version = "0.18.1" +version = "0.21.1" dependencies = [ "craft-web-api", "craft-web-host", @@ -4841,7 +4853,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfa" -version = "0.18.1" +version = "0.21.1" dependencies = [ "base64 0.23.1", "image", @@ -4855,7 +4867,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfdf" -version = "0.18.1" +version = "0.21.1" dependencies = [ "base64 0.23.1", "pdfcraft-annot", diff --git a/Cargo.toml b/Cargo.toml index d566a69d1f..18fa9ed37b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor"] default-members = ["crates/*", "apps/pdfcraft", "apps/pdfcraft-cli", "xtask"] [workspace.package] -version = "0.18.1" +version = "0.21.1" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -27,46 +27,48 @@ craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1 craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # Shared system spooler; document composition remains in pdfcraft-print. craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } +# ICC transforms for app-managed print colour; at craft-print's commit, so one craft-libs checkout. +craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } # Whether a path stored in a document may be opened (GoToR link targets). craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", features = ["egui"] } # internal -pdfcraft-geom = { path = "crates/geom", version = "0.18.1" } -pdfcraft-filters = { path = "crates/filters", version = "0.18.1" } -pdfcraft-cos = { path = "crates/cos", version = "0.18.1" } -pdfcraft-crypt = { path = "crates/crypt", version = "0.18.1" } -pdfcraft-organize = { path = "crates/organize", version = "0.18.1" } -pdfcraft-measure = { path = "crates/measure", version = "0.18.1" } -pdfcraft-annot = { path = "crates/annot", version = "0.18.1" } -pdfcraft-audit = { path = "crates/audit", version = "0.18.1" } -pdfcraft-fonts = { path = "crates/fonts", version = "0.18.1" } -pdfcraft-model = { path = "crates/model", version = "0.18.1" } -pdfcraft-content = { path = "crates/content", version = "0.18.1" } -pdfcraft-redact = { path = "crates/redact", version = "0.18.1" } -pdfcraft-print = { path = "crates/print", version = "0.18.1" } -pdfcraft-xfdf = { path = "crates/xfdf", version = "0.18.1" } -pdfcraft-sign = { path = "crates/sign", version = "0.18.1" } -pdfcraft-optimize = { path = "crates/optimize", version = "0.18.1" } -pdfcraft-a11y = { path = "crates/a11y", version = "0.18.1" } -pdfcraft-oracle = { path = "crates/oracle", version = "0.18.1" } -pdfcraft-testkit = { path = "crates/testkit", version = "0.18.1" } -pdfcraft-ocr = { path = "crates/ocr", version = "0.18.1" } -pdfcraft-js = { path = "crates/js", version = "0.18.1" } -pdfcraft-compare = { path = "crates/compare", version = "0.18.1" } -pdfcraft-preflight = { path = "crates/preflight", version = "0.18.1" } -pdfcraft-export = { path = "crates/export", version = "0.18.1" } -pdfcraft-edit = { path = "crates/edit", version = "0.18.1" } -pdfcraft-create = { path = "crates/create", version = "0.18.1" } -pdfcraft-forms = { path = "crates/forms", version = "0.18.1" } -pdfcraft-xfa = { path = "crates/xfa", version = "0.18.1" } -pdfcraft-render = { path = "crates/render", version = "0.18.1" } -pdfcraft-engine = { path = "crates/engine", version = "0.18.1" } -pdfcraft-platform = { path = "crates/platform", version = "0.18.1" } -pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.18.1" } -pdfcraft-automation = { path = "crates/automation", version = "0.18.1", default-features = false } -pdfcraft-prc = { path = "crates/prc", version = "0.18.1" } -pdfcraft-prc-render = { path = "crates/prc-render", version = "0.18.1" } +pdfcraft-geom = { path = "crates/geom", version = "0.21.0" } +pdfcraft-filters = { path = "crates/filters", version = "0.21.0" } +pdfcraft-cos = { path = "crates/cos", version = "0.21.0" } +pdfcraft-crypt = { path = "crates/crypt", version = "0.21.0" } +pdfcraft-organize = { path = "crates/organize", version = "0.21.0" } +pdfcraft-measure = { path = "crates/measure", version = "0.21.0" } +pdfcraft-annot = { path = "crates/annot", version = "0.21.0" } +pdfcraft-audit = { path = "crates/audit", version = "0.21.0" } +pdfcraft-fonts = { path = "crates/fonts", version = "0.21.0" } +pdfcraft-model = { path = "crates/model", version = "0.21.0" } +pdfcraft-content = { path = "crates/content", version = "0.21.0" } +pdfcraft-redact = { path = "crates/redact", version = "0.21.0" } +pdfcraft-print = { path = "crates/print", version = "0.21.0" } +pdfcraft-xfdf = { path = "crates/xfdf", version = "0.21.0" } +pdfcraft-sign = { path = "crates/sign", version = "0.21.0" } +pdfcraft-optimize = { path = "crates/optimize", version = "0.21.0" } +pdfcraft-a11y = { path = "crates/a11y", version = "0.21.0" } +pdfcraft-oracle = { path = "crates/oracle", version = "0.21.0" } +pdfcraft-testkit = { path = "crates/testkit", version = "0.21.0" } +pdfcraft-ocr = { path = "crates/ocr", version = "0.21.0" } +pdfcraft-js = { path = "crates/js", version = "0.21.0" } +pdfcraft-compare = { path = "crates/compare", version = "0.21.0" } +pdfcraft-preflight = { path = "crates/preflight", version = "0.21.0" } +pdfcraft-export = { path = "crates/export", version = "0.21.0" } +pdfcraft-edit = { path = "crates/edit", version = "0.21.0" } +pdfcraft-create = { path = "crates/create", version = "0.21.0" } +pdfcraft-forms = { path = "crates/forms", version = "0.21.0" } +pdfcraft-xfa = { path = "crates/xfa", version = "0.21.0" } +pdfcraft-render = { path = "crates/render", version = "0.21.0" } +pdfcraft-engine = { path = "crates/engine", version = "0.21.0" } +pdfcraft-platform = { path = "crates/platform", version = "0.21.0" } +pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.21.0" } +pdfcraft-automation = { path = "crates/automation", version = "0.21.0", default-features = false } +pdfcraft-prc = { path = "crates/prc", version = "0.21.0" } +pdfcraft-prc-render = { path = "crates/prc-render", version = "0.21.0" } # pdf hayro = "0.7.1" diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 59d30930ff..bf3df8d0a4 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -1,7 +1,8 @@ //! Create a valid, reviewable structure tree for untagged PDFs. //! -//! Each text object becomes a paragraph, which can then be retagged with the existing -//! tag-property tool after human review. Painted paths, shadings and images outside text objects +//! Each text object becomes a paragraph, or a heading when its size or weight sets it apart from +//! the body text ([`crate::autotag_roles`]); either can be retagged with the existing tag-property +//! tool after human review. Painted paths, shadings and images outside text objects //! are wrapped in `/Artifact BMC … EMC` so the resulting page passes the tagged-content check. use std::collections::{HashMap, HashSet}; @@ -9,6 +10,8 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::{Op, Pieces, serialize_ops}; use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; +use crate::autotag_roles::{self, Evidence}; + const MAX_PAGES: usize = 1_000; const MAX_PAGE_BYTES: usize = 16 * 1024 * 1024; const MAX_OPERATORS: usize = 100_000; @@ -22,6 +25,7 @@ const MAX_TEXT_OBJECTS: usize = 4_096; pub struct AutoTagResult { pub pages: usize, pub paragraphs: usize, + pub headings: usize, } #[derive(Debug, thiserror::Error, PartialEq)] @@ -44,6 +48,7 @@ struct Plan { page: ObjRef, ops: Vec, spans: Vec<(usize, usize)>, + evidence: Vec, artifacts: HashMap, } @@ -98,6 +103,14 @@ fn page_xobjects(doc: &Document, page: &Dict) -> Dict { .unwrap_or_default() } +fn page_fonts(doc: &Document, page: &Dict) -> Dict { + page.get(b"Resources") + .map(|r| doc.resolve(r)) + .and_then(|r| r.as_dict().and_then(|r| r.get(b"Font").map(|x| doc.resolve(x)))) + .and_then(|x| x.as_dict().cloned()) + .unwrap_or_default() +} + fn harmless_outside(op: &[u8]) -> bool { matches!( op, @@ -183,7 +196,13 @@ fn harmless_inside(op: &[u8]) -> bool { /// `totals` is the decoded bytes and operators of the pages planned so far, bounded by /// `MAX_TOTAL_BYTES` and `MAX_TOTAL_OPERATORS`. -fn plan_page(doc: &Document, page: &pdfcraft_model::Page, index: usize, totals: &mut (usize, usize)) -> Result { +fn plan_page( + doc: &Document, + page: &pdfcraft_model::Page, + index: usize, + totals: &mut (usize, usize), + fonts: &mut autotag_roles::Fonts, +) -> Result { if page.dict.contains(b"StructParents") { return Err(unsupported(index, "page already has a structure parent key")); } @@ -248,7 +267,9 @@ fn plan_page(doc: &Document, page: &pdfcraft_model::Page, index: usize, totals: if path.is_some() { return Err(unsupported(index, "unterminated path object")); } - Ok(Plan { page: page.obj, ops: parsed.ops, spans, artifacts }) + let resources = page_fonts(doc, &page.dict); + let evidence = autotag_roles::evidence(doc, fonts, &resources, &parsed.ops, &spans); + Ok(Plan { page: page.obj, ops: parsed.ops, spans, evidence, artifacts }) } fn plan(doc: &Document) -> Result, AutoTagError> { @@ -268,8 +289,9 @@ fn plan(doc: &Document) -> Result, AutoTagError> { let mut out = Vec::with_capacity(pages.len()); let mut total = 0usize; let mut totals = (0usize, 0usize); + let mut fonts = autotag_roles::Fonts::default(); for (index, page) in pages.iter().enumerate() { - let planned = plan_page(doc, page, index, &mut totals)?; + let planned = plan_page(doc, page, index, &mut totals, &mut fonts)?; total = total.saturating_add(planned.spans.len()); out.push(planned); } @@ -279,16 +301,21 @@ fn plan(doc: &Document) -> Result, AutoTagError> { Ok(out) } -fn mark_ops(ops: Vec, starts: &HashMap, ends: &HashSet, artifacts: &HashMap) -> Result, AutoTagError> { +fn mark_ops( + ops: Vec, + starts: &HashMap, + ends: &HashSet, + artifacts: &HashMap, +) -> Result, AutoTagError> { let artifact_ends: HashSet = artifacts.values().copied().collect(); let extra = starts.len().saturating_add(artifacts.len()).saturating_mul(2); let mut marked = Vec::with_capacity(ops.len().saturating_add(extra)); for (at, op) in ops.into_iter().enumerate() { let is_artifact = artifacts.contains_key(&at); - if let Some(mcid) = starts.get(&at) { + if let Some(&(mcid, role)) = starts.get(&at) { let mut properties = Dict::new(); - properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(*mcid).map_err(|_| AutoTagError::TooManyPages)?)); - marked.push(Op::new("BDC", vec![Object::name("P"), Object::Dict(properties)])); + properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); + marked.push(Op::new("BDC", vec![Object::name(role), Object::Dict(properties)])); } else if is_artifact { marked.push(Op::new("BMC", vec![Object::name("Artifact")])); } @@ -334,16 +361,18 @@ fn write_structure( Ok(()) } -/// Tag every text object in an untagged PDF as a paragraph and mark painted paths, shadings and +/// Tag every text object in an untagged PDF as a paragraph or a heading and mark painted paths, shadings and /// images outside text objects as artifacts. The full structure tree, parent tree and /// marked-content IDs are written together. Unsupported operators are refused before any -/// mutation. On success the caller can use tag-property edits to assign reviewed heading roles. +/// mutation. On success the caller can use tag-property edits to correct the roles after review. pub fn autotag_text(doc: &mut Document) -> Result { let plans = plan(doc)?; + let all: Vec = plans.iter().flat_map(|p| p.evidence.iter().copied()).collect(); + let mut roles = autotag_roles::roles(&all).into_iter(); let mut next = doc.clone(); let catalog = next.root().ok_or(AutoTagError::NoCatalog)?; let (root, document) = (next.add(Object::Null), next.add(Object::Null)); - let (mut children, mut numbers, mut tagged_pages, mut paragraphs) = (Vec::new(), Vec::new(), 0usize, 0usize); + let (mut children, mut numbers, mut tagged_pages, mut paragraphs, mut headings) = (Vec::new(), Vec::new(), 0usize, 0usize, 0usize); for page in plans { if page.spans.is_empty() && page.artifacts.is_empty() { continue; @@ -353,18 +382,23 @@ pub fn autotag_text(doc: &mut Document) -> Result { let key = i64::try_from(tagged_pages).map_err(|_| AutoTagError::TooManyPages)?; let mut parents = Vec::with_capacity(page.spans.len()); for (mcid, (first, last)) in page.spans.iter().enumerate() { + let role = roles.next().unwrap_or("P"); let mut element = Dict::new(); element.set(b"Type".to_vec(), Object::name("StructElem")); - element.set(b"S".to_vec(), Object::name("P")); + element.set(b"S".to_vec(), Object::name(role)); element.set(b"P".to_vec(), Object::Ref(document)); element.set(b"Pg".to_vec(), Object::Ref(page.page)); element.set(b"K".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); let reference = next.add(element); parents.push(Object::Ref(reference)); children.push(Object::Ref(reference)); - starts.insert(*first, mcid); + starts.insert(*first, (mcid, role)); ends.insert(*last); - paragraphs = paragraphs.saturating_add(1); + if role == "P" { + paragraphs = paragraphs.saturating_add(1); + } else { + headings = headings.saturating_add(1); + } } next.update_dict(page.page, |d| d.set(b"StructParents".to_vec(), Object::Int(key)))?; numbers.extend([Object::Int(key), Object::Array(parents)]); @@ -376,5 +410,5 @@ pub fn autotag_text(doc: &mut Document) -> Result { } write_structure(&mut next, catalog, root, document, children, numbers, tagged_pages)?; *doc = next; - Ok(AutoTagResult { pages: tagged_pages, paragraphs }) + Ok(AutoTagResult { pages: tagged_pages, paragraphs, headings }) } diff --git a/crates/a11y/src/autotag_roles.rs b/crates/a11y/src/autotag_roles.rs new file mode 100644 index 0000000000..a72e14a417 --- /dev/null +++ b/crates/a11y/src/autotag_roles.rs @@ -0,0 +1,382 @@ +//! Heading roles for auto-tagged text objects, from the size and weight of the text each shows. +//! +//! The body style is the size (to the half point) carrying the most characters in the document, +//! and whether most of those characters are bold. A text object is a heading when it shows at most +//! three lines and 200 characters, including a letter, and is either 15% larger than the body or, +//! where the body is not bold, bold at no less than 90% of the body size on at most two lines. +//! Levels rank the heading styles (size, then bold) over the whole document, largest first, capped +//! at 6, and never skip a level after the heading before (the `heading-nesting` check). + +use std::collections::HashMap; +use std::rc::Rc; + +use pdfcraft_content::{Matrix, Op}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object}; +use pdfcraft_fonts::pdf::Metrics; + +/// A line this much larger than body text reads as a heading. +const HEADING_SIZE_RATIO: f64 = 1.15; +/// Bold text this much of the body size or more can be a heading by weight. +const WEIGHT_SIZE_RATIO: f64 = 0.9; +const MAX_LINES: usize = 3; +const MAX_WEIGHT_LINES: usize = 2; +const MAX_CHARS: usize = 200; +/// Saved graphics states followed; deeper `q` nesting keeps the current state. +const MAX_SAVED: usize = 64; +/// Distinct fonts measured per document; text in any further font counts as unstyled. +const MAX_FONTS: usize = 256; + +/// What one text object shows. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub(crate) struct Evidence { + /// The size, in user space, carrying the most of the object's characters. + pub size: f64, + /// Most of the object's characters are in a bold font. + pub bold: bool, + pub lines: usize, + /// Characters shown, whitespace excluded. + pub chars: usize, + pub letters: bool, +} + +/// The structure type for each text object, in document order: `H1`–`H6` or `P`. +pub(crate) fn roles(objects: &[Evidence]) -> Vec<&'static str> { + const HEADINGS: [&str; 6] = ["H1", "H2", "H3", "H4", "H5", "H6"]; + let Some((body, body_bold)) = body_style(objects) else { return vec!["P"; objects.len()] }; + let candidate = |e: &Evidence| { + let fits = e.lines >= 1 && e.lines <= MAX_LINES && e.chars <= MAX_CHARS && e.letters; + let by_size = e.size >= body * HEADING_SIZE_RATIO; + let by_weight = e.bold && !body_bold && e.lines <= MAX_WEIGHT_LINES && e.size >= body * WEIGHT_SIZE_RATIO; + fits && (by_size || by_weight) + }; + let mut styles: Vec<(i64, bool)> = objects.iter().filter(|e| candidate(e)).map(|e| (half_points(e.size), e.bold)).collect(); + styles.sort_by(|a, b| b.cmp(a)); + styles.dedup(); + let ranks: HashMap<(i64, bool), usize> = styles.into_iter().enumerate().map(|(rank, style)| (style, rank)).collect(); + let mut previous = 0usize; + objects + .iter() + .map(|e| { + if !candidate(e) { + return "P"; + } + let rank = ranks.get(&(half_points(e.size), e.bold)).copied().unwrap_or(0); + let level = rank.saturating_add(1).min(previous.saturating_add(1)).min(HEADINGS.len()); + previous = level; + HEADINGS.get(level.saturating_sub(1)).copied().unwrap_or("P") + }) + .collect() +} + +fn half_points(size: f64) -> i64 { + // Sizes are finite and bounded by `evidence`; `as` saturates in any case. + (size * 2.0).round() as i64 +} + +/// The size carrying the most characters, and whether most of them are bold. +fn body_style(objects: &[Evidence]) -> Option<(f64, bool)> { + let mut by_size: HashMap = HashMap::new(); + for e in objects.iter().filter(|e| e.chars > 0) { + let entry = by_size.entry(half_points(e.size)).or_default(); + entry.0 = entry.0.saturating_add(e.chars); + if e.bold { + entry.1 = entry.1.saturating_add(e.chars); + } + } + let (size, (chars, bold)) = by_size.into_iter().filter(|&(s, _)| s > 0).max_by_key(|&(s, (n, _))| (n, s))?; + Some((size as f64 / 2.0, bold.saturating_mul(2) > chars)) +} + +#[derive(Clone)] +struct State { + ctm: Matrix, + tm: Matrix, + tlm: Matrix, + size: f64, + leading: f64, + /// Text render mode 3 or 7: shown text that paints nothing, such as an OCR layer or a + /// clip-only object. + invisible: bool, + metrics: Option>, +} + +/// Font metrics for a whole document, measured once per font object and capped at +/// [`MAX_FONTS`]. +#[derive(Default)] +pub(crate) struct Fonts { + by_ref: HashMap>>, + measured: usize, +} + +impl Fonts { + fn get(&mut self, doc: &Document, font: &Object) -> Option> { + let key = match font { + Object::Ref(r) => Some(*r), + _ => None, + }; + if let Some(found) = key.and_then(|r| self.by_ref.get(&r)) { + return found.clone(); + } + if self.measured >= MAX_FONTS { + return None; + } + self.measured = self.measured.saturating_add(1); + let metrics = doc.resolve(font).as_dict().map(|d| Rc::new(Metrics::from_dict(doc, d))); + if let Some(r) = key { + self.by_ref.insert(r, metrics.clone()); + } + metrics + } +} + +#[derive(Default)] +struct Shown { + by_size: HashMap, + lines: usize, + shown: bool, + moved: bool, + letters: bool, +} + +impl Shown { + fn show(&mut self, state: &State, bytes: &[u8]) { + let Some(metrics) = state.metrics.as_ref().filter(|_| !state.invisible) else { return }; + let text = metrics.decode(bytes); + let chars = text.chars().filter(|c| !c.is_whitespace()).count(); + if chars == 0 { + return; + } + self.letters |= text.chars().any(char::is_alphabetic); + if !self.shown { + (self.shown, self.lines) = (true, 1); + } else if self.moved { + self.lines = self.lines.saturating_add(1); + } + self.moved = false; + let [_, _, c, d, _, _] = state.tm.then(&state.ctm).0; + let size = (state.size * c.hypot(d)).abs(); + if !size.is_finite() || size <= 0.0 || size > 1e6 { + return; + } + let entry = self.by_size.entry(half_points(size)).or_default(); + entry.0 = entry.0.saturating_add(chars); + if metrics.bold { + entry.1 = entry.1.saturating_add(chars); + } + } + + /// A line move counts once text is shown after it, so blank lines and trailing moves do not. + fn new_line(&mut self) { + self.moved = self.shown; + } + + fn finish(self) -> Evidence { + let chars = self.by_size.values().fold(0usize, |n, v| n.saturating_add(v.0)); + let Some((size, (count, bold))) = self.by_size.into_iter().max_by_key(|&(s, (n, _))| (n, s)) else { + return Evidence::default(); + }; + Evidence { size: size as f64 / 2.0, bold: bold.saturating_mul(2) > count, lines: self.lines, chars, letters: self.letters } + } +} + +/// The evidence for each `(BT, ET)` operator span of a page, in order. `resources` is the page's +/// `/Font` resource dictionary; `fonts` is shared by the document's pages. +pub(crate) fn evidence(doc: &Document, fonts: &mut Fonts, resources: &Dict, ops: &[Op], spans: &[(usize, usize)]) -> Vec { + let mut by_name: HashMap, Option>> = HashMap::new(); + let mut state = + State { ctm: Matrix::IDENTITY, tm: Matrix::IDENTITY, tlm: Matrix::IDENTITY, size: 0.0, leading: 0.0, invisible: false, metrics: None }; + let (mut saved, mut unsaved) = (Vec::new(), 0usize); + let mut out = Vec::with_capacity(spans.len()); + let mut spans = spans.iter().peekable(); + let mut current: Option<(usize, Shown)> = None; + for (at, op) in ops.iter().enumerate() { + if current.is_none() && spans.peek().is_some_and(|s| s.0 == at) { + current = spans.next().map(|s| (s.1, Shown::default())); + } + let object = current.as_mut().map(|c| &mut c.1); + match op.op.as_slice() { + b"q" if saved.len() < MAX_SAVED => saved.push(state.clone()), + b"q" => unsaved = unsaved.saturating_add(1), + b"Q" if unsaved > 0 => unsaved -= 1, + b"Q" => { + if let Some(s) = saved.pop() { + state = s; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + state.ctm = Matrix(m).then(&state.ctm); + } + } + b"BT" => (state.tm, state.tlm) = (Matrix::IDENTITY, Matrix::IDENTITY), + b"Tf" => { + state.size = op.num(1).unwrap_or(state.size); + state.metrics = op + .name(0) + .and_then(|name| by_name.entry(name.to_vec()).or_insert_with(|| resources.get(name).and_then(|f| fonts.get(doc, f))).clone()); + } + b"TL" => state.leading = op.num(0).unwrap_or(state.leading), + b"Tr" => state.invisible = matches!(op.num(0), Some(m) if m == 3.0 || m == 7.0), + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if op.op == b"TD" { + state.leading = -y; + } + state.tlm = Matrix::translate(x, y).then(&state.tlm); + state.tm = state.tlm; + if y != 0.0 + && let Some(o) = object + { + o.new_line(); + } + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + let moved = m[5] != state.tlm.0[5]; + (state.tm, state.tlm) = (Matrix(m), Matrix(m)); + if moved && let Some(o) = object { + o.new_line(); + } + } + } + b"T*" | b"'" | b"\"" => { + state.tlm = Matrix::translate(0.0, -state.leading).then(&state.tlm); + state.tm = state.tlm; + if let Some(o) = object { + o.new_line(); + if op.op != b"T*" + && let Some(Object::String(s)) = op.operands.last() + { + o.show(&state, &s.bytes); + } + } + } + b"Tj" => { + if let (Some(o), Some(Object::String(s))) = (object, op.operands.first()) { + o.show(&state, &s.bytes); + } + } + b"TJ" => { + if let (Some(o), Some(Object::Array(items))) = (object, op.operands.first()) { + for item in items { + if let Object::String(s) = item { + o.show(&state, &s.bytes); + } + } + } + } + _ => {} + } + if current.as_ref().is_some_and(|c| c.0 == at) + && let Some((_, o)) = current.take() + { + out.push(o.finish()); + } + } + out.resize(spans.len().saturating_add(out.len()), Evidence::default()); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(size: f64, chars: usize) -> Evidence { + Evidence { size, bold: false, lines: 1, chars, letters: true } + } + + fn bold(e: Evidence) -> Evidence { + Evidence { bold: true, ..e } + } + + #[test] + fn a_line_larger_than_the_body_is_a_heading_and_body_text_is_not() { + let objects = [text(20.0, 10), text(12.0, 400), text(12.0, 300)]; + assert_eq!(roles(&objects), ["H1", "P", "P"]); + } + + #[test] + fn the_body_is_the_size_with_the_most_characters_not_the_most_objects() { + // Ten short 9 pt captions and three 12 pt body lines: the body lines hold more characters, + // so 12 pt is the body. Counting objects would make 9 pt the body and the body lines headings. + let mut objects = vec![text(12.0, 150); 3]; + objects.extend(std::iter::repeat_n(text(9.0, 5), 10)); + assert!(roles(&objects).iter().all(|r| *r == "P")); + } + + #[test] + fn bold_at_body_size_is_a_heading_only_where_the_body_is_not_bold() { + let plain_body = [bold(text(12.0, 15)), text(12.0, 500)]; + assert_eq!(roles(&plain_body), ["H1", "P"]); + let bold_body = [bold(text(12.0, 15)), bold(text(12.0, 500))]; + assert_eq!(roles(&bold_body), ["P", "P"]); + } + + #[test] + fn long_text_numbers_alone_and_four_lines_stay_paragraphs() { + let long = text(20.0, MAX_CHARS + 1); + let number = Evidence { letters: false, ..text(20.0, 2) }; + let four_lines = Evidence { lines: 4, ..text(20.0, 40) }; + let two_bold_lines = Evidence { lines: 3, ..bold(text(12.0, 40)) }; + assert_eq!(roles(&[long, number, four_lines, two_bold_lines, text(12.0, 2_000)]), ["P"; 5]); + } + + #[test] + fn levels_rank_styles_largest_first_and_never_skip_a_level() { + // Styles: 24 pt, 18 pt, bold 12 pt. The first heading met is 18 pt, ranked second, which + // would skip H1: it becomes H1, and the bold heading after it H2, not H3. + let objects = [text(18.0, 10), bold(text(12.0, 10)), text(24.0, 10), text(18.0, 10), bold(text(12.0, 10)), text(12.0, 3_000)]; + assert_eq!(roles(&objects), ["H1", "H2", "H1", "H2", "H3", "P"]); + } + + #[test] + fn levels_stop_at_six() { + let mut objects: Vec = (0..8).map(|i| text(40.0 - f64::from(i) * 2.0, 10)).collect(); + objects.push(text(12.0, 5_000)); + assert_eq!(roles(&objects), ["H1", "H2", "H3", "H4", "H5", "H6", "H6", "H6", "P"]); + } + + #[test] + fn a_document_without_text_has_only_paragraphs() { + assert_eq!(roles(&[Evidence::default(), Evidence::default()]), ["P", "P"]); + assert!(roles(&[]).is_empty()); + } + + /// The evidence for each `BT`..`ET` of `content`, drawn in Helvetica-Bold as `/F1`. + fn walk(content: &[u8]) -> Vec { + let doc = Document::new_empty(); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type1")); + font.set(b"BaseFont".to_vec(), Object::name("Helvetica-Bold")); + let mut resources = Dict::new(); + resources.set(b"F1".to_vec(), Object::Dict(font)); + let ops = pdfcraft_content::parse(content).ops; + let at = |name: &[u8]| -> Vec { ops.iter().enumerate().filter(|(_, o)| o.op == name).map(|(i, _)| i).collect() }; + let spans: Vec<(usize, usize)> = at(b"BT").into_iter().zip(at(b"ET")).collect(); + evidence(&doc, &mut Fonts::default(), &resources, &ops, &spans) + } + + #[test] + fn size_comes_from_the_font_size_the_text_matrix_and_the_ctm() { + let found = walk( + b"q 2 0 0 2 0 0 cm BT /F1 1 Tf 9 0 0 9 10 10 Tm (Title) Tj ET Q BT /F1 12 Tf 10 50 Td (one) Tj T* (two) ' 0 -14 Td (three) Tj ET BT ET", + ); + assert_eq!(found[0], Evidence { size: 18.0, bold: true, lines: 1, chars: 5, letters: true }); + assert_eq!(found[1], Evidence { size: 12.0, bold: true, lines: 3, chars: 11, letters: true }); + assert_eq!(found[2], Evidence::default()); + assert_eq!(found.len(), 3); + } + + #[test] + fn line_moves_leading_invisible_text_and_absurd_sizes() { + let found = walk( + b"BT /F1 10 Tf 1 0 0 1 0 90 Tm (a) Tj 1 0 0 1 40 90 Tm (b) Tj 1 0 0 1 0 70 Tm (c) Tj ET BT /F1 10 Tf 14 TL 0 0 Td (a) Tj 0 0 (b) \" 0 -12 TD (c) Tj T* (d) Tj ET BT /F1 10 Tf 3 Tr (hidden) Tj 7 Tr (clip) Tj 0 Tr (shown) Tj ET BT /F1 2000000 Tf (huge) Tj ET", + ); + assert_eq!(found[0].lines, 2, "a Tm on the same baseline is not a new line"); + assert_eq!(found[1].lines, 4, "\" and TD each start a line, and T* uses TD's leading"); + assert_eq!(found[2].chars, 5, "render mode 3 and 7 text is not counted"); + assert_eq!(found[3], Evidence::default(), "a size past 1e6 is dropped"); + } +} diff --git a/crates/a11y/src/headings.rs b/crates/a11y/src/headings.rs index 235ba4b452..a9de970b65 100644 --- a/crates/a11y/src/headings.rs +++ b/crates/a11y/src/headings.rs @@ -206,7 +206,9 @@ fn sub(doc: &Document, d: &Dict, key: &[u8]) -> Dict { pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { let res = sub(doc, page, b"Resources"); let (fonts, props) = (sub(doc, &res, b"Font"), sub(doc, &res, b"Properties")); - let mut metrics: HashMap, Metrics> = HashMap::new(); + let mut metrics: HashMap, std::rc::Rc> = HashMap::new(); + // Built once per font object, however many resource names alias it. + let mut by_font: HashMap> = HashMap::new(); // The id each open marked-content sequence is in: its own, else the enclosing one's. let mut stack: Vec> = Vec::new(); let mut out: HashMap = HashMap::new(); @@ -255,9 +257,30 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { gs.font = op.name(0).map(<[u8]>::to_vec); if let Some(name) = op.name(0) && !metrics.contains_key(name) - && let Some(f) = fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()) { - metrics.insert(name.to_vec(), Metrics::from_dict(doc, &f)); + // Keyed by the object at the end of the reference chain, so names that reach one + // font through different chains share its metrics (the hop limit is resolve's). + let font = fonts.get(name).and_then(|o| { + let (mut cur, mut last) = (o.clone(), None); + for _ in 0..16 { + let Object::Ref(r) = cur else { break }; + last = Some(r); + cur = (*doc.get(r)).clone(); + } + last + }); + let built = match font.and_then(|r| by_font.get(&r)) { + Some(m) => Some(m.clone()), + None => { + fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().map(|d| std::rc::Rc::new(Metrics::from_dict(doc, d)))) + } + }; + if let Some(m) = built { + if let Some(r) = font { + by_font.insert(r, m.clone()); + } + metrics.insert(name.to_vec(), m); + } } } b"Tc" => gs.char_spacing = op.num(0).unwrap_or(0.0), diff --git a/crates/a11y/src/lib.rs b/crates/a11y/src/lib.rs index 50e632dec8..d3492f50e7 100644 --- a/crates/a11y/src/lib.rs +++ b/crates/a11y/src/lib.rs @@ -15,6 +15,7 @@ use pdfcraft_cos::Document; mod alt; mod autotag; mod autotag_fields; +mod autotag_roles; mod content; mod headings; mod layout_table; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 890eb33e9f..4a981aaf5b 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -50,7 +50,7 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { "", ); let result = autotag_text(&mut doc).unwrap(); - assert_eq!(result, AutoTagResult { pages: 2, paragraphs: 3 }); + assert_eq!(result, AutoTagResult { pages: 2, paragraphs: 2, headings: 1 }); let catalog = doc.get(doc.root().unwrap()); let root = catalog.as_dict().unwrap().reference(b"StructTreeRoot").unwrap(); let tree = doc.get(root); @@ -62,14 +62,18 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { assert_eq!(entries[2].as_int(), Some(1)); assert_eq!(entries[1].as_array().unwrap().len(), 2); assert_eq!(entries[3].as_array().unwrap().len(), 1); - for page in pdfcraft_model::pages(&doc) { + // The 18 pt line stands out from the 12 pt body: a heading, the rest paragraphs. + let first_marks = ["/H1 <> BDC", "/P <> BDC"]; + for (page, first) in pdfcraft_model::pages(&doc).iter().zip(first_marks) { let content = page.dict.get(b"Contents").unwrap(); let stream = doc.resolve(content); let Object::Stream(stream) = &*stream else { panic!("marked page content") }; let text = String::from_utf8(stream.decoded_strict().unwrap()).unwrap(); - assert!(text.contains("/P <> BDC"), "{text}"); + assert!(text.contains(first), "{text}"); assert!(text.contains("EMC"), "{text}"); } + let kinds: Vec = tags(&doc).iter().map(|t| t.kind.clone()).collect(); + assert_eq!(kinds, ["Document", "H1", "P", "P"]); let report = check(&doc, &Options::default()); assert_eq!(status(&report, Rule::TaggedPdf), Status::Passed); assert_eq!(status(&report, Rule::TaggedContent), Status::Passed); @@ -104,7 +108,7 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { let mut predicted = pdfcraft_cos::Stream::flate(Dict::new(), &rows); predicted.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); doc.set(ObjRef::new(4, 0), predicted); - assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1 }); + assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1, headings: 0 }); let page = &pdfcraft_model::pages(&doc)[0]; let written = doc.resolve(page.dict.get(b"Contents").unwrap()); let Object::Stream(written) = &*written else { panic!("page content") }; @@ -157,7 +161,7 @@ fn autotag_text_marks_vector_graphics_and_images_as_artifacts() { "", ); let result = autotag_text(&mut doc).unwrap(); - assert_eq!(result, AutoTagResult { pages: 1, paragraphs: 1 }); + assert_eq!(result, AutoTagResult { pages: 1, paragraphs: 1, headings: 0 }); let report = check(&doc, &Options { rules: [Rule::TaggedPdf, Rule::TaggedContent].into(), pages: None }); assert_eq!(status(&report, Rule::TaggedPdf), Status::Passed); assert_eq!(status(&report, Rule::TaggedContent), Status::Passed); diff --git a/crates/annot/src/modify.rs b/crates/annot/src/modify.rs index ae2222b387..6566337ca1 100644 --- a/crates/annot/src/modify.rs +++ b/crates/annot/src/modify.rs @@ -857,19 +857,38 @@ pub fn set_points(doc: &mut Document, page: usize, index: usize, points: &[[f64; /// Scale a drawing's strokes into `rect`. The margin between the strokes and the old `/Rect` (half /// the line width and a little more) stays the same on each side, so the line isn't clipped. +/// Most stroke points a stored drawing may have for resizing or erasing it. Counted as each +/// stroke is resolved, so one stroke array referenced again and again can't expand past it. +const MAX_STORED_INK_POINTS: usize = 1_000_000; +/// Most points the eraser densifies a drawing into before testing them against its path. +const MAX_ERASE_SAMPLES: usize = 4_000_000; + +/// The strokes of an ink annotation's `/InkList`, at most [`MAX_STORED_INK_POINTS`] points. +fn ink_strokes(doc: &Document, d: &Dict) -> Result>, AnnotError> { + let too_many = || AnnotError::Invalid(format!("the drawing has more than {MAX_STORED_INK_POINTS} points")); + let list = d.get(b"InkList").map(|l| doc.resolve(l)); + let mut points = 0usize; + let mut strokes = Vec::new(); + for s in list.as_deref().and_then(Object::as_array).map(Vec::as_slice).unwrap_or_default() { + let stroke = doc.resolve(s); + let Some(a) = stroke.as_array() else { + strokes.push(Vec::new()); + continue; + }; + // Charged before the coordinates are read: a shared array costs each time it is listed. + points = points.saturating_add(a.len() / 2); + if points > MAX_STORED_INK_POINTS { + return Err(too_many()); + } + let v: Vec = a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); + strokes.push(v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect()); + } + Ok(strokes) +} + fn resize_ink(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta) -> Result<(), AnnotError> { let d = annot_dict(doc, r); - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; let Some(inner) = bounds(strokes.iter().flatten().copied()) else { return Err(AnnotError::Invalid("the drawing has no strokes".into())); }; @@ -1475,17 +1494,7 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] if path.is_empty() || !path.iter().all(|p| finite(p)) || !radius.is_finite() || radius <= 0.0 { return Err(AnnotError::Invalid("invalid eraser path".into())); } - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; // Distance from a point to the eraser's path (segments, or a single point). let near = |p: [f64; 2]| -> bool { if path.len() == 1 { @@ -1501,17 +1510,27 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] }; let mut kept: Vec> = Vec::new(); let mut erased = false; + let mut samples = 0usize; for s in &strokes { // Densify so long segments are cut where the eraser crosses them. let mut pts: Vec<[f64; 2]> = Vec::new(); for w in s.windows(2) { let n = ((w[1][0] - w[0][0]).hypot(w[1][1] - w[0][1]) / (radius / 2.0)).ceil().clamp(1.0, 2000.0) as usize; + samples = samples.saturating_add(n); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } for k in 0..n { let t = k as f64 / n as f64; pts.push([w[0][0] + (w[1][0] - w[0][0]) * t, w[0][1] + (w[1][1] - w[0][1]) * t]); } } if let Some(last) = s.last() { + // The final point is a sample too. + samples = samples.saturating_add(1); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } pts.push(*last); } let mut run: Vec<[f64; 2]> = Vec::new(); @@ -1537,6 +1556,12 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] delete_annotation(doc, page, index)?; return Ok(true); } + // The retained runs are stored as the new `/InkList`: past the cap, writing them would leave a + // drawing that later `resize_ink` and `erase_ink` refuse to read. + let points: usize = kept.iter().map(Vec::len).sum(); + if points > MAX_STORED_INK_POINTS { + return Err(AnnotError::Invalid(format!("erasing would leave a drawing with more than {MAX_STORED_INK_POINTS} points"))); + } let width = d.get(b"BS").and_then(|b| b.as_dict()).and_then(|b| b.get(b"W")).and_then(|w| w.as_f64()).unwrap_or(1.0); let rect = grow(bounds(kept.iter().flatten().copied()).unwrap_or_default(), width / 2.0 + 1.0); doc.update_dict(r, |d| { diff --git a/crates/annot/src/tests.rs b/crates/annot/src/tests.rs index ce5a8c1604..980959d4d9 100644 --- a/crates/annot/src/tests.rs +++ b/crates/annot/src/tests.rs @@ -4797,6 +4797,73 @@ fn freehand_highlight_is_compatible_ink_with_marker_blending() { } } +/// A drawing whose /InkList lists one stroke array over and over is refused once its points +/// pass the cap, counted as each stroke is resolved, before they are collected or scaled. +#[test] +fn ink_points_are_counted_as_strokes_resolve() { + let numbers: String = (0..2000).map(|k| format!(" {}", k % 100)).collect(); + let refs = " 5 0 R".repeat(501); + let doc_with = |refs: &str| { + doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + format!("[{numbers}]"), + ]) + }; + // 501 × 1,000 points is under the cap: the points are read (this bare drawing then has no + // appearance it can be redrawn with, which is another matter). + let mut doc = doc_with(&refs); + let under = set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")); + assert!(!matches!(&under, Err(AnnotError::Invalid(m)) if m.contains("points")), "{under:?}"); + // 1,001 × 1,000 points is over it. + let mut doc = doc_with(&" 5 0 R".repeat(1001)); + match set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 1.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// The eraser's densified samples have a total: a tiny eraser over long strokes is refused +/// instead of interpolating millions of points. +#[test] +fn the_eraser_densifies_within_a_total() { + let zigzag: String = (0..2100).map(|k| format!(" {} {}", if k % 2 == 0 { 0 } else { 5000 }, k)).collect(); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 5000 2100] /InkList [[{zigzag}]] >>"), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 0.01, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// Each stroke's final point is a sample too: 500,000 strokes of two points at 8 samples a segment +/// are 4,000,000 densified samples and as many final points again, past the total. +#[test] +fn the_eraser_counts_each_strokes_final_point() { + let refs = " 5 0 R".repeat(500_000); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + "[0 0 8 0]".into(), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 2.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + /// A 3D annotation is PDF 1.6 and a PRC model 1.7 (ISO 32000-2 §13.6): an older document's /// catalog says so, and a newer one is left as it is. #[test] diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index a825b1fe52..60a5d0fa06 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -28,6 +28,8 @@ default = ["mcp"] mcp = ["dep:base64"] [dev-dependencies] +# Synthetic printer profiles for the app-managed print colour tests. +craft-cms = { workspace = true } pdfcraft-fonts = { workspace = true } png = "0.18" flate2.workspace = true diff --git a/crates/automation/src/a11y.rs b/crates/automation/src/a11y.rs index b17192aab0..cab0fb9a61 100644 --- a/crates/automation/src/a11y.rs +++ b/crates/automation/src/a11y.rs @@ -217,7 +217,7 @@ impl Automation { let tags = self.accessibility_tags(a)?; let out = json!({ "tags": tags["tags"], "count": tags["count"], - "note": "Text objects were tagged as paragraphs. Review heading roles, reading order and remaining accessibility findings manually.", + "note": "Text objects were tagged as headings (H1-H6, by size and weight against the body text) or paragraphs. Review heading roles, reading order and remaining accessibility findings manually.", }); Ok(with_warnings(out, warnings)) } diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index 617b3d0f8b..65731d1fb3 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -726,6 +726,7 @@ impl Automation { "redact_clear" => self.redact_clear(&a)?, "doc_hidden_info" => self.doc_hidden_info(&a)?, "printers" => self.printers()?, + "print_color_profiles" => self.print_color_profiles()?, "printer_options" => self.printer_options(&a)?, "printer_papers" => self.printer_papers(&a)?, "link_list" => self.link_list(&a)?, diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 7fa63d1a1d..5c49b25b23 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -30,6 +30,7 @@ pub mod mcp; mod measure; mod model3d; mod model_render; +mod print_colour; mod printing; mod reading; mod redact; diff --git a/crates/automation/src/print_colour.rs b/crates/automation/src/print_colour.rs new file mode 100644 index 0000000000..22279fbd8b --- /dev/null +++ b/crates/automation/src/print_colour.rs @@ -0,0 +1,90 @@ +//! App-managed print colour for `doc_print` (`color_management` app) and the installed printer +//! profiles it can name (`print_color_profiles`). + +use pdfcraft_engine::print::colour::{self, Intent, ManagedColour, PrinterProfile}; +use serde_json::{Value, json}; + +use crate::{Args, Automation, Result, ToolError, failed}; + +/// An invalid-arguments error saying `m`. +fn bad(m: impl Into) -> ToolError { + ToolError::InvalidArgs(m.into()) +} + +impl Automation { + /// `doc_print`'s colour handling: `None` for `printer` (the default), else the profile, intent + /// and black point compensation the pictures are converted with. + pub(crate) fn managed_colour(&self, a: &Args) -> Result> { + let app = match a.opt_str("color_management")?.unwrap_or("printer") { + "printer" => false, + "app" => true, + m => return Err(bad(format!("unknown color_management {m:?}: printer (the printer manages colour) or app"))), + }; + let asked = ["color_profile", "rendering_intent", "black_point_compensation"].into_iter().find(|k| a.get(k).is_some()); + if !app { + return match asked { + Some(k) => Err(bad(format!("{k} needs color_management app"))), + None => Ok(None), + }; + } + let name = a + .opt_str("color_profile")? + .ok_or_else(|| bad("color_management app needs color_profile: an ICC file or a name from print_color_profiles"))?; + let profile = self.printer_profile(name)?; + let intent = match a.opt_str("rendering_intent")? { + None => Intent::RelativeColorimetric, + Some(i) => { + Intent::parse(i).ok_or_else(|| bad(format!("unknown rendering_intent {i:?}: perceptual, relative, saturation or absolute")))? + } + }; + let black_point = a.opt_bool("black_point_compensation")?.unwrap_or(true); + Ok(Some(ManagedColour { profile, intent, black_point })) + } + + /// The profile `name` names: an ICC file, else an installed profile's name or file name. A + /// file is tried first so naming one doesn't read every installed profile. + fn printer_profile(&self, name: &str) -> Result { + let file = self.resolve_for(name, crate::file_access::Access::Read, true); + if let Ok(path) = &file + && path.is_file() + { + return PrinterProfile::read(path).map_err(bad); + } + let installed = match self.installed_profiles() { + Ok(list) => list, + // Confined without read authority: say why the file was refused, if it was. + Err(e) => return Err(file.err().unwrap_or(e)), + }; + if let Some(p) = colour::find_installed(&installed, name) { + return PrinterProfile::read(&p.path).map_err(failed); + } + file?; + Err(bad(format!("color_profile {name:?} is neither an installed profile (print_color_profiles) nor an ICC file"))) + } + + /// The profiles that can be named: the system's, or, when file access is confined, those in + /// the read root or `--root` (as `text_font_files` does), each file checked against it, links + /// followed, before it is read. + fn installed_profiles(&self) -> Result> { + let Some(roots) = &self.roots else { + let Some(root) = &self.root else { return Ok(colour::installed_profiles()) }; + let inside = |path: &std::path::Path| self.resolve(&path.to_string_lossy(), false).is_ok(); + return Ok(colour::installed_profiles_where(std::slice::from_ref(root), &inside)); + }; + let read = roots + .read_root() + .ok_or_else(|| failed("automation filesystem access is not granted: read authority is absent: installed colour profiles"))?; + let folders = vec![self.resolve(&read.to_string_lossy(), false)?]; + let inside = |path: &std::path::Path| roots.check(&path.to_string_lossy(), crate::file_access::Access::Read).is_ok(); + Ok(colour::installed_profiles_where(&folders, &inside)) + } + + pub(crate) fn print_color_profiles(&self) -> Result { + let list: Vec = self + .installed_profiles()? + .into_iter() + .map(|p| json!({ "name": p.name, "path": p.path.to_string_lossy(), "color_space": if p.gray { "gray" } else { "rgb" } })) + .collect(); + Ok(json!({ "count": list.len(), "profiles": list })) + } +} diff --git a/crates/automation/src/printing.rs b/crates/automation/src/printing.rs index 3d7a76865a..d099d4dc01 100644 --- a/crates/automation/src/printing.rs +++ b/crates/automation/src/printing.rs @@ -118,7 +118,7 @@ fn spool_job( title: String, duplex: Option, media: Option<&str>, - raster: Option, + raster: Option<&print::Raster>, sheets: &[print::Sheet], ) -> Result { let options = match a.get("options") { @@ -338,12 +338,19 @@ impl Automation { let (paper, media) = resolve_paper(a.opt_str("paper")?, resolved_printer.as_deref())?; // Print as image (Acrobat's Advanced Print Setup): pictures at `dpi`, grey when // `grayscale` is also asked for. - let raster = if a.opt_bool("as_image")?.unwrap_or(false) { + // App-managed colour converts those pictures into the printer's profile, so it prints + // as image whether or not as_image is given. + let colour = self.managed_colour(a)?; + let as_image = match a.opt_bool("as_image")? { + Some(false) if colour.is_some() => return Err(bad("color_management app prints the pages as images: leave out as_image false")), + asked => asked.unwrap_or(false) || colour.is_some(), + }; + let raster = if as_image { let dpi = a.opt_num("dpi")?.unwrap_or(300.0); if !(72.0..=600.0).contains(&dpi) { return Err(bad("dpi must be between 72 and 600")); } - Some(print::Raster { dpi, grayscale: a.opt_bool("grayscale")?.unwrap_or(false) }) + Some(print::Raster { dpi, grayscale: a.opt_bool("grayscale")?.unwrap_or(false), colour }) } else { None }; @@ -367,7 +374,15 @@ impl Automation { (Some(p), None) => Destination::File(self.resolve(p, true)?), (None, Some(printer)) => { let actual_printer = resolved_printer.as_deref().unwrap_or(printer); - Destination::Printer(spool_job(a, actual_printer, self.doc(a)?.name.clone(), duplex, media.as_deref(), settings.raster, &laid_out)?) + Destination::Printer(spool_job( + a, + actual_printer, + self.doc(a)?.name.clone(), + duplex, + media.as_deref(), + settings.raster.as_ref(), + &laid_out, + )?) } _ => return Err(bad("pass either path (save the print-ready PDF) or printer (a name from printers, or \"default\")")), }; @@ -379,6 +394,10 @@ impl Automation { }; let bytes = self.session.print_pdf(id, &settings).map_err(failed)?; let mut out = json!({ "sheets": sheets, "pages": settings.pages.len(), "bytes": bytes.len() }); + if let Some(c) = settings.raster.as_ref().and_then(|r| r.colour.as_ref()) { + out["color_management"] = + json!({ "profile": c.profile.name(), "rendering_intent": c.intent.id(), "black_point_compensation": c.black_point }); + } if !scripts.is_empty() { out["will_print"] = crate::a11y::form_tools::js_output_json(&scripts); } @@ -542,10 +561,10 @@ mod tests { /// grayscale, which Microsoft Print to PDF keeps no setting for and the job was refused. #[test] fn grey_pictures_dont_ask_the_driver_for_grayscale() { - let grey = print::Raster { dpi: 300.0, grayscale: true }; - let colour = print::Raster { dpi: 300.0, grayscale: false }; + let grey = print::Raster { dpi: 300.0, grayscale: true, colour: None }; + let colour = print::Raster { dpi: 300.0, grayscale: false, colour: None }; let job = |args: Value, raster: Option| { - spool_job(&Args(&args), "Office", String::new(), None, Some("Letter"), raster, &[]).unwrap().grayscale + spool_job(&Args(&args), "Office", String::new(), None, Some("Letter"), raster.as_ref(), &[]).unwrap().grayscale }; assert!(job(json!({ "grayscale": true }), None), "vector pages: the driver makes them grey"); assert!(!job(json!({ "grayscale": true }), Some(grey)), "grey pictures: nothing for the driver to do"); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 2a7fceb6f0..78393385e5 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -726,7 +726,7 @@ pub fn tools() -> Vec { t( "doc_hidden_info", "Find hidden information", - "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts.", + "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts. A page that can't be read in full (its content streams don't decode, or they or a form they draw hold bytes the parser can't read or an inline image whose end is a guess) counts as one item of hidden-text and of hidden-layers: removing either category then fails on that page. With form-fields removed too, a field's appearance becomes page content and is read the same way, though it isn't counted here.", ) .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), @@ -751,6 +751,9 @@ pub fn tools() -> Vec { .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t("printers", "List printers", "The printers the system's print spooler knows (CUPS on macOS and Linux), with the default marked.").ro().with(schema(json!({}), &[])), + t("print_color_profiles", "List printer colour profiles", "The RGB and grey ICC profiles installed on this computer that doc_print can convert the pages into (color_management app): name, path and color_space. Windows: the system colour folder; macOS: the ColorSync folders; Linux: the shared and per-user icc folders. CMYK profiles are left out: the printed pictures are RGB.") + .ro() + .with(schema(json!({}), &[])), t("printer_options", "List printer options", "A printer driver's own job options (CUPS: from its PPD), such as the paper tray, paper type or finishing: key, label, group, default and choices. Pass chosen values to doc_print as options. Empty on Windows, where the driver's preferences window holds them.") .ro() .with(schema(json!({ "printer": { "type": "string" } }), &["printer"])), @@ -764,7 +767,7 @@ pub fn tools() -> Vec { t( "doc_print", "Print", - "Print with Acrobat's Print dialog options, or save the print-ready PDF. pages: a range such as \"1-3, 6, 9-\" (page labels allowed, \"(n)\" for the nth page whatever its label, unless a page is labelled exactly \"(n)\", since labels are looked up first; default all); subset odd/even (by page number, as Acrobat counts them: pages 2, 7-10 with even prints 2, 8 and 10, not every second page of the range); reverse. layout: fit (default), actual, shrink, custom (scale %), multiple (per_sheet 2/4/6/9/16, which turns the sheet to suit the grid, or a custom grid of cols across by rows down, each 1-99, on the sheet as oriented (auto prints portrait); order, border, auto_rotate; cut-stack order arranges single-sided sheets for cutting into piles and stacking left to right, top to bottom, keeping sheet order within each pile; duplex must be off), booklet (booklet_subset both/front/back, binding left/right; a booklet sets two-sided itself, as Acrobat does: both flips on the short edge, front or back prints one-sided), poster (scale %, overlap pt, cut_marks). orientation auto/portrait/landscape; comments_forms document / document-and-markups (default) / document-and-stamps / form-fields-only; paper Letter/Legal/Tabloid, ARCH A/B/C/D/E1/E, ANSI C/D/E, A0/A1/A2/A3/A4/A5/A6, US #10 Envelope, a driver paper name from printer_papers, or dimensions \"x\"; when printer is given and paper is omitted, uses the printer's default paper if available; use layout actual for 100% in the print-ready PDF; printer drivers may still substitute paper or scale. Then path (save) or printer (a name or \"default\") with copies, collate, duplex off/long-edge/short-edge, grayscale, and options (driver options from printer_options, CUPS).", + "Print with Acrobat's Print dialog options, or save the print-ready PDF. pages: a range such as \"1-3, 6, 9-\" (page labels allowed, \"(n)\" for the nth page whatever its label, unless a page is labelled exactly \"(n)\", since labels are looked up first; default all); subset odd/even (by page number, as Acrobat counts them: pages 2, 7-10 with even prints 2, 8 and 10, not every second page of the range); reverse. layout: fit (default), actual, shrink, custom (scale %), multiple (per_sheet 2/4/6/9/16, which turns the sheet to suit the grid, or a custom grid of cols across by rows down, each 1-99, on the sheet as oriented (auto prints portrait); order, border, auto_rotate; cut-stack order arranges single-sided sheets for cutting into piles and stacking left to right, top to bottom, keeping sheet order within each pile; duplex must be off), booklet (booklet_subset both/front/back, binding left/right; a booklet sets two-sided itself, as Acrobat does: both flips on the short edge, front or back prints one-sided), poster (scale %, overlap pt, cut_marks). orientation auto/portrait/landscape; comments_forms document / document-and-markups (default) / document-and-stamps / form-fields-only; paper Letter/Legal/Tabloid, ARCH A/B/C/D/E1/E, ANSI C/D/E, A0/A1/A2/A3/A4/A5/A6, US #10 Envelope, a driver paper name from printer_papers, or dimensions \"x\"; when printer is given and paper is omitted, uses the printer's default paper if available; use layout actual for 100% in the print-ready PDF; printer drivers may still substitute paper or scale. Then path (save) or printer (a name or \"default\") with copies, collate, duplex off/long-edge/short-edge, grayscale, and options (driver options from printer_options, CUPS). color_management app (Acrobat's Color Handling, the app managing colours) prints the pages as images converted into color_profile with rendering_intent and black_point_compensation; turn the driver's own colour management off for that job, or the printer converts them again.", ) .with(schema( json!({ @@ -798,6 +801,10 @@ pub fn tools() -> Vec { "as_image": { "type": "boolean", "description": "Print as image (Acrobat's Advanced Print Setup): every page rendered to a picture at dpi before it is placed on the paper, for printers that skip text, shading or transparency; the print is no longer searchable text, and with grayscale the pictures are grey, so the printer isn't asked for grayscale (a driver without that setting still prints them)." }, "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution for as_image (default 300)." }, "options": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Printer driver options from printer_options: key → choice value." }, + "color_management": { "type": "string", "enum": ["printer", "app"], "description": "Who converts the colours: printer (default, the driver does) or app, which prints the pages as images (dpi, grayscale apply) converted from sRGB into color_profile." }, + "color_profile": { "type": "string", "description": "For color_management app: a name or file name from print_color_profiles, or the path of an RGB or grey ICC printer profile." }, + "rendering_intent": { "type": "string", "enum": ["perceptual", "relative", "saturation", "absolute"], "description": "For color_management app: how colours the printer can't reach are mapped (default relative colorimetric)." }, + "black_point_compensation": { "type": "boolean", "description": "For color_management app: map sRGB black onto the printer's darkest black, so shadows stay apart instead of clipping (default true; no effect with absolute)." }, }), &["doc"], )), @@ -1764,7 +1771,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; a picture's stencil /Mask and one-bit /SMask are stored again the same way, at their size, not resampled and sharing no dictionary, counted as images; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 54e10a035e..5251bf96ce 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -509,6 +509,25 @@ fn autotag_apply_writes_reviewable_tags_and_survives_save_reopen() { assert!(pages.iter().all(|page| page.dict.contains(b"StructParents"))); } +/// #7062: Apply tags short text larger than the body as headings, ranked across the pages, and +/// the body text as paragraphs. +#[test] +fn autotag_apply_tags_headings_by_size_across_pages() { + let dir = workdir("autotag-headings"); + let body = "BT /FR 11 Tf 72 670 Td (Body text that runs long enough to be the most common style) Tj ET"; + let page1 = format!("BT /FB 18 Tf 72 700 Td (Introduction) Tj ET {body} BT /FB 14 Tf 72 640 Td (Background) Tj ET"); + let page2 = format!("BT /FB 18 Tf 72 700 Td (Results) Tj ET {body}"); + std::fs::write(dir.join("headed.pdf"), bold_and_regular_pdf(&page1, &page2)).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "headed.pdf" }))["doc"].as_u64().unwrap(); + let applied = ok(&mut a, "accessibility_autotag_apply", json!({ "doc": doc })); + let types: Vec<&str> = applied["tags"].as_array().unwrap().iter().map(|t| t["type"].as_str().unwrap()).collect(); + assert_eq!(types, ["Document", "H1", "P", "H2", "H1", "P"]); + let check = ok(&mut a, "accessibility_check", json!({ "doc": doc, "rules": ["heading-nesting"] })); + let nesting = check["results"].as_array().unwrap().iter().find(|r| r["rule"] == "heading-nesting").unwrap(); + assert_eq!(nesting["status"], "passed", "{check}"); +} + #[test] fn autotag_form_fields_writes_reviewable_widget_tags() { let dir = workdir("autotag-form-fields"); @@ -531,6 +550,12 @@ fn styled_headings_pdf() -> Vec { let page1 = "BT /FB 18 Tf 72 700 Td (Introduction) Tj ET BT /FR 11 Tf 72 670 Td (Some text) Tj ET \ BT /FB 14 Tf 72 640 Td (Background) Tj ET"; let page2 = "BT /FB 18 Tf 72 700 Td (Results) Tj ET BT /FR 11 Tf 72 670 Td (More text) Tj ET"; + bold_and_regular_pdf(page1, page2) +} + +/// A two-page PDF whose pages draw `page1` and `page2` with `/FB` (Helvetica-Bold) and `/FR` +/// (Helvetica). +fn bold_and_regular_pdf(page1: &str, page2: &str) -> Vec { let objs: Vec = vec![ "<< /Type /Catalog /Pages 2 0 R >>".into(), "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 612 792] /Resources << /Font << /FB 7 0 R /FR 8 0 R >> >> >>".into(), @@ -8986,6 +9011,61 @@ fn doc_optimize_stores_black_and_white_images_as_ccitt_group_4() { } } +/// A mixed-raster page: a flat colour layer drawn through a one-bit soft mask, [`scanned_page`]'s +/// scan, which the page no longer draws by name. `doc_optimize` stores the mask as Group 4, at its +/// size, still the picture's soft mask, and counts it as an image; with `mono: retain` it stays. +#[test] +fn doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4() { + use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; + let dir = workdir("optimize-mono-mask"); + let mut doc = Document::open(std::sync::Arc::new(scanned_page())).unwrap(); + let find = |doc: &Document, kind: &[u8], key: &[u8]| { + doc.object_numbers().into_iter().map(|num| ObjRef { num, generation: doc.generation(num) }).find(|r| match &*doc.get(*r) { + Object::Stream(s) => s.dict.name(key) == Some(kind), + Object::Dict(d) => d.name(key) == Some(kind), + _ => false, + }) + }; + let scan = find(&doc, b"Image", b"Subtype").unwrap(); + let page = find(&doc, b"Page", b"Type").unwrap(); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("XObject")); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(250)); + d.set(b"Height".to_vec(), Object::Int(100)); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"SMask".to_vec(), Object::Ref(scan)); + let picture = doc.add(Object::Stream(Stream::flate(d, &[0, 102, 204].repeat(250 * 100)))); + doc.update_dict(page, |p| { + let mut x = Dict::new(); + x.set(b"Scan".to_vec(), Object::Ref(picture)); + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + p.set(b"Resources".to_vec(), Object::Dict(res)); + }) + .unwrap(); + std::fs::write(dir.join("masked.pdf"), write_full(&doc, &SaveOptions::default()).unwrap()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "masked.pdf" }))["doc"].as_u64().unwrap(); + let kept = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "kept.pdf", "mono": { "compression": "retain" } })); + assert_eq!(kept["images_recompressed"].as_u64(), Some(0), "{kept}"); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "small.pdf" })); + assert_eq!((r["images"].as_u64(), r["images_recompressed"].as_u64()), (Some(2), Some(1)), "the picture and its mask: {r}"); + // In the copy written, the picture's soft mask is Group 4, at its size. + let small = Document::open(std::sync::Arc::new(std::fs::read(dir.join("small.pdf")).unwrap())).unwrap(); + let picture = small + .object_numbers() + .into_iter() + .map(|num| small.get(ObjRef { num, generation: small.generation(num) })) + .find(|o| matches!(&**o, Object::Stream(s) if s.dict.contains(b"SMask"))) + .unwrap(); + let Object::Stream(picture) = &*picture else { unreachable!() }; + let Object::Stream(mask) = &*small.get(picture.dict.reference(b"SMask").unwrap()) else { panic!("the soft mask is a stream") }; + assert_eq!(mask.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..])); + assert_eq!((mask.dict.int(b"Width"), mask.dict.int(b"Height")), (Some(1000), Some(400))); +} + /// A scan drawn at 600 ppi is averaged down to 300 ppi by default. `mono: { downsample: false }` /// keeps its size, `ppi` and `above_ppi` choose others, and values of the wrong type are refused. #[test] @@ -12430,6 +12510,81 @@ fn doc_print_as_image() { } } +/// #6269: color_management app prints converted pictures and says how; the profile must be an +/// RGB or grey one, and the colour arguments need app. +#[test] +fn doc_print_with_app_managed_colour() { + let dir = workdir("print-colour"); + std::fs::write(dir.join("wide.icc"), craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("press.icc"), craft_cms::Builtin::CoatedCmyk.profile().to_bytes().as_slice()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let args = |extra: Value| { + let mut v = json!({ "doc": doc, "pages": "1", "dpi": 72, "path": "managed.pdf" }); + v.as_object_mut().unwrap().extend(extra.as_object().unwrap().clone()); + v + }; + let r = ok(&mut a, "doc_print", args(json!({ "color_management": "app", "color_profile": "wide.icc", "rendering_intent": "perceptual" }))); + assert_eq!(r["color_management"]["rendering_intent"], "perceptual", "{r}"); + assert_eq!(r["color_management"]["black_point_compensation"], true, "{r}"); + let out = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join("managed.pdf")).unwrap())).unwrap(); + let pictures = + out.scan_objects().filter(|(_, o)| matches!(&**o, pdfcraft_cos::Object::Stream(st) if st.dict.name(b"Subtype") == Some(b"Image"))).count(); + assert_eq!(pictures, 1, "app-managed colour prints the page as a picture"); + assert!(ok(&mut a, "doc_print", args(json!({})))["color_management"].is_null(), "the printer manages colour by default"); + for (extra, says) in [ + (json!({ "color_management": "app", "color_profile": "press.icc" }), "RGB or grey"), + (json!({ "color_management": "app" }), "color_profile"), + (json!({ "color_management": "app", "color_profile": "missing.icc" }), "print_color_profiles"), + (json!({ "color_profile": "wide.icc" }), "color_management app"), + (json!({ "color_management": "app", "color_profile": "wide.icc", "rendering_intent": "vivid" }), "rendering_intent"), + (json!({ "color_management": "app", "color_profile": "wide.icc", "as_image": false }), "as_image"), + (json!({ "color_management": "driver" }), "color_management"), + ] { + match a.call("doc_print", &args(extra.clone())) { + Err(ToolError::InvalidArgs(m)) => assert!(m.contains(says), "{extra}: {m}"), + other => panic!("{extra}: {other:?}"), + } + } + let listed = ok(&mut a, "print_color_profiles", json!({})); + assert_eq!(listed["count"].as_u64(), listed["profiles"].as_array().map(|p| p.len() as u64), "{listed}"); + // Under a root the profiles listed are the root's RGB and grey ones. + assert_eq!(listed["count"], 1, "{listed}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Under `--root`, a profile link inside the root that leads out of it is neither listed nor +/// found by name. +#[test] +fn colour_profile_links_out_of_the_root_are_not_read() { + let dir = workdir("print-colour-link"); + let outside = dir.with_extension("outside"); + let _ = std::fs::remove_dir_all(&outside); + std::fs::create_dir_all(&outside).unwrap(); + let profile = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + std::fs::write(dir.join("wide.icc"), profile.as_slice()).unwrap(); + std::fs::write(outside.join("other.icc"), profile.as_slice()).unwrap(); + #[cfg(unix)] + let linked = std::os::unix::fs::symlink(outside.join("other.icc"), dir.join("leak.icc")).is_ok(); + #[cfg(windows)] + let linked = std::os::windows::fs::symlink_file(outside.join("other.icc"), dir.join("leak.icc")).is_ok(); + #[cfg(not(any(unix, windows)))] + let linked = false; + // Windows needs Developer Mode or elevation to make links. + if !linked { + eprintln!("skipped: symbolic links can't be made here"); + return; + } + let mut a = auto(&dir); + let listed = ok(&mut a, "print_color_profiles", json!({})); + assert_eq!(listed["count"], 1, "only the profile inside the root: {listed}"); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let print = json!({ "doc": doc, "pages": "1", "dpi": 72, "path": "m.pdf", "color_management": "app", "color_profile": "leak" }); + assert!(a.call("doc_print", &print).is_err(), "a link out of the root isn't found by name"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); +} + /// Without a root, `write_output` still writes through a staging file (#2166): the folder is /// made, nothing is left beside the file, and a folder at the path is refused. #[test] diff --git a/crates/automation/tests/it/automation_roots.rs b/crates/automation/tests/it/automation_roots.rs index e65c582a73..ecc65c742c 100644 --- a/crates/automation/tests/it/automation_roots.rs +++ b/crates/automation/tests/it/automation_roots.rs @@ -200,6 +200,30 @@ fn tools_that_read_a_path_are_held_to_the_read_root() { ok(&mut server, "text_font_files", json!({"directories": [s(&base.join("in"))]})); } +/// The installed colour profiles are outside the read root: confined, `print_color_profiles` +/// lists and `doc_print` names only the profiles inside it, and without read authority none. +#[test] +fn colour_profiles_are_looked_for_inside_the_read_root() { + let (mut server, base) = confined("profiles", true, true); + let profile = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + std::fs::write(base.join("in/wide.icc"), profile.as_slice()).unwrap(); + std::fs::write(base.join("outside/other.icc"), profile.as_slice()).unwrap(); + let listed = ok(&mut server, "print_color_profiles", json!({})); + let paths: Vec<&str> = listed["profiles"].as_array().unwrap().iter().filter_map(|p| p["path"].as_str()).collect(); + assert_eq!(paths.len(), 1, "{listed}"); + assert!(paths[0].ends_with("wide.icc"), "{listed}"); + ok(&mut server, "doc_open", json!({"path": s(&base.join("in/a.pdf"))})); + let print = + |name: &str| json!({"doc": 1, "pages": "1", "dpi": 72, "path": s(&base.join("out/m.pdf")), "color_management": "app", "color_profile": name}); + ok(&mut server, "doc_print", print("wide")); + let (error, text) = call(&mut server, "doc_print", print("other")); + assert!(error, "a profile outside the read root isn't found by name: {text}"); + refused(&mut server, "doc_print", print(&s(&base.join("outside/other.icc"))), "outside the read root"); + + let (mut unread, _) = confined("profiles-unread", false, true); + refused(&mut unread, "print_color_profiles", json!({}), "read authority is absent"); +} + #[test] fn tools_that_write_a_path_are_held_to_the_write_root() { let (mut server, base) = confined("writes", true, true); diff --git a/crates/automation/tests/it/model_render.rs b/crates/automation/tests/it/model_render.rs index c7dcca1af6..4323ca50e7 100644 --- a/crates/automation/tests/it/model_render.rs +++ b/crates/automation/tests/it/model_render.rs @@ -9,6 +9,11 @@ use serde_json::{Value, json}; /// file structure with no product tree, in hexadecimal for the stream's `/ASCIIHexDecode`. const TRIANGLE_PRC_HEX: &str = "505243c91f0000c91f00000100000002000000030000000400000000000000000000000000000000000000010000000500000006000000070000000800000000000000060000006b0000009a000000a3000000ab000000cd000000d5000000dd000000e500000000000000505243c91f0000c91f0000050000000600000007000000080000000000000000000000000000000000000000000000789c63000000010001789c030000000001789c9b71c087758da2422b43084fe3140e038713464cd759381a162800006ed70828789c030000000001789c030000000001789c030000000001"; +/// The same triangle stored as a `TESS_3D_Compressed` whose normals are recalculated, so it is +/// drawn only once its triangles are rebuilt. `prc-render`'s tests build it +/// (`the_automation_fixture_is_the_compressed_triangle` keeps this copy their output). +const COMPRESSED_TRIANGLE_PRC_HEX: &str = "505243c91f0000c91f00000100000002000000030000000400000000000000000000000000000000000000010000000500000006000000070000000800000000000000060000006b0000009a000000a3000000ab000000d2000000da000000e2000000ea00000000000000505243c91f0000c91f0000050000000600000007000000080000000000000000000000000000000000000000000000789c63000000010001789c030000000001789c9b71c08775aded1d0624d0d2c808055a8d0c8c8ccc0c0c0c0a0a223ce2e50c2c00bed0061f789c030000000001789c030000000001789c030000000001"; + /// A one-page PDF whose page holds a 3D annotation over a stream of `/Subtype /{subtype}`. fn pdf(subtype: &str, hex: &str) -> Vec { let stream = format!("<< /Type /3D /Subtype /{subtype} /Filter /ASCIIHexDecode /Length {} >>\nstream\n{hex}>\nendstream", hex.len() + 1); @@ -82,6 +87,24 @@ fn a_prc_model_is_drawn_over_the_background() { assert!(a.call("model_render", &json!({ "doc": doc, "page": 1, "model": 1 })).is_ok()); } +#[test] +fn a_compressed_prc_model_is_rebuilt_and_drawn() { + let (dir, mut a) = setup("prc-compressed"); + std::fs::write(dir.join("in/compressed.pdf"), pdf("PRC", COMPRESSED_TRIANGLE_PRC_HEX)).unwrap(); + let doc = open(&mut a, "compressed.pdf"); + let args = json!({ "doc": doc, "page": 1, "width": 64, "height": 48, "direction": [0, 0, -1], "perspective": false }); + let out = a.call("model_render", &args).unwrap(); + let Content::Png { data, .. } = &out[0] else { panic!("expected a PNG") }; + assert_eq!(rgb(data, 0, 0), [255, 255, 255], "the corner is background"); + assert_eq!(rgb(data, 32, 24), [190, 192, 200], "the centre is the triangle, lit head on"); + // Its apex is up: the drawn span narrows from the base to the top, and the centre column + // is covered over many rows. + let (top, bottom) = rows(data, 32); + assert!(bottom - top > 20, "rows {top}..{bottom}"); + let (near_top, near_bottom) = (extent(data, top + 4), extent(data, bottom - 2)); + assert!(near_bottom.1 - near_bottom.0 > 2 * (near_top.1 - near_top.0), "{near_top:?} {near_bottom:?}"); +} + #[test] fn a_u3d_model_or_a_missing_one_is_a_clear_error() { let (_dir, mut a) = setup("u3d"); diff --git a/crates/content/src/lib.rs b/crates/content/src/lib.rs index fb17a903d1..f77c54e1dc 100644 --- a/crates/content/src/lib.rs +++ b/crates/content/src/lib.rs @@ -58,6 +58,23 @@ impl Op { pub fn last_name(&self) -> Option<&[u8]> { self.operands.last().and_then(Object::as_name) } + + /// Operand `k` counted from the end (0 is the last), as the renderer reads an operator's + /// operands: `(x) /OC /MC0 BDC` marks with `/OC /MC0`, and `(x) /F1 10 Tf` selects `/F1`. + pub fn back(&self, k: usize) -> Option<&Object> { + let i = self.operands.len().checked_sub(k)?.checked_sub(1)?; + self.operands.get(i) + } + + /// [`Op::back`] as a name. + pub fn name_back(&self, k: usize) -> Option<&[u8]> { + self.back(k).and_then(Object::as_name) + } + + /// [`Op::back`] as a number. + pub fn num_back(&self, k: usize) -> Option { + self.back(k).and_then(Object::as_f64) + } } /// A parsed stream. diff --git a/crates/content/src/tests.rs b/crates/content/src/tests.rs index 9ca12c4536..89f258853d 100644 --- a/crates/content/src/tests.rs +++ b/crates/content/src/tests.rs @@ -10,6 +10,17 @@ thread_local! { pub(crate) static PIECE_VISITS: std::cell::Cell = const { std::cell::Cell::new(0) }; } +#[test] +fn operands_are_counted_from_the_end() { + let op = parse(b"(x) /OC /MC0 BDC").ops.remove(0); + assert_eq!((op.name_back(1), op.name_back(0)), (Some(&b"OC"[..]), Some(&b"MC0"[..]))); + assert_eq!(op.back(2).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"x".to_vec())); + assert_eq!((op.back(3), op.back(usize::MAX)), (None, None)); + let op = parse(b"/F1 10 Tf").ops.remove(0); + assert_eq!((op.name_back(1), op.num_back(0), op.num_back(1)), (Some(&b"F1"[..]), Some(10.0), None)); + assert_eq!(Op::new("Q", vec![]).back(0), None); +} + #[test] fn splicing_visits_each_piece_a_few_times() { let n = 5_000; diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 7d64596bbb..b956cb7079 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -14042,6 +14042,39 @@ fn review_inside_scope_group_move_matches_direct_transform_under_outer_scale() { } } +/// A page that names one font under 300 resource names builds its metrics once: each alias +/// used to build (and keep) a set of its own. +#[test] +fn font_aliases_share_one_set_of_metrics() { + let fonts: String = (0..300).map(|i| format!(" /A{i} 5 0 R")).collect(); + let content: String = format!("BT 72 700 Td {} ET", (0..300).map(|i| format!("/A{i} 12 Tf (x) Tj ")).collect::()); + let objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font <<{fonts} >> >> >>"), + format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + text::METRICS_BUILT.with(|n| n.set(0)); + let lines = text::text_lines(&doc, 0).unwrap(); + assert!(lines.iter().map(|l| l.text.matches('x').count()).sum::() >= 300, "{lines:?}"); + let built = text::METRICS_BUILT.with(std::cell::Cell::get); + assert!(built < 10, "metrics built {built} times for one font"); +} + /// Each `Tj`/`TJ` moves the text matrix by the width of its text. A run shown later in the same /// text object without a `Td` or `Tm` of its own (a raised run in another colour, a column after /// a `TJ` gap) is placed by those advances. Rewriting the paragraph before it must keep them, as @@ -14850,6 +14883,38 @@ fn a_thai_edit_that_spends_its_allowance_to_the_byte_goes_through() { assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); } +/// Only a read the allowance refuses leaves a font of the face unread for want of it. With the +/// allowance spent to the byte as above, another font of the face, whose ToUnicode is an empty +/// stream behind a filter nothing decodes (unread with any allowance), still has its program read, +/// which the allowance refuses: the edit is refused, rather than going through without that font. +#[test] +fn a_font_of_the_face_unread_on_its_own_once_the_allowance_is_spent_still_refuses_the_edit() { + let mut program = legacy_thai_program(); + program.resize((16 << 20) - 192, 0); + let mut unicode = LEGACY_THAI_UNICODE.to_vec(); + unicode.resize(256, b' '); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &program, &unicode); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + // Not a subset, so its metrics read no program, only the ToUnicode. + let mut other = legacy_thai_font(&mut doc, "THSarabunPSK", &legacy_thai_program(), LEGACY_THAI_UNICODE); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + other.set(b"ToUnicode".to_vec(), Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))))); + let other = doc.add(Object::Dict(other)); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |page| { + let Some(Object::Dict(mut resources)) = page.get(b"Resources").cloned() else { panic!("fixture") }; + let Some(Object::Dict(mut fonts)) = resources.get(b"Font").cloned() else { panic!("fixture") }; + fonts.set(b"F2".to_vec(), Object::Ref(other)); + resources.set(b"Font".to_vec(), Object::Dict(fonts)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + }) + .unwrap(); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let error = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).unwrap_err().to_string(); + assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); +} + /// A THSarabunPSK-named subset that has OpenType Thai mark positioning is a modern face: it is /// shaped with its own layout like any source font, not by the legacy shaper (which would ask /// for Windows PUA alternates such a face doesn't need). diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index b58c41303f..eba8834c2b 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -54,6 +54,8 @@ pub(crate) use fallback::{SOURCE_SPACE_CALLS, take_source_character_measurements #[cfg(test)] use fallback::{add_type3_font, is_type3_fallback}; pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; +#[cfg(test)] +pub(crate) use interp::METRICS_BUILT; use interp::{Origin, Paint, union}; #[cfg(test)] pub(crate) use lines::dropped_advances_within; diff --git a/crates/edit/src/text/editor.rs b/crates/edit/src/text/editor.rs index 4f4abf3a43..e6df1dbfee 100644 --- a/crates/edit/src/text/editor.rs +++ b/crates/edit/src/text/editor.rs @@ -2,7 +2,7 @@ use super::blocks::{BlockStyle, TextBlock}; use super::fallback::MAX_TYPE3_GLYPHS; use super::lines::{page_dict, text_lines}; use super::rewrite::rewrite_block_from; -use super::{DecodeBudget, Dict, Document, EditError, Metrics, Object, TextLine, TextSession}; +use super::{DecodeBudget, Dict, Document, EditError, Object, TextLine, TextSession}; /// An embedded source font or an approved bundled document-font choice. /// Source bytes stay in memory and are never installed or copied into another document. @@ -45,7 +45,7 @@ pub fn text_editor_fonts(doc: &Document, page: usize) -> Result, Option, Result, String>); +#[cfg(test)] +thread_local! { + /// Font metrics built by [`interpret`] on this thread, for the test that keeps font aliases + /// sharing one set. + pub(crate) static METRICS_BUILT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// A `Do` met while interpreting: its operator index, XObject name and the state there. type Invocation = (usize, Vec, Ts); @@ -976,6 +983,7 @@ pub(super) fn interpret( let fonts_res = super::resource_dict(doc, resources, b"Font", &mut held_fonts).unwrap_or(&none); let states = super::resource_dict(doc, resources, b"ExtGState", &mut held_states).unwrap_or(&none); let mut out: Vec = Vec::new(); + let mut by_font: HashMap = HashMap::new(); let mut ts = carry.ts.clone(); let mut at_bt = (0usize, ts.clone()); let mut stack: Vec<(Ts, Matrix, Matrix)> = std::mem::take(&mut carry.stack); @@ -1105,11 +1113,23 @@ pub(super) fn interpret( ts.size = op.num(1).unwrap_or(ts.size); if let Some(name) = op.name(0) { let (m, face, vertical) = cache.entry(name.to_vec()).or_insert_with(|| { + // Names that refer to one font object share what was built for it: a + // thousand aliases of one font are one set of metrics, not a thousand. + let font = fonts_res.get(name).and_then(Object::as_ref); + if let Some(built) = font.and_then(|r| by_font.get(&r)) { + return built.clone(); + } + #[cfg(test)] + METRICS_BUILT.with(|n| n.set(n.get() + 1)); let dict = fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); let metrics = dict.as_ref().map_or_else(Metrics::fallback, |d| Metrics::from_dict(doc, d)); let vertical = dict.as_ref().map_or(Ok(None), |d| vertical_advance(doc, d).map_err(|e| e.to_string())); let face = dict.filter(is_type3_fallback).map(|_| metrics.base_font.clone()); - (Rc::new(metrics), face, vertical) + let built: CachedFont = (Rc::new(metrics), face, vertical); + if let Some(r) = font { + by_font.insert(r, built.clone()); + } + built }); ts.font = Some((name.to_vec(), m.clone())); ts.face = face.clone(); diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index f5b924c921..4278dbb95f 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -58,6 +58,8 @@ getrandom = "0.4" getrandom = { version = "0.4", features = ["wasm_js"] } [dev-dependencies] +# Synthetic printer profiles for the app-managed print colour tests. +craft-cms = { workspace = true } pdfcraft-testkit = { workspace = true } # JBIG2 fixtures (`encode_jbig2_generic`). pdfcraft-filters = { workspace = true } diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index c98893ab9c..8a166330b9 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -960,41 +960,60 @@ pub(crate) fn recovered_render_password(cos: &pdfcraft_cos::Document) -> Option< /// the document as it was. pub(crate) fn shown_under_redactions( after: &pdfcraft_cos::Document, - marks: &[pdfcraft_redact::Mark], + marks: &[(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)], overlays: &[pdfcraft_cos::ObjRef], - pages: &[pdfcraft_render::PageInfo], password: Option<&str>, ) -> Result<(), EditError> { use pdfcraft_cos::Object; use pdfcraft_redact::RedactError; - let mut marked: Vec = marks.iter().map(|m| m.page).collect(); + let mut shown = after.clone(); + let model = pdfcraft_model::pages(&shown); + // A batch can move pages after the marks were collected: a mark is found by the page object it + // was placed on, not by the index that page had when it was drawn. A page the batch deleted + // takes its marks with it, so they are not checked against whatever page now holds the index. + // One index, built once: each mark is resolved against it instead of scanning every page, + // and the marks are grouped by resolved page in the same pass. + let mut index: std::collections::HashMap = std::collections::HashMap::new(); + for (i, p) in model.iter().enumerate() { + index.entry(p.obj).or_insert(i); + } + let mut marked: Vec = Vec::new(); + let mut rects_of: std::collections::HashMap> = std::collections::HashMap::new(); + for (obj, m) in marks { + let Some(&pi) = index.get(obj) else { continue }; + if !rects_of.contains_key(&pi) { + marked.push(pi); + } + rects_of.entry(pi).or_default().extend(m.rects.iter().map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])])); + } marked.sort_unstable(); - marked.dedup(); if marked.is_empty() { return Ok(()); } - let mut shown = after.clone(); - let model = pdfcraft_model::pages(&shown); for &pi in &marked { let Some(page) = model.get(pi) else { continue }; let Some(Object::Array(list)) = page.dict.get(b"Contents").map(|c| (*shown.resolve(c)).clone()) else { continue }; let kept: Vec = list.into_iter().filter(|o| !o.as_ref().is_some_and(|r| overlays.contains(&r))).collect(); shown.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(kept))).map_err(|e| EditError::Write(e.to_string()))?; } - let bytes = write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?; + let bytes = Arc::new(write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?); + // Page geometry comes from the document as it is now written: a page moved earlier in the batch + // brings its own size and rotation with it. + let pages = pdfcraft_render::inspect(bytes.clone(), password).map_err(|e| EditError::Write(e.to_string()))?.pages; + // The model counts one entry per `/Kids` entry; the renderer one per page it draws. If they + // disagree, a mark's index could land on another page's text, and the check cannot confirm + // the redaction. + if pages.len() != model.len() { + return Err(RedactError::AmbiguousPages.into()); + } let config = RenderConfig { password: password.map(Arc::from), ..Default::default() }; - let mut renderer = pdfcraft_render::PageRenderer::new(Arc::new(bytes), config); + let mut renderer = pdfcraft_render::PageRenderer::new(bytes, config); let mut residue = 0; for &pi in &marked { let request = pdfcraft_render::RenderRequest { page: pi, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; // A page that cannot be read back cannot be confirmed clean. let (Some(info), Some(text)) = (pages.get(pi), renderer.render(request).text) else { return Err(RedactError::Unreadable(pi + 1).into()) }; - let rects: Vec<[f64; 4]> = marks - .iter() - .filter(|m| m.page == pi) - .flat_map(|m| m.rects.iter()) - .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]) - .collect(); + let rects = rects_of.get(&pi).cloned().unwrap_or_default(); for glyph in text.glyphs.iter().filter(|g| !g.text.trim().is_empty()) { let [x0, y0, x1, y1] = info.view_rect_to_user(glyph.rect).map(f64::from); let area = (x1 - x0) * (y1 - y0); diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 57e5729fc8..0a7a348fca 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -253,6 +253,11 @@ pub(crate) struct EditCtx { /// The overlay streams the edit's applied redactions added, which alone the renderer's check /// leaves out. pub(crate) redaction_overlays: Vec, + /// The marks the edit's applied redactions took out (a batch's included), as they were when + /// each apply ran, for the renderer's check. + /// Each applied mark with the page object it was drawn on, which a later page move changes the + /// index of but not the identity. + pub(crate) redaction_marks: Vec<(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)>, /// What the edit's applied redactions left as it was that still holds the removed text. pub(crate) redaction_elsewhere: Option, /// What became of the PDF/A and PDF/UA identification when the edit's applied redactions @@ -310,6 +315,7 @@ impl EditCtx { rng: getrandom::fill, renamed_fields: Vec::new(), redaction_overlays: Vec::new(), + redaction_marks: Vec::new(), redaction_elsewhere: None, redaction_identification: pdfcraft_redact::Identification::default(), prepared: None, @@ -529,7 +535,17 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_organize::insert_blank_page(doc, *at, *width, *height)?; } Edit::SetInfo { key, value } => pdfcraft_organize::set_info(doc, key, value)?, - Edit::DuplicatePages { pages, at } => pdfcraft_organize::duplicate_pages(doc, pages, *at)?, + Edit::DuplicatePages { pages, at } => { + // A copy of a page cannot be tied back to the marks an earlier redaction of this edit + // applied: text put back under a mark would sit on the copy, unchecked. Fail closed. + if !cx.redaction_marks.is_empty() { + return Err(EditError::Invalid( + "a batch that applies redactions can't also copy pages: the copies can't be checked against the marks; the document was not changed" + .into(), + )); + } + pdfcraft_organize::duplicate_pages(doc, pages, *at)? + } Edit::ReplacePages { pages, name, bytes, src_pages } => { let src = crate::sources::open_source_into(name, bytes, doc)?; pdfcraft_organize::replace_pages(doc, pages, &src, src_pages)?; @@ -991,8 +1007,15 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_edit::add_content(doc, *page, &content)?; } Edit::ApplyRedactions { pages } => { + let model = pdfcraft_model::pages(doc); + cx.redaction_marks.extend( + pdfcraft_redact::marks(doc) + .into_iter() + .filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))) + .filter_map(|m| model.get(m.page).map(|p| (p.obj, m))), + ); let report = pdfcraft_redact::apply(doc, pages.as_deref())?; - cx.redaction_overlays = report.overlays; + cx.redaction_overlays.extend(report.overlays); cx.redaction_elsewhere = Some(report.still_holding); cx.redaction_identification = report.identification; } diff --git a/crates/engine/src/print_raster.rs b/crates/engine/src/print_raster.rs index a6076d5b05..7a3bbc40d5 100644 --- a/crates/engine/src/print_raster.rs +++ b/crates/engine/src/print_raster.rs @@ -3,7 +3,8 @@ //! shading or transparency. The pages are rendered the way the viewer shows them (reading //! colours off, the print's own colours) with only what the print's content choice keeps, made //! into an image-only document, and that document is imposed on the paper like any other. The -//! print is no longer searchable text. +//! print is no longer searchable text. With app-managed colour the pictures are converted into +//! the printer's ICC profile ([`pdfcraft_print::colour`]). use pdfcraft_cos::{SaveOptions, write_incremental}; use pdfcraft_create::ImageResolution; @@ -51,6 +52,11 @@ pub fn rasterize(doc: &Document, pages: &[usize], raster: &Raster, content: Cont } let mut rgba = r.rgba.to_vec(); over_white(&mut rgba, raster.grayscale); + // App-managed colour: the picture's values become the printer's, so it is placed as + // device colour the driver passes on unconverted. + if let Some(colour) = &raster.colour { + colour.convert(&mut rgba).map_err(|e| format!("page {}: {e}", page + 1))?; + } images.push((format!("page-{}.png", page + 1), encode_png(r.width, r.height, &rgba)?)); } let dpi = used.unwrap_or(dpi); diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index 2d805e92a9..52922da290 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -186,6 +186,7 @@ impl Session { } cx.renamed_fields.clear(); cx.redaction_overlays.clear(); + cx.redaction_marks.clear(); // `next` is a copy: if the edit fails or crashes, the document is unchanged. run_checked_edit(&mut next, &edit, &mut cx)?; // A field's enter or exit scripts that changed nothing leave no edit behind (no undo step, @@ -196,11 +197,20 @@ impl Session { return Ok(None); } // Redaction removed what its own interpreter places under the marks. The marks were drawn - // over the page as the renderer shows it, so the renderer has the last word. - if let Edit::ApplyRedactions { pages } = &edit { - let marks: Vec = - pdfcraft_redact::marks(&editor.cos).into_iter().filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))).collect(); - guard(|| shown_under_redactions(&next, &marks, &cx.redaction_overlays, &doc.info.pages, doc.password.as_deref())) + // over the page as the renderer shows it, so the renderer has the last word: on every + // apply the edit ran, a batch's included. + if !cx.redaction_marks.is_empty() { + // A batch that deletes a marked page and adds pages cannot be tied back to the marks: + // a copy of a page may carry the text the redaction removed from the original only. + // Fail closed. + let before: std::collections::HashSet<_> = pdfcraft_model::pages(&editor.cos).iter().map(|p| p.obj).collect(); + let after: std::collections::HashSet<_> = pdfcraft_model::pages(&next).iter().map(|p| p.obj).collect(); + if cx.redaction_marks.iter().any(|(o, _)| !after.contains(o)) && after.iter().any(|o| !before.contains(o)) { + return Err(EditError::Invalid( + "a batch that deletes a page with a redaction mark and adds pages can't be verified; the document was not changed".into(), + )); + } + guard(|| shown_under_redactions(&next, &cx.redaction_marks, &cx.redaction_overlays, doc.password.as_deref())) .unwrap_or_else(|m| Err(EditError::Invalid(format!("{} failed unexpectedly ({m}); the document was not changed", edit.label()))))?; } // Renamed fields are said, not slipped in: a repair-log note (with the caveat that diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index 2f2774ed22..85a6d24532 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -236,8 +236,9 @@ impl Session { // Table 22, bit 12 clear: only a low-level representation may be printed, so the pages // always go as pictures, at no more than the low resolution. let low_res = doc.permissions().is_some_and(|p| !p.print_high_quality()).then(|| print::Raster { - dpi: settings.raster.map_or(LOW_RESOLUTION_PRINT_DPI, |r| r.dpi.min(LOW_RESOLUTION_PRINT_DPI)), - grayscale: settings.raster.is_some_and(|r| r.grayscale), + dpi: settings.raster.as_ref().map_or(LOW_RESOLUTION_PRINT_DPI, |r| r.dpi.min(LOW_RESOLUTION_PRINT_DPI)), + grayscale: settings.raster.as_ref().is_some_and(|r| r.grayscale), + colour: settings.raster.as_ref().and_then(|r| r.colour.clone()), }); // Print as image: the pages used become pictures first, then the picture document is // imposed in their place (its page k is the k-th distinct source page used). diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4e24a4f608..bf1a4ebd01 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1935,8 +1935,8 @@ fn low_resolution_printing_prints_pictures_of_at_most_150_dpi() { let low = s.open("low.pdf", None, bytes.clone(), None).unwrap(); let width = f64::from(s.get(low).unwrap().info.pages[0].width); let actual = print::Settings { pages: vec![0], layout: print::Layout::Size(print::SizeMode::Actual), ..Default::default() }; - for raster in [None, Some(print::Raster { dpi: 600.0, grayscale: false })] { - let printed = s.print_pdf(low, &print::Settings { raster, ..actual.clone() }).unwrap(); + for raster in [None, Some(print::Raster { dpi: 600.0, grayscale: false, colour: None })] { + let printed = s.print_pdf(low, &print::Settings { raster: raster.clone(), ..actual.clone() }).unwrap(); let mut s2 = Session::new(); let id2 = s2.open("print.pdf", None, Arc::new(printed.clone()), None).unwrap(); assert!(page_texts(&s2, id2).iter().all(|t| t.trim().is_empty()), "{raster:?}: no text is left"); @@ -3542,9 +3542,10 @@ fn the_renderer_check_refuses_text_still_shown_under_a_mark() { mark_shown_text(&mut s, id, "SECRET42"); let doc = s.get(id).unwrap(); let mut cos = doc.editor.as_ref().unwrap().cos.clone(); - let marks = pdfcraft_redact::marks(&cos); + let model = pdfcraft_model::pages(&cos); + let marks: Vec<_> = pdfcraft_redact::marks(&cos).into_iter().filter_map(|m| model.get(m.page).map(|p| (p.obj, m))).collect(); let refused = |cos: &pdfcraft_cos::Document| { - let shown = shown_under_redactions(cos, &marks, &[], &doc.info.pages, None); + let shown = shown_under_redactions(cos, &marks, &[], None); matches!(shown, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0) }; assert!(refused(&cos)); @@ -3933,8 +3934,8 @@ fn printing_applies_the_print_auto_state() { } // Hidden in the panel, View entry (no Print recommendation) leaves the preview too. assert!(printed_blue[1] * 4 < printed_blue[0] * 3, "the preview follows the panel: {printed_blue:?} blue"); - for raster in [None, Some(print::Raster { dpi: 72.0, grayscale: false })] { - let bytes = Arc::new(s.print_pdf(id, &print::Settings { pages: vec![0], raster, ..Default::default() }).unwrap()); + for raster in [None, Some(print::Raster { dpi: 72.0, grayscale: false, colour: None })] { + let bytes = Arc::new(s.print_pdf(id, &print::Settings { pages: vec![0], raster: raster.clone(), ..Default::default() }).unwrap()); let mut out = Session::new(); let printed = out.open("print.pdf", None, bytes.clone(), None).unwrap(); let (red, green, blue) = colours(&render_page(&out, printed)); @@ -5838,7 +5839,7 @@ fn assert_signature_artwork_survives_trust(mut cos: pdfcraft_cos::Document, expe } let mut settings = print::Settings { pages: vec![0], paper: (200.0, 200.0), ..Default::default() }; let original_vector_print = printed_pixels(session.print_pdf(id, &settings).unwrap()); - settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false, colour: None }); let original_raster_print = printed_pixels(session.print_pdf(id, &settings).unwrap()); session.set_trusted_certificates(vec![anchor.clone()]); let doc = session.get(id).unwrap(); @@ -6203,7 +6204,7 @@ fn named_layer_states_reach_exports_and_both_print_modes() { assert!(text.contains("Signature valid") && !text.contains("Not Verified"), "{text:?}"); let mut settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; let (_, _, vector) = printed_pixels(s.print_pdf(id, &settings).unwrap()); - settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false, colour: None }); let (_, _, raster) = printed_pixels(s.print_pdf(id, &settings).unwrap()); for (mode, pixels) in [("vector", &vector), ("raster", &raster)] { assert!(pixels_near(pixels, VALID_GREEN) > 500, "{mode} print draws the valid state"); @@ -11318,7 +11319,7 @@ fn printing_as_image_sends_pictures_instead_of_pages() { let settings = print::Settings { pages: vec![1, 2, 1], layout: print::Layout::Size(print::SizeMode::Actual), - raster: Some(print::Raster { dpi: 100.0, grayscale: false }), + raster: Some(print::Raster { dpi: 100.0, grayscale: false, colour: None }), ..Default::default() }; let bytes = s.print_pdf(id, &settings).unwrap(); @@ -11340,7 +11341,8 @@ fn printing_as_image_sends_pictures_instead_of_pages() { assert!((w as f64 - page_size.0 * 100.0 / 72.0).abs() <= 1.0, "{w} px for {} pt", page_size.0); assert!(images.iter().all(|d| d.name(b"ColorSpace") == Some(b"DeviceRGB")), "{images:?}"); // Grayscale renders the pictures grey (the printer is not merely asked for monochrome). - let grey = s.print_pdf(id, &print::Settings { raster: Some(print::Raster { dpi: 72.0, grayscale: true }), ..settings.clone() }).unwrap(); + let grey = + s.print_pdf(id, &print::Settings { raster: Some(print::Raster { dpi: 72.0, grayscale: true, colour: None }), ..settings.clone() }).unwrap(); let mut s3 = Session::new(); let id3 = s3.open("grey.pdf", None, Arc::new(grey), None).unwrap(); let mut ex = crate::export::Exporter::new(s3.get(id3).unwrap()); @@ -11349,6 +11351,44 @@ fn printing_as_image_sends_pictures_instead_of_pages() { assert!(img.pixels().all(|p| p[0] == p[1] && p[1] == p[2]), "every pixel grey"); } +/// #6269: with app-managed colour the printed pictures carry the printer profile's values, not +/// sRGB's: a pure green page converted into a wider-gamut RGB profile gains red; without it the +/// pictures stay sRGB. +#[test] +fn app_managed_colour_converts_the_printed_pictures() { + let pdf = pdfcraft_testkit::Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Contents 4 0 R >>") + .stream("", "0 1 0 rg 0 0 100 100 re f") + .build(); + let mut s = Session::new(); + let id = s.open("green.pdf", None, Arc::new(pdf), None).unwrap(); + let wide = craft_cms::Builtin::AdobeRgbCompat.profile().to_bytes(); + let colour = print::colour::ManagedColour { + profile: print::colour::PrinterProfile::from_bytes(&wide).unwrap(), + intent: print::colour::Intent::RelativeColorimetric, + black_point: true, + }; + let middle = |colour: Option| { + let settings = print::Settings { + pages: vec![0], + layout: print::Layout::Size(print::SizeMode::Actual), + raster: Some(print::Raster { dpi: 72.0, grayscale: false, colour }), + ..Default::default() + }; + let mut out = Session::new(); + let printed = out.open("print.pdf", None, Arc::new(s.print_pdf(id, &settings).unwrap()), None).unwrap(); + let png = crate::export::Exporter::new(out.get(printed).unwrap()).png(0, 72.0).unwrap(); + let img = image::load_from_memory(&png).unwrap().to_rgba8(); + img.get_pixel(img.width() / 2, img.height() / 2).0 + }; + let srgb = middle(None); + assert!(srgb[0] < 10 && srgb[1] > 245, "sRGB green: {srgb:?}"); + let converted = middle(Some(colour)); + assert!(converted[0] > 100 && converted[1] > 240, "the profile's values for sRGB green: {converted:?}"); +} + /// A one-page PDF: a JPEG logo (DCTDecode) at the top and a big text title below it. fn jpeg_cover_pdf(logo_y: i32) -> Vec { let mut jpeg = Vec::new(); @@ -13159,7 +13199,7 @@ fn buttons_and_combs_left_to_the_viewer_are_displayed() { } assert_eq!(count(&shown, (95, 84, 97, 106), &blue), 0, "no line inside a cell"); // Print as Image: the page as a picture, at 72 dpi so it is the size of the page here. - let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false }; + let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false, colour: None }; let pictures = crate::print_raster::rasterize(s.get(id).unwrap(), &[0], &raster, pdfcraft_print::Content::Document).unwrap(); let printed = render(Arc::new(pictures)); assert!(count(&printed, (24, 154, 56, 186), &red) > 60, "the check box prints"); @@ -13329,7 +13369,7 @@ fn need_appearances_buttons_are_displayed_from_their_captions() { let mut s = Session::new(); let id = s.open("form.pdf", None, Arc::new(pdf(true)), None).unwrap(); let shown = render(s.get(id).unwrap().display.clone()); - let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false }; + let raster = pdfcraft_print::Raster { dpi: 72.0, grayscale: false, colour: None }; let pictures = crate::print_raster::rasterize(s.get(id).unwrap(), &[0], &raster, pdfcraft_print::Content::Document).unwrap(); let printed = render(Arc::new(pictures)); for (page, how) in [(&shown, "shown"), (&printed, "printed")] { @@ -13778,6 +13818,66 @@ fn image_transform_engine_refuses_singular_matrices_without_mutation() { } } +/// The renderer's check covers a redaction applied inside a batch too, with the marks as they +/// were when it ran: one the batch itself added is checked, and text a later edit of the batch +/// puts back under a mark refuses the whole batch. +#[test] +fn redactions_applied_in_a_batch_get_the_renderer_check() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + // Text added where the mark was shows under it again: the renderer sees it and refuses. + let under = AddedText { rect: [150.0, 130.0, 300.0, 160.0], text: "SECRET42 SECRET42".into(), ..Default::default() }; + let refused = s.apply(id, Edit::Batch { label: "Redact then write".into(), edits: vec![apply.clone(), Edit::AddText { page: 0, text: under }] }); + assert!(matches!(refused, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Applied alone in a batch, the redaction goes through, and the text is gone. + s.apply(id, Edit::Batch { label: "Redact".into(), edits: vec![apply] }).unwrap(); + assert!(!page_texts(&s, id)[0].contains("SECRET42"), "{:?}", page_texts(&s, id)); + assert_eq!(s.get(id).unwrap().redaction_marks(), 0); +} + +/// A batch that deletes a page holding a redaction mark and adds pages is refused whole: the +/// marks cannot be tied to copies, so redaction fails closed. Without added pages it goes through. +#[test] +fn a_batch_that_deletes_a_marked_page_and_adds_pages_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let edits = vec![ + Edit::ApplyRedactions { pages: None }, + Edit::InsertBlankPage { at: 1, width: 612.0, height: 792.0 }, + Edit::DeletePages { pages: vec![0] }, + ]; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits }); + assert!(matches!(refused, Err(EditError::Invalid(_))), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); +} + +/// A batch that applies redactions and then copies a page is refused whole: the copy can't be tied +/// to the marks, so text restored under one would be unchecked. Copying first, or alone, is fine. +#[test] +fn a_batch_that_applies_redactions_and_copies_a_page_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + let copy = Edit::DuplicatePages { pages: vec![0], at: None }; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits: vec![apply.clone(), copy.clone()] }); + assert!(matches!(&refused, Err(EditError::Invalid(m)) if m.contains("copy pages")), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Copying before the redaction is applied puts the mark on the copy too, and both are checked. + s.apply(id, Edit::Batch { label: "y".into(), edits: vec![copy, apply] }).unwrap(); + assert!(page_texts(&s, id).iter().all(|t| !t.contains("SECRET42")), "{:?}", page_texts(&s, id)); +} + /// A reset in an opening script reaches a field of a row an earlier opening script added, before /// the form is laid out again with the row's widgets: its data goes back to its default and stays /// so when the form is saved. diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index f68761e02e..7ae75ddf31 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -70,7 +70,10 @@ pub struct Metrics { /// A stream these metrics come from (a CMap, the program, `/CIDToGIDMap`, a Type 3 glyph /// description) was left unread because the decode allowance [`Metrics::from_dict_within`] /// was given was spent: what it would have decided (code lengths, CIDs, widths, meanings, a - /// Type 3 font's height) is a guess. + /// Type 3 font's height) is a guess. It is set exactly when the allowance counted a refusal + /// during the read ([`DecodeBudget::refused`]), which callers that share it may refuse on; a + /// read that fails without one, such as an empty stream behind a filter nothing decodes, + /// doesn't set it, with or without anything left. pub past_allowance: bool, } @@ -420,12 +423,16 @@ impl<'a> Reads<'a> { /// stage past `max` is unread. When what is left is more than what the stages before that one /// made and `max`, it is charged those two and isn't refused; otherwise, with more left than /// the stages' total (behind one filter, `max`), it is charged that total and isn't refused; - /// with no more left than that total, it spends all of it and counts as refused. + /// with no more left than that total, it spends all of it and counts as refused. Only what + /// the allowance counts as refused ([`DecodeBudget::refused`]) is noted: a read that fails + /// without one, such as an empty stream behind a filter nothing decodes, fails the same with + /// more left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. fn decode(&mut self, stream: &Stream, max: usize) -> Option> { + let refused = self.budget.refused(); match self.budget.decode_stages_within(self.doc, stream, max) { Ok(data) => Some(data), Err(_) => { - self.refused |= self.budget.remaining() == 0; + self.refused |= self.budget.refused() > refused; None } } @@ -491,10 +498,14 @@ fn type3_glyph_box(doc: &Document, font: &Dict, reads: &mut Reads<'_>) -> Option break; } let before = reads.budget.remaining(); + let refused = reads.budget.refused(); let read = reads.budget.decode_with_damage(doc, s, MAX_TYPE3_GLYPH_BYTES.min(left)).map(|(data, _)| data); left = left.saturating_sub(before.saturating_sub(reads.budget.remaining())); let Ok(data) = read else { - if reads.budget.remaining() == 0 { + // As in `Reads::decode`, only a refusal the allowance counts: a description that fails + // without one (an empty one nothing decodes, once the allowance is spent) leaves the + // rest to be read, or refused above. + if reads.budget.refused() > refused { // The rest go unread too, for want of the reading's allowance. reads.refused = true; complete = false; @@ -810,6 +821,20 @@ fn number(t: &Tok) -> Option { } } +/// The font's `/BaseFont` (a Type 3 font's descriptor `/FontName` when it has none), read without +/// building its metrics, so it decodes no CMap. +pub fn base_font_name(doc: &Document, font: &Dict) -> String { + font.name(b"BaseFont") + .map(|name| String::from_utf8_lossy(name).into_owned()) + .or_else(|| { + (font.name(b"Subtype") == Some(b"Type3")) + .then(|| dict(doc, font.get(b"FontDescriptor"))) + .flatten() + .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) + }) + .unwrap_or_default() +} + /// Codespace ranges and CID mappings from an embedded CMap stream, at most /// [`MAX_CMAP_RANGES`] of each (every glyph searches them). #[allow(clippy::type_complexity)] @@ -898,16 +923,7 @@ impl Metrics { pub fn from_dict_within(doc: &Document, font: &Dict, budget: &mut DecodeBudget) -> Self { let mut reads = Reads::new(doc, budget); let mut m = Self::read_metrics(doc, font, &mut reads); - let base = font - .name(b"BaseFont") - .map(|name| String::from_utf8_lossy(name).into_owned()) - .or_else(|| { - (font.name(b"Subtype") == Some(b"Type3")) - .then(|| dict(doc, font.get(b"FontDescriptor"))) - .flatten() - .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) - }) - .unwrap_or_default(); + let base = base_font_name(doc, font); m.subset = base.len() > 7 && base.as_bytes()[6] == b'+' && base[..6].bytes().all(|b| b.is_ascii_uppercase()); let (bold, italic) = style_from_name(&base); m.bold |= bold; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 975b71289e..74880c94a4 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -309,17 +309,27 @@ fn a_fonts_streams_are_paid_for_from_its_allowance() { } /// With nothing left, an empty stream still reads, as the allowance reads one, and is no reason to -/// call the font read past it; a stream with data is left unread, and is. +/// call the font read past it; a stream with data is left unread, and is. An empty stream behind a +/// filter nothing decodes is unread with any allowance, so it isn't either: the font is read past +/// its allowance exactly when the allowance counts a refusal, which is what callers refuse on. #[test] fn an_empty_stream_reads_with_nothing_left() { - let past = |data: &[u8]| { + let past = |filter: Option<&str>, data: &[u8]| { let mut doc = Document::new_empty(); - let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), data.to_vec())))); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); let f = font(&mut doc, vec![("Subtype", Object::name("TrueType")), ("ToUnicode", unicode)]); - Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(0, 0)).past_allowance + let mut budget = DecodeBudget::new(0, 0); + let past = Metrics::from_dict_within(&doc, &f, &mut budget).past_allowance; + assert_eq!(past, budget.refused() > 0, "{filter:?}"); + past }; - assert!(!past(b"")); - assert!(past(b"1 beginbfchar <41> <0058> endbfchar")); + assert!(!past(None, b"")); + assert!(!past(Some("NoSuchFilter"), b"")); + assert!(past(None, b"1 beginbfchar <41> <0058> endbfchar")); } /// A stream behind one filter past its own limit, with more left than its limit, is left unread as @@ -424,6 +434,45 @@ fn each_stage_of_a_metrics_stream_is_held_to_its_limit() { assert_eq!(read(MAX_CID_TO_GID / 4), (false, false)); } +/// The editor's program list reads a legacy Thai font's metrics after its program. With the program +/// spending the allowance to the byte, a ToUnicode that is an empty stream behind a filter nothing +/// decodes, unread with any allowance, doesn't put the metrics past the allowance: the program is +/// listed. A ToUnicode with data is refused by the allowance, and the program isn't. +#[test] +fn an_editors_program_is_listed_past_a_stream_unread_on_its_own() { + let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let listed = |filter: Option<&str>, data: &[u8]| { + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); + let program = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), bytes.to_vec())))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("TrueType")), + ("BaseFont", Object::name("THSarabunPSK")), + ("FirstChar", Object::Int(65)), + ("LastChar", Object::Int(65)), + ("Widths", Object::Array(vec![Object::Int(500)])), + ("Encoding", Object::name("WinAnsiEncoding")), + ("FontDescriptor", Object::Dict(descriptor)), + ("ToUnicode", unicode), + ], + ); + let mut budget = DecodeBudget::new(bytes.len(), 0); + let listed = embedded_sfnt_within(&doc, &f, &mut budget).is_some(); + assert_eq!(budget.remaining(), 0, "the program spends the allowance to the byte"); + (listed, budget.refused()) + }; + assert_eq!(listed(Some("NoSuchFilter"), b""), (true, 0)); + assert_eq!(listed(None, b"1 beginbfchar <41> <0058> endbfchar"), (false, 1)); +} + /// The editor's program is held to its limit as the metrics reads' streams are: a 12 MiB program /// whose Flate data is about 8 MiB reads behind ASCIIHex and Flate as behind Flate alone, though /// the two stages make about 20 MiB together. A program of the limit reads behind either, and one a @@ -969,6 +1018,41 @@ fn type3_glyph_boxes_are_paid_for_from_the_allowance() { assert!(m.past_allowance, "a byte short, the second description is left unread"); } +/// A glyph description that fails on its own once the allowance is spent to the byte, an empty one +/// behind a filter nothing decodes, fails the same with more left: it doesn't make the metrics read +/// past the allowance, and the descriptions after it are read, or refused by the allowance. +#[test] +fn a_type3_description_unread_on_its_own_is_not_past_the_allowance() { + let mut doc = Document::new_empty(); + let procs = ["1 0 0 -20 1 300 d1", "", "1 0 0 0 1 400 d1"]; + let mut f = type3(&mut doc, [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0, 0, 0, 0], &procs); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let unread = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new())))); + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.set(b"g1".to_vec(), unread); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + with_differences(&mut f, &["g0", "g1", "g2"].map(String::from)); + let first = procs[0].len(); + let whole = Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(usize::MAX, 0)); + assert!(!whole.past_allowance); + assert_eq!((whole.ascent, whole.descent), (400.0, -20.0)); + // The first description spends it to the byte; the third, which has data, is refused. + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!(budget.refused(), 1); + assert!(m.past_allowance); + // Without the third, nothing is refused, and the metrics aren't past the allowance. + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.remove(b"g2"); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!((budget.refused(), budget.remaining()), (0, 0)); + assert!(!m.past_allowance); + assert_eq!((m.ascent, m.descent), (300.0, -20.0)); +} + /// A Type 3 font measured from its glyphs' `d1` boxes keeps them over a descriptor's `/Ascent` /// and `/Descent`, which are optional for Type 3 (ISO 32000-2 Table 120). #[test] @@ -1739,6 +1823,16 @@ fn low_codes_resolve_to_the_covering_range() { assert_eq!((u.covering(0xFFFF).is_some(), u.covering(0x1_0001).is_some(), u.covering(0x1_0002).is_none()), (true, true, true)); } +#[test] +fn base_font_name_matches_the_metrics_name() { + let doc = Document::new_empty(); + let mut f = Dict::new(); + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+Body")); + assert_eq!(base_font_name(&doc, &f), "ABCDEF+Body"); + assert_eq!(base_font_name(&doc, &f), Metrics::from_dict(&doc, &f).base_font); + assert_eq!(base_font_name(&doc, &Dict::new()), ""); +} + #[test] fn refused_subset_reads_do_not_encode_unverified_glyphs() { let mut doc = Document::new_empty(); diff --git a/crates/forms/src/author/lifecycle.rs b/crates/forms/src/author/lifecycle.rs index c079b40f13..b2de1321a0 100644 --- a/crates/forms/src/author/lifecycle.rs +++ b/crates/forms/src/author/lifecycle.rs @@ -263,10 +263,26 @@ pub fn delete_widget(doc: &mut Document, name: &str, widget: usize) -> Result<() fn delete_found_field(doc: &mut Document, f: Field) -> Result<(), FormError> { let widgets: std::collections::HashSet = f.widgets.iter().map(|w| w.obj).collect(); drop_from_pages(doc, &widgets)?; + let mut parent = doc.get(f.obj).as_dict().and_then(|d| d.reference(b"Parent")); detach(doc, f.obj)?; // Out of the calculation order too: a `/CO` entry would keep the field, its value and its // widgets reachable for a full save. - drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r)) + drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r))?; + // An ancestor left with no kids would become a field of its own, and keep the `/V` and `/DV` + // the deleted field inherited from it: it goes too, up the chain. + let mut gone = std::collections::HashSet::new(); + while let Some(p) = parent.filter(|p| gone.len() < 64 && !gone.contains(p)) { + let Some(d) = doc.get(p).as_dict().cloned() else { break }; + let empty = d.get(b"Kids").map(|o| doc.resolve(o)).and_then(|o| o.as_array().map(Vec::is_empty)) == Some(true); + if !empty { + break; + } + parent = d.reference(b"Parent"); + gone.insert(p); + drop_from_pages(doc, &std::iter::once(p).collect())?; + detach(doc, p)?; + } + drop_from_calculation_order(doc, |r| gone.contains(&r)) } /// Take the annotations `widgets` off every page's `/Annots`, held directly or by reference. diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 7d8bdc4673..309d0766d0 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -1664,6 +1664,38 @@ fn properties_rename_and_delete() { assert_eq!(annots.as_array().unwrap().len(), 13 - 3, "the deleted widgets left the page"); } +/// A deleted field's ancestor left with no kids stays in `/Fields` as a field of its own, with the +/// `/V` the deleted field inherited from it: it leaves too. An ancestor a sibling still has stays. +#[test] +fn deleting_the_last_kid_removes_the_emptied_ancestors_and_their_values() { + let form = |kids: &str, annots: &str| { + document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{annots}] >>"), + "<< /Fields [5 0 R] >>".into(), + "<< /FT /Tx /T (outer) /V (SECRET) /DV (SECRET) /Kids [6 0 R] >>".into(), + format!("<< /T (inner) /Parent 5 0 R /Kids [{kids}] >>"), + "<< /Type /Annot /Subtype /Widget /T (a) /Parent 6 0 R /Rect [50 700 250 720] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /T (b) /Parent 6 0 R /Rect [50 650 250 670] /P 3 0 R >>".into(), + ]) + }; + let top = |doc: &Document| { + let af = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"AcroForm").map(|o| doc.resolve(o)).unwrap().as_dict().unwrap().clone(); + doc.resolve(af.get(b"Fields").unwrap()).as_array().unwrap().len() + }; + let mut doc = form("7 0 R", "7 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 0, "the emptied ancestors left /Fields"); + let saved = pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(6).any(|w| w == b"SECRET"), "the inherited value is gone from a full save"); + + let mut doc = form("7 0 R 8 0 R", "7 0 R 8 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 1, "an ancestor with a surviving kid stays"); + assert!(fields(&doc).iter().any(|f| f.name == "outer.inner.b")); +} + /// A parent `MASTER` with kids `a` and `b` (each a field merged with its widget) and `e` (a field /// with two widget kids), and the top-level fields `c` and `d`. fn dotted_form() -> Document { @@ -5569,6 +5601,75 @@ fn a_page_widget_inheriting_password_flag_beyond_depth_32_is_recognized() { assert!(!has_password_values(&doc)); } +/// A field reachable both near the top of the tree and at the end of a long branch: the field +/// walk reaches it near the top and reads the password field below it, so the quick check must +/// not stop at the long branch's depth limit and conclude there is none. The password field's +/// widget is on the page but names no `/Parent`, so the page widgets alone don't find it either. +#[test] +fn a_password_field_past_the_depth_limit_on_one_branch_is_still_found() { + let chain = 64; + let x = 6 + chain; // the shared field + let y = x + 1; // the password field + let w = y + 1; // its widget + let mut objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{w} 0 R] >>"), + "<< /Fields [5 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + format!("<< /T (short) /Kids [{x} 0 R] >>"), + ]; + for d in 0..chain { + let next = if d + 1 == chain { x } else { 7 + d }; + objs.push(format!("<< /T (n{d}) /Kids [{next} 0 R] >>")); + } + objs.push(format!("<< /T (shared) /Kids [{y} 0 R] >>")); + objs.push(format!("<< /FT /Tx /T (pin) /Ff 8192 /V (deepsecret) /Kids [{w} 0 R] >>")); + objs.push("<< /Type /Annot /Subtype /Widget /Rect [50 600 150 620] /P 3 0 R >>".into()); + let mut doc = document(&objs); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD)), "the field walk reads the password field"); + assert!(has_password_flag(&doc)); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + +/// A password field below an unlisted field whose other widget is on the page: the field walk +/// adopts the whole unlisted field from that widget, so the quick check looks below it too. +#[test] +fn a_password_field_below_an_adopted_page_field_is_found() { + let mut doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [6 0 R] >>".into(), + "<< /Fields [] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (grp) /Kids [6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /Parent 5 0 R /Rect [50 700 150 720] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (pin) /Ff 8192 /V (secret8) /Parent 5 0 R /Rect [0 0 0 0] >>".into(), + ]); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD)), "the field walk adopts the password field"); + assert!(has_password_flag(&doc)); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + +/// A nameless widget listed in `/Fields` and on no page, whose `/Parent` is an unlisted password +/// field: the field walk adopts the parent, so the quick check follows the entry's `/Parent` too. +#[test] +fn a_password_parent_of_a_nameless_fields_entry_is_found() { + let mut doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R >>".into(), + "<< /Fields [6 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (pin) /FT /Tx /Ff 8192 /V (secret8) /Kids [6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /Parent 5 0 R /Rect [0 0 0 0] >>".into(), + ]); + assert!(fields(&doc).iter().any(|f| f.has(flags::PASSWORD) && f.value.iter().any(|v| v == "secret8")), "the field walk reads the parent"); + assert!(has_password_flag(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); +} + /// #4018: a value that doesn't fit its box at the field's font size is reported (it is drawn cut /// at the box). An auto-sized field shrinks to fit, and a short or empty value fits. #[test] @@ -6770,6 +6871,64 @@ fn locked_button_export_edits_leave_the_document_unchanged() { } } +/// A widget reached from `/Fields` as an unnamed kid of a plain field, whose own `/Parent` is an +/// unlisted password field: the quick check must still read that `/Parent` chain. +#[test] +fn a_page_widget_seen_under_fields_still_has_its_parent_chain_checked() { + let doc = document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [6 0 R] >>".into(), + "<< /Fields [5 0 R] /DA (/Helv 0 Tf 0 g) >>".into(), + "<< /T (grp) /Kids [6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /Parent 7 0 R /Rect [50 700 150 720] /P 3 0 R >>".into(), + "<< /FT /Tx /T (pin) /Ff 8192 /V (secret9) /Kids [6 0 R] >>".into(), + ]); + assert!(has_password_flag(&doc)); +} + +/// A radio button that is read-only on its own (its own `/Ff`) can't be picked, and a selected one +/// can't be switched away from, though the group as a whole is writable. +#[test] +fn a_read_only_radio_button_is_neither_picked_nor_left() { + let lock = |doc: &mut Document, num: u32| doc.update_dict(pdfcraft_cos::ObjRef::new(num, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + let mut doc = radio_group_form(); + lock(&mut doc, 7); // Medium + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("M".into()))), Err(FormError::ReadOnly(_)))); + set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))).expect("an unlocked button is picked"); + // Small (selected) locked: the group stays on it. + let mut doc = radio_group_form(); + lock(&mut doc, 6); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// The group's `/V` names its selection even when the button's `/AS` says nothing: a read-only +/// button that `/V` selects is not switched away from either. +#[test] +fn a_read_only_radio_button_selected_by_the_group_value_is_not_left() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| { + d.set(b"Ff".to_vec(), Object::Int(1)); + d.set(b"AS".to_vec(), Object::name("Off")); + }) + .unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// Clearing the form skips a group held by a read-only button and still clears the rest. +#[test] +fn clearing_the_form_goes_on_past_a_group_held_by_a_read_only_button() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + doc.update_dict(pdfcraft_cos::ObjRef::new(9, 0), |d| d.set(b"DV".to_vec(), pdfcraft_cos::PdfString::text("Bob"))).unwrap(); + assert_eq!(reset(&mut doc, None).unwrap(), 1); + let all = fields(&doc); + assert_eq!(field(&all, "size").value, ["S"]); + assert_eq!(field(&all, "name").value, ["Bob"]); +} + #[test] /// Cosmetic field updates skip calculations, while a value write still triggers the chain. fn script_cosmetic_mutations_do_not_run_the_calculation_chain() { @@ -6905,6 +7064,26 @@ fn radio_buttons_without_as_sharing_the_value_show_one_on() { assert_eq!(on_widgets(&doc, "r").0, [false, true, false]); } +/// Two buttons share the on state `Yes`; the first is on by its `/AS` and the second is read-only +/// and off. `/V` is `Yes` for both, but only the first is selected: picking another option switches +/// the first off, which is allowed, and the read-only one never changes. +#[test] +fn a_read_only_button_sharing_the_on_state_of_the_selected_one_does_not_block() { + let mut doc = radio_without_as(49152, "/Ff 1 /AS /Off"); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"AS".to_vec(), Object::name("Yes"))).unwrap(); + set_value(&mut doc, "r", &FieldValue::Radio(Some("No".into()))).expect("the writable selected button may be switched off"); + assert_eq!(on_widgets(&doc, "r").0, [false, false, true]); +} + +/// Selecting another button through `FieldValue::Widget` can't switch a read-only selected one off. +#[test] +fn a_read_only_selected_radio_button_is_not_left_by_picking_a_widget() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Widget(2)), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + /// Table 227, ReadOnly: a processor "shall not allow a user to change the value". A button whose own /// dictionary sets it can't be turned on by number. #[test] diff --git a/crates/forms/src/values.rs b/crates/forms/src/values.rs index af2c5e1aca..3600695c03 100644 --- a/crates/forms/src/values.rs +++ b/crates/forms/src/values.rs @@ -356,6 +356,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: let Some(state) = f.widgets.get(*i).and_then(|w| w.on_state.clone()) else { return invalid(format!("{:?} has no button {} with an on state", f.name, i.saturating_add(1))); }; + if f.kind == FieldKind::Radio && radio_change_blocked(f, Some(&state), Some(*i)) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, Some(&state), Some(*i)) } (FieldKind::CheckBox, v) => { @@ -406,6 +409,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: if choice.is_none() && f.has(flags::NO_TOGGLE_TO_OFF) && !f.value.is_empty() { return invalid(format!("{:?} must keep one option selected", f.name)); } + if radio_change_blocked(f, choice.as_deref(), None) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, choice.as_deref(), None) } (FieldKind::Combo | FieldKind::List, FieldValue::Choice(_) | FieldValue::Text(_) | FieldValue::ChoiceIndices(_)) => { @@ -541,17 +547,49 @@ pub(crate) fn scroll_widgets(doc: &mut Document, f: &Field, picked: &[usize], co Ok(()) } +/// The one widget of an exclusive radio group that turns on when `on` is chosen (`pick` when it has +/// that state, else the one already on, else the first); `None` when `on` is `None` or the group +/// turns every widget with that state on together. +fn chosen_widget(f: &Field, on: Option<&str>, pick: Option) -> Option { + let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); + on.filter(|_| exclusive).and_then(|c| { + let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); + let n = f.widgets.len(); + pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) + }) +} + +/// Whether setting a radio group to `on` (through widget `pick`, when the caller chose a widget) +/// would turn a read-only button on or off: such a button can't be picked, and a selected one can't +/// be switched off by picking another. Judged per widget, by the state `set_states` would give it, +/// so buttons sharing an on-state name are told apart by their `/AS`; the group's `/V` stands for +/// the selection only when no button's `/AS` says which is on. +fn radio_change_blocked(f: &Field, on: Option<&str>, pick: Option) -> bool { + let chosen = chosen_widget(f, on, pick); + let before = |i: usize, w: &Widget| { + let Some(s) = w.on_state.as_deref() else { return false }; + if f.widget_on(i) { + return true; + } + // No button sharing this on state is on by its `/AS` and the group's `/V` names it: the + // first of them holds the selection, whatever its own `/AS` says (an `Off` among them). + let sharing = |x: &Widget| x.on_state.as_deref() == Some(s); + f.value.first().map(String::as_str) == Some(s) + && !f.widgets.iter().any(|x| sharing(x) && x.state.as_deref() == Some(s)) + && f.widgets.iter().position(sharing) == Some(i) + }; + f.widgets.iter().enumerate().any(|(i, w)| { + let after = on.is_some() && w.on_state.as_deref() == on && chosen.is_none_or(|k| k == i); + w.read_only && before(i, w) != after + }) +} + /// Check boxes and radio buttons: `/V` on the field, `/AS` on each widget. The widgets whose on /// state is `on` turn on together in a check box (ISO 32000-2 §12.7.5.2.3, EXAMPLE 2) and in a /// radio group with RadiosInUnison. Without it, at most one radio button is on (Table 229, /// §12.7.5.2.3): widget `pick` when it has that state, else the one already on, else the first. fn set_states(doc: &mut Document, f: &Field, on: Option<&str>, pick: Option) -> Result<(), FormError> { - let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); - let chosen = on.filter(|_| exclusive).and_then(|c| { - let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); - let n = f.widgets.len(); - pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) - }); + let chosen = chosen_widget(f, on, pick); // `/V` names the on state, the same for every widget that has it: which of them is on is // their `/AS` alone (§12.7.5.2.4). let v = on.unwrap_or("Off"); @@ -720,29 +758,69 @@ fn password_fields(doc: &Document) -> Vec { /// again only makes the answer cautious. It reads only `/Ff`, `/Kids` and `/Parent`, so a save /// or recovery snapshot of a form without such a field doesn't build every [`Field`]. Like /// [`fields`], it looks at the form's field tree and at the widgets on the pages, which may be -/// missing from `/Fields`. Each dictionary is visited once and at most 64 levels deep. +/// missing from `/Fields`, together with the whole tree above and below them that [`fields`] +/// adopts. Each dictionary is visited once. +/// +/// It must never answer `false` for a form whose [`fields`] include a password field, or a typed +/// password would be saved. So where it can't look as far as [`fields`] may (a branch deeper than +/// the field walk's limit, which depends on the order the tree is visited in, or a `/Parent` +/// chain too long to follow), it answers `true` and the caller reads every field. pub(crate) fn has_password_flag(doc: &Document) -> bool { + const MAX_FIELD_DEPTH: usize = 64; let sets_it = |r: ObjRef| doc.get(r).as_dict().and_then(|d| field_flags(doc, d)).is_some_and(|ff| ff & flags::PASSWORD != 0); - let top = acroform(doc).and_then(|af| af.get(b"Fields").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned())).unwrap_or_default(); - let mut stack: Vec<(Object, usize)> = top.into_iter().map(|o| (o, 0)).collect(); + let kids_of = + |r: ObjRef| doc.get(r).as_dict().and_then(|d| d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned())).unwrap_or_default(); let mut seen = std::collections::HashSet::new(); - while let Some((o, depth)) = stack.pop() { - let Object::Ref(r) = o else { continue }; - if depth > 64 || !seen.insert(r) { - continue; + // Whether a dictionary below `start` sets the flag, or the walk was cut short. + let subtree = |start: Vec, seen: &mut std::collections::HashSet| { + let mut stack: Vec<(Object, usize)> = start.into_iter().map(|o| (o, 0)).collect(); + while let Some((o, depth)) = stack.pop() { + let Object::Ref(r) = o else { continue }; + if !seen.insert(r) { + continue; + } + if depth > MAX_FIELD_DEPTH || sets_it(r) { + return true; + } + stack.extend(kids_of(r).into_iter().map(|k| (k, depth + 1))); } - if sets_it(r) { + false + }; + let top = acroform(doc).and_then(|af| af.get(b"Fields").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned())).unwrap_or_default(); + let top_refs: Vec = top.iter().filter_map(Object::as_ref).collect(); + if subtree(top, &mut seen) { + return true; + } + // A `/Fields` entry may itself name a field above it through `/Parent` (one `fields` adopts when + // the entry has no name of its own), whether or not a page lists it. + for r in top_refs { + let chain = field_chain(doc, r); + if chain.len() >= MAX_FIELD_DEPTH || chain.iter().copied().any(sets_it) { + return true; + } + if subtree(vec![Object::Ref(field_root(doc, &chain, r))], &mut seen) { return true; } - let kids = doc.get(r).as_dict().and_then(|d| d.get(b"Kids").map(|k| doc.resolve(k)).and_then(|k| k.as_array().cloned())).unwrap_or_default(); - stack.extend(kids.into_iter().map(|k| (k, depth + 1))); } - // Widgets on the pages, and the fields above them. - page_refs(doc).into_iter().any(|p| { + // Widgets on the pages, the fields above them, and the fields below the topmost one. + for p in page_refs(doc) { let annots = doc.get(p).as_dict().and_then(|d| d.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned())).unwrap_or_default(); - annots.iter().filter_map(Object::as_ref).filter(|r| !seen.contains(r)).any(|r| field_chain(doc, r).into_iter().any(sets_it)) - }) + // Every widget's `/Parent` chain is read, even one already reached from `/Fields`: the + // field walk can adopt it under a different parent there. Each chain is at most + // MAX_FIELD_DEPTH long, and `subtree` returns at once for a root it has seen. + for r in annots.iter().filter_map(Object::as_ref) { + let chain = field_chain(doc, r); + if chain.len() >= MAX_FIELD_DEPTH || chain.iter().copied().any(sets_it) { + return true; + } + let root = field_root(doc, &chain, r); + if subtree(vec![Object::Ref(root)], &mut seen) { + return true; + } + } + } + false } /// Whether any text field with the Password flag holds a value in `doc`. @@ -768,6 +846,12 @@ pub fn restore_password_values(doc: &mut Document, values: &[(String, String)]) Ok(()) } +/// The topmost dictionary of `chain` (from [`field_chain`]): a `/Parent` that is not a dictionary +/// ends the field tree, so the walk below it starts at the last one that is. +fn field_root(doc: &Document, chain: &[ObjRef], r: ObjRef) -> ObjRef { + chain.iter().rev().copied().find(|c| doc.get(*c).as_dict().is_some()).unwrap_or(r) +} + /// `r` and its field ancestors (`/Parent`), nearest first, at most 64 deep and without repeats. fn field_chain(doc: &Document, r: ObjRef) -> Vec { let mut chain = vec![r]; @@ -919,10 +1003,17 @@ pub fn reset_with_formatting( // A reset is not blocked by NoToggleToOff: go through the writer directly. let mut tmp = f.clone(); tmp.flags &= !flags::NO_TOGGLE_TO_OFF; - write_value_impl(doc, &tmp, &v, scripts, true)?; - changed += 1; - if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { - redraw_fields.insert(f.obj); + match write_value_impl(doc, &tmp, &v, scripts, true) { + Ok(()) => { + changed += 1; + if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { + redraw_fields.insert(f.obj); + } + } + // a read-only button of the group holds the value: this field is left as it is, and the + // rest of the form is still cleared + Err(FormError::ReadOnly(_)) => {} + Err(e) => return Err(e), } } let calculation_runner: &mut dyn Scripts = if calculate_with_scripts { scripts } else { &mut NoScripts }; diff --git a/crates/js/src/document.rs b/crates/js/src/document.rs index f13c9434e9..cccf89a7a2 100644 --- a/crates/js/src/document.rs +++ b/crates/js/src/document.rs @@ -1,4 +1,6 @@ -use super::host::{FieldKey, arg, colour, error, function, host, read, s, set_value, strings, text, text_line, value_of, write}; +use super::host::{ + FieldKey, arg, charge, colour, error, function, host, kind_of, read, s, strings, text, text_bytes, text_line, value_for, value_of, write, +}; use super::model::{DISPLAY_HIDDEN, DISPLAY_VISIBLE, FieldType, Request}; use super::{Attribute, Context, JsArray, JsObject, JsResult, JsString, JsValue, NativeFunction, ObjectInitializer, js_string}; @@ -41,8 +43,9 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let value_get = getter!(ctx, n, |f, c| value_of(&f, c)); let value_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { - let vals = strings(&arg(args, 0), c)?; - write(c, name, |f| set_value(f, vals))?; + let vals = value_for(kind_of(c, name)?, strings(&arg(args, 0), c)?); + charge(c, name, |f| text_bytes(&f.value), text_bytes(&vals))?; + write(c, name, |f| f.value = vals)?; Ok(JsValue::undefined()) } function(ctx, NativeFunction::from_copy_closure_with_captures(set, n.clone())) @@ -104,6 +107,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let text_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.text_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.text_color = Some(v.into()))?; Ok(JsValue::undefined()) } @@ -113,6 +117,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let fill_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.fill_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.fill_color = Some(v.into()))?; Ok(JsValue::undefined()) } diff --git a/crates/js/src/formcalc/access.rs b/crates/js/src/formcalc/access.rs index 26de7ba9eb..3c323ca088 100644 --- a/crates/js/src/formcalc/access.rs +++ b/crates/js/src/formcalc/access.rs @@ -33,9 +33,12 @@ impl Interp<'_> { Ok(()) } - /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. + /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. Every node visited costs a + /// step, kept or not, and the call itself one more: a search that matches nothing still + /// costs work when it is repeated over a long list. pub(super) fn gather(&mut self, out: &mut Vec, seen: &mut HashSet, more: impl IntoIterator) -> R<()> { for i in more { + self.step()?; if seen.insert(i) { out.push(i); if out.len() > MAX_LIST { @@ -43,7 +46,7 @@ impl Interp<'_> { } } } - self.charge(1) + self.step() } /// The nodes of `v`, each once. @@ -294,8 +297,10 @@ impl Interp<'_> { } } - /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. + /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. A duplicate costs its step + /// too: `all` on a list of same-named siblings makes one for every pair. fn list_push(&mut self, out: &mut Vec, seen: &mut HashSet, v: Val) -> R<()> { + self.step()?; if let Val::Node(i) = v && !seen.insert(i) { @@ -305,7 +310,7 @@ impl Interp<'_> { if out.len() > MAX_LIST { return Err(LIST_TOO_LONG.into()); } - self.charge(1) + Ok(()) } /// Property `name` of form object `i` as FormCalc reads it: its value, attributes, diff --git a/crates/js/src/formcalc/dates.rs b/crates/js/src/formcalc/dates.rs index 4d9befeab2..f201a23525 100644 --- a/crates/js/src/formcalc/dates.rs +++ b/crates/js/src/formcalc/dates.rs @@ -56,6 +56,9 @@ const CENTURY_SPLIT: i64 = 30; const MONTHS: [&str; 12] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; +/// Characters of the longest month name: all a month token needs to read of the text. +const LONGEST_MONTH: usize = 9; + /// Parse `text` as a date by `picture` (`YYYY-MM-DD`, `MM/DD/YYYY`, `MMM D, YYYY`, …). pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { let pic: Vec = picture.chars().collect(); @@ -71,8 +74,9 @@ pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { } i += run; if c == 'M' && run >= 3 { - // Month name. - let rest: String = txt.get(j..).unwrap_or(&[]).iter().collect(); + // Month name: only as much of the text as a name can be, so a picture of many + // month tokens does not copy the rest of a long text for each of them. + let rest: String = txt.get(j..).unwrap_or(&[]).iter().take(LONGEST_MONTH).collect(); let lower = rest.to_ascii_lowercase(); let found = MONTHS .iter() diff --git a/crates/js/src/host.rs b/crates/js/src/host.rs index 6134bc4afe..86811171bc 100644 --- a/crates/js/src/host.rs +++ b/crates/js/src/host.rs @@ -27,6 +27,10 @@ pub(crate) struct Host { pub(crate) reset_limit_exceeded: bool, /// Bytes of text kept in alerts, console lines and requests, against [`xfa::MAX_PAYLOAD`]. pub(crate) kept: usize, + /// Bytes of text the script has added to the fields' values and colours, net of what it + /// replaced, against [`xfa::MAX_PAYLOAD`]: each assignment alone is capped, so without this + /// many fields could each hold a maximal array. + pub(crate) retained: usize, /// Alerts, console lines and requests the limits left out. pub(crate) dropped: usize, /// Layers a script turned on or off (`ocg.state`), by their places in [`DocInfo::layers`]. @@ -264,12 +268,42 @@ pub(crate) fn value_of(f: &FieldState, ctx: &mut Context) -> JsValue { } } -pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { - f.value = match f.kind { +/// Bytes of text in `v`, counting each string's heap size and its handle. +pub(crate) fn text_bytes(v: &[String]) -> usize { + v.iter().fold(0usize, |n, t| n.saturating_add(t.len()).saturating_add(std::mem::size_of::())) +} + +/// Charge the replacement of the text `old` selects from field `name` with `new_bytes` of text to +/// the run's retained-text budget; an assignment the budget has no room for is an error and +/// charges nothing. +pub(crate) fn charge(ctx: &Context, name: &FieldKey, old: impl FnOnce(&FieldState) -> usize, new_bytes: usize) -> JsResult<()> { + let i = field_index(ctx, name)?; + let mut h = host(ctx)?.borrow_mut(); + let freed = old(&h.fields[i]); + let total = h.retained.saturating_sub(freed).saturating_add(new_bytes); + if total > xfa::MAX_PAYLOAD { + return Err(error(format!("the fields' values and colours hold more than {} bytes of text", xfa::MAX_PAYLOAD))); + } + h.retained = total; + Ok(()) +} + +pub(crate) fn kind_of(ctx: &Context, name: &FieldKey) -> JsResult { + let i = field_index(ctx, name)?; + Ok(host(ctx)?.borrow().fields[i].kind) +} + +/// What a field of `kind` keeps of the values assigned to it. +pub(crate) fn value_for(kind: FieldType, vals: Vec) -> Vec { + match kind { FieldType::CheckBox | FieldType::RadioButton => vals.into_iter().filter(|v| v != "Off" && !v.is_empty()).take(1).collect(), FieldType::ListBox => vals, _ => vals.into_iter().take(1).collect(), - }; + } +} + +pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { + f.value = value_for(f.kind, vals); } #[cfg(test)] diff --git a/crates/js/src/run.rs b/crates/js/src/run.rs index b446b2cd77..2bbe216b3a 100644 --- a/crates/js/src/run.rs +++ b/crates/js/src/run.rs @@ -275,6 +275,7 @@ pub(crate) fn run_in_context( reset_calls: setup.reset_calls, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, layers_changed: BTreeSet::new(), })); diff --git a/crates/js/src/tests.rs b/crates/js/src/tests.rs index 224fc09283..8668311b8d 100644 --- a/crates/js/src/tests.rs +++ b/crates/js/src/tests.rs @@ -87,6 +87,17 @@ fn calculate_scripts_read_fields_and_set_the_value() { assert!(o.rc); } +/// An array of field values copies each entry: one long string listed many times is held to +/// the run's byte budget, not copied once per listing. +#[test] +fn an_array_of_values_stops_at_the_byte_budget() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < N; i++) a.push(s); getField('qty').value = a;"; + let o = go(&fill.replace('N', "600"), &Event::field("Calculate", "total", "")); + assert!(o.error.as_deref().is_some_and(|e| e.contains("byte limit exceeded")), "{:?}", o.error); + let o = go(&fill.replace('N', "400"), &Event::field("Calculate", "total", "")); + assert_eq!(o.error, None); +} + /// `event.value` reads and writes as a plain value, numbers included, and the outcome says whether /// the script assigned it, even to the value it already had. #[test] @@ -528,6 +539,7 @@ fn app_alert_titles_are_clipped_and_charged() { reset_calls: 0, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, layers_changed: Default::default(), }; @@ -1114,6 +1126,24 @@ mod xfa_model { assert_eq!(o.result.as_deref(), Some("7")); } + /// A copy of a subform costs its nodes, names and SOMs: instances of one with a long name + /// stop at the run's byte budget, well before the instance cap. + #[test] + fn instances_of_a_subform_with_long_names_stop_at_the_byte_budget() { + let mut f = form(); + let table = &mut f.children[0].children[4]; + assert_eq!(table.name, "table"); + for row in &mut table.children { + row.children.push(field(&"n".repeat(1 << 20), "", "", false)); + } + let ev = XfaEvent { activity: "click".into(), target: "form1[0].page1[0].go[0]".into(), ..Default::default() }; + let o = run_xfa("for (var i = 0; i < 100; i++) table._row.addInstance(1); table._row.count", &ev, &XfaDoc::default(), &f, Limits::default()); + assert_eq!(o.error, None, "{:?}", o.error); + let count: usize = o.result.as_deref().unwrap_or_default().parse().expect("a count"); + assert!((3..50).contains(&count), "{count} instances"); + assert_eq!(o.effects.iter().filter(|e| matches!(e, XfaEffect::AddInstance { .. })).count(), count - 2); + } + /// On screen every page is a sheet of its own: `xfa.layout.sheet` counts it from 0, as /// `absPage` does (forms show `sheet(this) + 1`), and `sheetCount` is the page count. #[test] @@ -1626,6 +1656,10 @@ mod formcalc { #[test] fn date2num_rejects_days_past_the_end_of_the_month() { + // A month name is read from the text's next few characters only, whatever follows. + let serial = calc(r#"Date2Num("2024-09-01")"#); + assert_eq!(calc(r#"Date2Num("September 1, 2024, and a note that runs on", "MMMM D, YYYY")"#), serial); + assert_eq!(calc(r#"Date2Num("Sep 1, 2024", "MMM D, YYYY")"#), serial); assert_eq!(calc(r#"Date2Num("2023-02-30")"#), "0"); assert_eq!(calc(r#"Date2Num("31/04/2024", "DD/MM/YYYY")"#), "0"); assert_eq!(calc(r#"Date2Num("2023-02-29")"#), "0"); @@ -2642,6 +2676,18 @@ fn a_plain_form_sees_the_xfa_version_but_not_installed() { assert_eq!(o.alerts, [Message::plain("old viewer")]); } +/// Arrays assigned to several fields are held to one run-wide budget, and replacing a field's +/// array frees its charge. +#[test] +fn retained_array_text_is_charged_across_fields() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < 320; i++) a.push(s);"; + let one = |tail: &str| go(&format!("{fill} {tail}"), &Event::field("Calculate", "total", "")); + let o = one("getField('qty').textColor = a; getField('qty').textColor = a; getField('zip').textColor = ['RGB'];"); + assert_eq!(o.error, None, "replacing an array frees its charge: {:?}", o.error); + let o = one("getField('qty').textColor = a; getField('zip').fillColor = a;"); + assert!(o.error.as_deref().is_some_and(|e| e.contains("hold more than")), "{:?}", o.error); +} + #[test] fn string_arrays_refuse_aggregate_text_before_request_clipping() { let o = go("resetForm(new Array(65).fill('x'.repeat(1 << 19)));", &Event::doc("Console")); diff --git a/crates/js/src/xfa/globals.rs b/crates/js/src/xfa/globals.rs index de57d7fd49..46dc8a713b 100644 --- a/crates/js/src/xfa/globals.rs +++ b/crates/js/src/xfa/globals.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use super::host::host; use super::instances::instance_manager; use super::model::{MAX_DEPTH, XfaDoc, XfaEffect, XfaEvent, clip, payload, reset_names}; @@ -379,7 +381,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current "DataView", "Intl", ]; - let mut defined: Vec = RESERVED.iter().map(ToString::to_string).collect(); + let mut defined: HashSet = RESERVED.iter().map(ToString::to_string).collect(); for scope in scopes { let kids: Vec<(usize, String, bool)> = { let h = host(ctx)?.borrow(); @@ -398,7 +400,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current let im = instance_manager(ctx, scope, &name)?; ctx.register_global_property(JsString::from(format!("_{name}").as_str()), im, rw)?; } - defined.push(name); + defined.insert(name); } } Ok(()) diff --git a/crates/js/src/xfa/instances.rs b/crates/js/src/xfa/instances.rs index 2985794dc3..c737ea9db9 100644 --- a/crates/js/src/xfa/instances.rs +++ b/crates/js/src/xfa/instances.rs @@ -137,6 +137,17 @@ fn child_som(parent_som: String, name: &str, index: usize) -> String { } } +/// Bytes a copy of `src`'s subtree keeps: a node, its name and its SOM (the copy's SOMs differ +/// in their indices only) for every node [`clone_subtree`] copies. +fn subtree_bytes(h: &XHost, src: usize, depth: usize) -> usize { + let Some(n) = h.nodes.get(src).filter(|_| depth <= MAX_DEPTH) else { return 0 }; + let own = std::mem::size_of::().saturating_add(n.name.len()).saturating_add(n.som.len()); + n.children + .iter() + .filter(|&&k| !h.nodes.get(k).is_some_and(|c| c.gone || c.repeatable && c.index > 0)) + .fold(own, |sum, &k| sum.saturating_add(subtree_bytes(h, k, depth + 1))) +} + /// A copy of `src`'s subtree (values cleared) under `parent`, returning the copy's index. fn clone_subtree(h: &mut XHost, src: usize, parent: Option, depth: usize) -> Option { if depth > MAX_DEPTH || h.nodes.len() >= MAX_NODES { @@ -214,7 +225,10 @@ impl XHost { let name = h.nodes.get(first).map(|n| n.name.to_string()).unwrap_or_default(); let parent_som = h.nodes.get(parent).map(|p| p.som.to_string()).unwrap_or_default(); let som = child_som(parent_som, &name, live.len()); - if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len()) { + // The copy's nodes, their names and SOMs, are charged to the run's byte budget before + // they are made: a subtree with long names must not be copied past it. + let copied = subtree_bytes(h, first, 0); + if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len().saturating_add(copied)) { return None; } let new = clone_subtree(h, first, Some(parent), 0)?; diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 08fb23d22b..2eae5e8656 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -32,10 +32,15 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going damaged partway, which may draw an image larger past the damage), no image is resampled. Colour and grayscale images above a threshold are resampled with a bicubic (Catmull-Rom) filter to a target resolution and stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A - stencil `/Mask` has its own resolution, so the picture changes under it and it stays as it is. - An image is replaced only if the result is smaller. Images over 80 million pixels are left - alone by the image steps, here and below, whatever their settings (memory); Clean Up may still - Flate an unfiltered or LZW one. Left alone by these colour and grey + stencil `/Mask` has its own resolution, so the picture changes under it and it keeps its size and + its bits (stored again as a picture's mask, below). + An image is replaced only if the result is smaller. Images over 80 million pixels are left alone + by the image steps, here and below, whatever their settings (memory), except that the monochrome + step below takes an image already of one bit a pixel, a mask included, while its rows hold no more + bytes than 80 million pixels of eight bits (each row starts on a new byte: a byte for each 8 + samples and one for any left over): up to 640 million pixels when it is wide, and a strip one + sample wide no taller than one of eight bits. Clean Up may still Flate an unfiltered or LZW one. + Left alone by these colour and grey settings: CMYK and other colour spaces, image masks, colour-key masks, decode arrays, more than 8 bits, JPEG 2000, JBIG2 and CCITT (black-and-white images, CCITT and JBIG2 among them, are below). A JPEG is read with its colour transform (an Adobe marker's, else `/ColorTransform`, ISO 32000-2 §7.4.8), so one @@ -108,10 +113,19 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going - Apart from the downsampling it is lossless: the samples keep their meaning, so the colour space, decode array and `/Mask` stay, but for packed RGB, which becomes grey with one decode pair. Group 4 wins on scans, whose ragged edges defeat Flate; on clean computer-drawn 1-bit - art Flate can be smaller. Not yet: inline images, an image's own `/SMask` or `/Mask` - image (only images drawn on a page are visited), Group 4 for rows wider than 32,767 - samples, and JBIG2 past 65,535 samples on a side, where the decoder stops (its 2^28 in all - is past the 80 million pixels above). + art Flate can be smaller. + - **A picture's masks:** after the pictures, each picture's stencil `/Mask` and one-bit + `/SMask` (a picture's own images, which the pages don't draw by name), once however many + pictures name it, is stored again as the compression asks when that is smaller, at its size: + its bits stay as they are, so it masks as it did. A stencil the pages also draw by name is one + of the images above; a soft mask they draw is still stored as a mask. It isn't resampled, as its resolution isn't measured (the walk measures the + picture's), and it shares no JBIG2 dictionary, as only images named directly do. A soft mask of + more bits stays as it is, to be resampled with its picture, which wants it at eight. The report + counts masks as images. + - Not yet: inline images, a picture's mask resampled, or packed to one bit from more, Group 4 + for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side or 2^28 in all, + where the decoder stops: with JBIG2 chosen, a one-bit image larger than that stays unless + it is averaged. - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller, a dictionary shared by up to 16 images when that is smaller still. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 diff --git a/crates/optimize/src/audit.rs b/crates/optimize/src/audit.rs index 789bf2e89d..b06d76d6c7 100644 --- a/crates/optimize/src/audit.rs +++ b/crates/optimize/src/audit.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; -use pdfcraft_cos::{Dict, Document, MAX_DECODED, Object, XrefEntry}; +use pdfcraft_cos::{Dict, Document, MAX_DECODED, ObjRef, Object, XrefEntry}; /// The categories, in Acrobat's order. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -91,6 +91,9 @@ const MAX_FORM_DEPTH: usize = 64; struct Audit<'a> { doc: &'a Document, owner: HashMap, + /// Resource dictionaries (by reference) already walked: one shared by many pages, forms or + /// appearances is walked once, as claiming is the same each time. + walked: HashSet, } impl Audit<'_> { @@ -124,6 +127,18 @@ impl Audit<'_> { o.map(|x| self.doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default() } + /// The resource dictionary `holder` points at (if it does), by kind: [`Self::resources`], + /// once per dictionary given by reference. + fn resources_of(&mut self, holder: Option<&Object>, depth: usize) { + if let Some(Object::Ref(r)) = holder + && !self.walked.insert(*r) + { + return; + } + let res = self.resolve_dict(holder); + self.resources(&res, depth); + } + /// A resource dictionary, by kind; `depth` counts the form XObjects it is nested in. fn resources(&mut self, res: &Dict, depth: usize) { let xobjects = self.resolve_dict(res.get(b"XObject")); @@ -135,8 +150,7 @@ impl Audit<'_> { // A form: its own resources first, by kind, then the rest of it. Owning it // first stops a form that draws itself. self.owner.insert(r.num, SpaceCategory::XObjectForms); - let inner = self.resolve_dict(s.dict.get(b"Resources")); - self.resources(&inner, depth + 1); + self.resources_of(s.dict.get(b"Resources"), depth + 1); self.claim(&Object::Dict(s.dict.clone()), SpaceCategory::XObjectForms); } _ => self.claim(x, SpaceCategory::XObjectForms), @@ -172,8 +186,7 @@ impl Audit<'_> { _ => Vec::new(), }; for dict in streams { - let res = self.resolve_dict(dict.get(b"Resources")); - self.resources(&res, 1); + self.resources_of(dict.get(b"Resources"), 1); } } } @@ -216,19 +229,17 @@ fn object_stream(doc: &Document, num: u32) -> Option { /// Audit the space a document uses; `file_len` is the size of the file on disk (the rest is /// overhead). pub fn audit_space(doc: &Document, file_len: u64) -> Vec { - let mut a = Audit { doc, owner: HashMap::new() }; + let mut a = Audit { doc, owner: HashMap::new(), walked: HashSet::new() }; let cat = doc.root().and_then(|r| doc.get(r).as_dict().cloned()).unwrap_or_default(); let pages = pdfcraft_model::pages(doc); // Order matters: the most specific owners first. Resources by kind before anything that // uses them: a content stream's dictionary may point at what it draws (PdfCraft's /PCAdded // records an added image), and form fields and comments draw with fonts and images too. for p in &pages { - let res = a.resolve_dict(p.dict.get(b"Resources")); - a.resources(&res, 0); + a.resources_of(p.dict.get(b"Resources"), 0); } let form = a.resolve_dict(cat.get(b"AcroForm")); - let form_resources = a.resolve_dict(form.get(b"DR")); - a.resources(&form_resources, 0); + a.resources_of(form.get(b"DR"), 0); for p in &pages { for x in annotations(doc, &p.dict) { a.appearance_resources(&x); diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index aa52c20750..9ccca4791f 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -30,17 +30,21 @@ pub const MIN_FRAGMENTS: usize = 4; /// Most pixels a merged image may have (memory). const MAX_PIXELS: u64 = 80_000_000; -/// Fragment groups a page's search for runs may read per content operator, and at least. A run -/// that is not one picture is searched again from each of its groups, so without a limit images -/// drawn over one another take time quadratic in their number; past it, the rest of the page is -/// left as it is. -const SCAN_PER_OP: usize = 16; +/// Content operators a page's search for runs may read per content operator of the page, and at +/// least. A run that is not one picture is searched again from each of its groups, so without a +/// limit images drawn over one another take time quadratic in their number; past it, the rest of +/// the page is left as it is. Every operator of a group counts (its `cm` operators are composed +/// each time), not just the group; a group of four operators costs four, so the allowance is 16 +/// groups per operator, as before, and groups with many `cm` operators pay for them. +const SCAN_PER_OP: usize = 64; const MIN_SCAN: usize = 1 << 16; #[cfg(test)] thread_local! { /// Fragment groups read, for the test that keeps a page's search linear in its content. static GROUP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Operators group reads walked, for the test that keeps the search linear in operators. + static OP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } /// One fragment: its image, its box in the coordinates the run is drawn in, and its size. @@ -53,21 +57,44 @@ struct Fragment { /// The ops `q`, one or more `cm`, `Do`, `Q` starting at `i`: the index after them, the matrix /// and the name drawn. -fn group_at(ops: &[Op], i: usize) -> Option<(usize, Matrix, Vec)> { +fn group_at(ops: &[Op], i: usize, reads: &std::cell::Cell) -> Option<(usize, Matrix, Vec)> { #[cfg(test)] GROUP_READS.with(|c| c.set(c.get().saturating_add(1))); - if !ops.get(i)?.is("q") { + let mut looked = 0usize; + let found = group_scan(ops, i, &mut looked); + // Every operator looked at is charged to the page's search allowance, whether or not the + // group turns out to be one. + reads.set(reads.get().saturating_add(looked)); + #[cfg(test)] + OP_READS.with(|c| c.set(c.get().saturating_add(looked))); + found +} + +/// [`group_at`]'s search, counting in `looked` each operator it examines. +fn group_scan(ops: &[Op], i: usize, looked: &mut usize) -> Option<(usize, Matrix, Vec)> { + let q = ops.get(i)?; + *looked += 1; + if !q.is("q") { return None; } let mut j = i + 1; let mut m = Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]); while let Some(op) = ops.get(j).filter(|o| o.is("cm")) { + *looked += 1; m = Matrix::from_operands(&op.operands)?.then(&m); j += 1; } - let draw = ops.get(j).filter(|o| o.is("Do"))?; + let draw = ops.get(j)?; + *looked += 1; + if !draw.is("Do") { + return None; + } let name = draw.name(0)?.to_vec(); - ops.get(j + 1).filter(|o| o.is("Q"))?; + let end = ops.get(j + 1)?; + *looked += 1; + if !end.is("Q") { + return None; + } (j > i + 1).then_some((j + 2, m, name)) } @@ -152,7 +179,7 @@ fn stacked(run: &mut [Fragment]) -> Option>> { /// The fragment groups from `i` on (at most `limit`) while they form a picture drawn row by row /// or strip by strip in one direction, up to a gap or a jump. -fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document) -> Vec { +fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document, reads: &std::cell::Cell) -> Vec { let mut run: Vec = Vec::new(); let mut row_x = [0.0f64, 0.0]; let mut cur_x = [0.0f64, 0.0]; @@ -163,7 +190,7 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & let mut complete = 0usize; let mut j = i; while run.len() < limit { - let Some((next, m, name)) = group_at(ops, j) else { break }; + let Some((next, m, name)) = group_at(ops, j, reads) else { break }; let Some((image, width, height, ..)) = fragment_image(doc, xobjects, &name) else { break }; let [a, _, _, d, e, f] = m.0; let f2 = Fragment { image, rect: [e, f, e + a, f + d], width, height }; @@ -240,10 +267,10 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & } /// The op index after the first `count` fragment groups from `i`. -fn run_end(ops: &[Op], i: usize, count: usize) -> usize { +fn run_end(ops: &[Op], i: usize, count: usize, reads: &std::cell::Cell) -> usize { let mut j = i; for _ in 0..count { - match group_at(ops, j) { + match group_at(ops, j, reads) { Some((next, ..)) => j = next, None => break, } @@ -386,9 +413,9 @@ fn merge_page( let mut out: Vec = Vec::with_capacity(ops.len()); let (mut i, mut merged, mut k) = (0, 0, 0); let budget = ops.len().saturating_mul(SCAN_PER_OP).max(MIN_SCAN); - let mut scanned = 0usize; + let reads = std::cell::Cell::new(0usize); while i < ops.len() { - if scanned > budget { + if reads.get() > budget { out.extend(ops.get(i..).unwrap_or_default().iter().cloned()); break; } @@ -396,7 +423,7 @@ fn merge_page( let mut run = Vec::new(); let mut kind: Option<(Object, usize)> = None; let mut j = i; - while let Some((next, m, name)) = group_at(&ops, j) { + while let Some((next, m, name)) = group_at(&ops, j, &reads) { let Some((image, width, height, cs, n)) = fragment_image(doc, &xobjects, &name) else { break }; let [a, b, c, d, e, f] = m.0; if b.abs() > 1e-9 || c.abs() > 1e-9 || !(a > 0.0 && d > 0.0) { @@ -410,13 +437,11 @@ fn merge_page( run.push(Fragment { image, rect: [e, f, e + a, f + d], width, height }); j = next; } - scanned = scanned.saturating_add(run.len()).saturating_add(1); // Not one picture as a whole: a prefix drawn row by row or strip by strip may be. let mut rows = if run.len() >= MIN_FRAGMENTS { stacked(&mut run) } else { None }; if run.len() >= MIN_FRAGMENTS && rows.is_none() { - run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc); - j = run_end(&ops, i, run.len()); - scanned = scanned.saturating_add(run.len().saturating_mul(2)).saturating_add(2); + run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc, &reads); + j = run_end(&ops, i, run.len(), &reads); rows = stacked(&mut run); } let Some(((cs, n), rows)) = kind.zip(rows).filter(|_| run.len() >= MIN_FRAGMENTS) else { @@ -773,4 +798,37 @@ mod tests { assert_eq!((merged, report.merged_images), (0, 0)); } } + + /// The same with many `cm` operators in each group: the search reads a bounded number of operators + /// for each one of the page, however the matrices are spread over the groups. + #[test] + fn groups_with_many_matrices_are_charged_for_each_operator() { + let mut doc = Document::new_empty(); + let mut image = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(1)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + image.set(k.as_bytes().to_vec(), v); + } + image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let image = doc.add(Object::Stream(Stream::from_raw(image, vec![0]))); + let groups = 400; + let content = doc.add(Object::Stream(Stream::flate( + Dict::new(), + format!("q 10 0 0 10 0 0 cm {}/Im Do Q\n", "1 0 0 1 0 0 cm ".repeat(49)).repeat(groups).as_bytes(), + ))); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Contents".to_vec(), Object::Ref(content)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + let page = doc.add(Object::Dict(page)); + OP_READS.with(|c| c.set(0)); + let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true, &mut ImageBudget::with_work(u64::MAX)); + assert_eq!(merged.ok(), Some(0)); + let reads = OP_READS.with(|c| c.get()); + assert!(reads <= 80 * 53 * groups, "{reads} operator reads"); + } } diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 7bfa44dafd..4037de82f3 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -13,6 +13,15 @@ use crate::{Compression, ImageSettings, MonoCompression, MonoSettings, OptimizeE /// Images with more pixels than this are left alone (memory). pub(crate) const MAX_PIXELS: u64 = 80_000_000; +/// The bytes an image of one bit a pixel holds as stored: each row starts on a new byte, a byte for +/// each 8 samples and one for any left over. The monochrome step takes such an image while these +/// are no more than [`MAX_PIXELS`], as many as that many pixels of eight bits hold: a wide one up +/// to 640 million pixels (an A0 page at 600 ppi is about 560 million), where at eight bits a pixel +/// it would be left alone, and a strip one sample wide no taller than one of eight bits. +fn one_bit_bytes(w: u64, h: u64) -> u64 { + w.div_ceil(8).saturating_mul(h) +} + /// The resources of a page, inherited from the page tree if the page has none. pub(crate) fn page_resources(doc: &Document, page: ObjRef) -> Option { held_page_resources(doc, page).map(|(resources, _)| resources) @@ -829,6 +838,21 @@ fn one_component(doc: &Document, cs: &Object) -> bool { } } +/// What the filters before a fax image's decoder may produce, its code: at most about a byte a +/// pixel and each row's end, as redaction allows, but no more than `MAX_PIXELS` bytes for its +/// pixels (and four a row), so a large page's code can't cost a byte a pixel. +fn fax_code_allowance(w: usize, h: usize) -> Option { + w.checked_mul(h)?.min(usize::try_from(MAX_PIXELS).ok()?).checked_add(h.checked_mul(4)?)?.checked_add(4096) +} + +/// Whether `d` is an image of one bit a pixel as stored: an image mask, or one bit in one colour +/// component. +fn one_bit_as_stored(doc: &Document, d: &Dict) -> bool { + let bits = d.int(b"BitsPerComponent"); + let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); + (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) +} + /// A pixel's samples as a 1 or a 0, or `None` for a colour that is neither. type Pixel = fn(&[u8]) -> Option; @@ -854,19 +878,23 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut let d = &s.dict; // Through u32 first, as everywhere here, so the product can't overflow. let (w, h) = (u32::try_from(d.int(b"Width")?).ok()?, u32::try_from(d.int(b"Height")?).ok()?); - if w == 0 || h == 0 || u64::from(w) * u64::from(h) > MAX_PIXELS { + // An image already of one bit a pixel is taken while its rows hold no more than `MAX_PIXELS` + // bytes (`one_bit_bytes`), one packed from more bits (2, 4 or 8) up to `MAX_PIXELS` pixels, as + // before: its samples are decoded as stored first, eight-bit RGB at three bytes a pixel. + let as_stored = one_bit_as_stored(doc, d); + let held = if as_stored { one_bit_bytes(u64::from(w), u64::from(h)) } else { u64::from(w) * u64::from(h) }; + if w == 0 || h == 0 || held > MAX_PIXELS { return None; } let (wu, hu) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); let stride = wu.div_ceil(8); let bits = d.int(b"BitsPerComponent"); let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); - if (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) { - // No further than its rows can need: a fax image's code (the filters before its decoder - // produce it) takes at most about a byte a pixel and each row's end, as redaction allows. + if as_stored { + // No further than its rows can need. let len = stride.checked_mul(hu)?; let max = match s.filters().last() { - Some((pdfcraft_filters::Filter::CcittFax, _)) => wu.checked_mul(hu)?.checked_add(hu.checked_mul(4)?)?.checked_add(4096)?, + Some((pdfcraft_filters::Filter::CcittFax, _)) => fax_code_allowance(wu, hu)?, // A JBIG2 page, which has to be the image's size, is `len` bytes as rows of a bit. _ => allowance(len, h)?, }; @@ -937,6 +965,7 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut _ => return None, }; let row_len = wu.checked_mul(n)?.checked_mul(bpc)?.div_ceil(8); + count_decoded(u64::from(w) * u64::from(h)); // Through its parameters given by reference too: without a PNG predictor's tag bytes taken // off, rows of 0 and 255 (or palette entries 0 and 1) would still pass below, each shifted. let samples = budget.decode_filters(doc, s, allowance(row_len.checked_mul(hu)?, h)?).ok()?; @@ -1364,9 +1393,9 @@ pub(crate) fn run( .collect(); let mut refs: Vec<(ObjRef, f64)> = ppi.into_iter().filter(|(r, _)| !masks.contains(r)).collect(); refs.sort_by_key(|(r, _)| (r.num, r.generation)); - let total = refs.len(); - let mut jbig2 = false; - let mut group: Vec = Vec::new(); + // Then the pictures' masks of one bit a sample, each once ([`decide_mask`]). + let mask_refs = picture_masks(doc, &refs); + let total = refs.len() + mask_refs.len(); // By number: a repaired object answers to two generations. let mut smask_uses: HashMap = HashMap::new(); for (r, _) in &refs { @@ -1374,23 +1403,96 @@ pub(crate) fn run( *smask_uses.entry(m.num).or_default() += 1; } } + let holds = settings.mono.compression == MonoCompression::Jbig2; + let threads = threads_for_items(doc, &refs, settings); + let mut out = Out { doc, report, budget, jbig2: false, group: Vec::new() }; + let pictures = Items { items: &refs, threads, first: 0, total }; + work_through( + &mut out, + &pictures, + &|doc, rest| batch_len(doc, rest, &smask_uses, holds), + &|doc, r, ppi, budget| decide(doc, r, ppi, defaults.contains(&r.num), settings, budget), + progress, + )?; + share(out.doc, &mut out.group, out.report); + let threads = threads_for_items(out.doc, &mask_refs, settings); + let masks = Items { items: &mask_refs, threads, first: refs.len(), total }; + // Each mask is an object of its own (by number), which only its own decision changes. + work_through( + &mut out, + &masks, + &|_, rest| rest.len().min(PICTURES_AT_ONCE), + &|doc, r, _, budget| decide_mask(doc, r, settings, budget), + progress, + )?; + if !progress(Stage::Images { done: total, total }) { + return Err(OptimizeError::Cancelled); + } + if out.jbig2 { + at_least_pdf_1_4(out.doc)?; + } + Ok(()) +} + +/// What the image pass changes and counts as it goes: the document, the report, the run's +/// allowance, whether it stored an image as JBIG2, and the JBIG2 images held to share a dictionary +/// ([`share`]). +struct Out<'a> { + doc: &'a mut Document, + report: &'a mut Report, + budget: &'a mut ImageBudget, + jbig2: bool, + group: Vec, +} + +/// Images the pass works through, each with its effective resolution, on up to `threads` threads; +/// for progress, the first of them is number `first` of the pass's `total`. +struct Items<'a> { + items: &'a [(ObjRef, f64)], + threads: usize, + first: usize, + total: usize, +} + +/// How many of `items` to work on at once: one below [`MIN_FOR_THREADS`], else as many as the +/// workers, and as [`PICTURE_MEMORY`] holds of the largest. +fn threads_for_items(doc: &Document, items: &[(ObjRef, f64)], settings: &Settings) -> usize { // Too few to be worth the threads, which keeps a small file's progress in step too. - let threads = if total < MIN_FOR_THREADS { - 1 - } else { - let largest = refs.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); - threads_for(crate::pool::workers(settings.workers), largest) - }; + if items.len() < MIN_FOR_THREADS { + return 1; + } + let largest = items.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); + threads_for(crate::pool::workers(settings.workers), largest) +} + +/// How [`work_through`] works out what's to become of an image, at its effective resolution, within +/// an allowance; threads share it. +type Decide<'a> = dyn Fn(&Document, ObjRef, f64, &mut ImageBudget) -> Decision + Sync + 'a; + +/// How many of the images left, from the first, [`work_through`] may work out beside each other. +type Batch<'a> = dyn Fn(&Document, &[(ObjRef, f64)]) -> usize + 'a; + +/// Works out what's to become of each of `items` by `decide`, and does it, in their order: a batch +/// of them beside each other where `batch` says they can be, one at a time otherwise. +fn work_through( + out: &mut Out<'_>, + items: &Items<'_>, + batch: &Batch<'_>, + decide: &Decide<'_>, + progress: &mut dyn FnMut(Stage) -> bool, +) -> Result<(), OptimizeError> { + let (threads, total) = (items.threads, items.total); let mut done = 0; - while let Some(rest) = refs.get(done..).filter(|rest| !rest.is_empty()) { - let run = if threads > 1 { batch_len(doc, rest, &smask_uses, settings.mono.compression == MonoCompression::Jbig2) } else { 0 }; + while let Some(rest) = items.items.get(done..).filter(|rest| !rest.is_empty()) { + let at = items.first.saturating_add(done); + let run = if threads > 1 { batch(out.doc, rest) } else { 0 }; let Some(batch) = rest.get(..run).filter(|batch| batch.len() > 1) else { - if !progress(Stage::Images { done, total }) { + if !progress(Stage::Images { done: at, total }) { return Err(OptimizeError::Cancelled); } if let Some((r, ppi)) = rest.first() { - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget); - apply(doc, decision, report, &mut jbig2, &mut group); + let decision = decide(out.doc, *r, *ppi, out.budget); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += 1; continue; @@ -1399,10 +1501,10 @@ pub(crate) fn run( // order below: a picture the order can't pay for is done again as if there were no threads, // so the same pictures are refused with any number of them. Once more than what's left has // been used, no more are started. - let allowed = budget.work_left(); + let allowed = out.budget.work_left(); let used = AtomicU64::new(0); let tried = { - let (doc, defaults) = (&*doc, &defaults); + let doc = &*out.doc; crate::pool::map( batch, threads, @@ -1411,36 +1513,74 @@ pub(crate) fn run( return None; } let mut own = ImageBudget::with_work(allowed); - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, &mut own); + let decision = decide(doc, *r, *ppi, &mut own); let cost = allowed.saturating_sub(own.work_left()); used.fetch_add(cost, Ordering::Relaxed); (own.refused() == 0).then_some((decision, cost)) }, - &mut |finished| progress(Stage::Images { done: done.saturating_add(finished), total }), + &mut |finished| progress(Stage::Images { done: at.saturating_add(finished), total }), )? }; for ((r, ppi), tried) in batch.iter().zip(tried) { let decision = match tried.flatten() { - Some((decision, cost)) if cost <= budget.work_left() => { - budget.charge(cost); + Some((decision, cost)) if cost <= out.budget.work_left() => { + out.budget.charge(cost); decision } - _ => decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget), + _ => decide(out.doc, *r, *ppi, out.budget), }; - apply(doc, decision, report, &mut jbig2, &mut group); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += batch.len(); } - share(doc, &mut group, report); - if !progress(Stage::Images { done: total, total }) { - return Err(OptimizeError::Cancelled); - } - if jbig2 { - at_least_pdf_1_4(doc)?; - } Ok(()) } +/// The masks of `pictures` the pass stores again after them, each once (by number): their soft +/// masks and their stencil masks, unless one is drawn as a picture of its own, and so is one of +/// them. A colour-key `/Mask` is an array, not an image. Their resolution isn't measured. +fn picture_masks(doc: &Document, pictures: &[(ObjRef, f64)]) -> Vec<(ObjRef, f64)> { + let drawn: HashSet = pictures.iter().map(|(r, _)| r.num).collect(); + let mut seen = HashSet::new(); + let mut masks = Vec::new(); + for (r, _) in pictures { + let object = doc.get(*r); + let Object::Stream(s) = &*object else { continue }; + for key in [&b"SMask"[..], b"Mask"] { + if let Some(m) = s.dict.reference(key) + && !drawn.contains(&m.num) + && seen.insert(m.num) + { + masks.push((m, f64::NAN)); + } + } + } + masks.sort_by_key(|(r, _)| (r.num, r.generation)); + masks +} + +/// Work out what's to become of mask `r` of a picture, changing nothing yet: stored again as the +/// monochrome settings ask ([`mono`]), at its size, when it has one bit a sample (a stencil mask, +/// or a soft mask of one bit) and that is smaller. The bits stay as they are, so it masks as it did. +/// It isn't resampled: its resolution isn't measured, and its edges are the picture's. A soft mask +/// of more bits is left to be resampled with its picture ([`process`]), which wants it at eight. It +/// shares no JBIG2 dictionary: those are for images named directly, which redaction can store again. +fn decide_mask(doc: &Document, r: ObjRef, settings: &Settings, budget: &mut ImageBudget) -> Decision { + let obj = doc.get(r); + let Object::Stream(s) = &*obj else { return Decision::NotAnImage }; + let stencil = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + if !stencil && s.dict.int(b"BitsPerComponent") != Some(1) { + return Decision::NotAnImage; + } + let refused = budget.refused(); + // Its resolution unmeasured, it isn't resampled. As where a default colour space is in force: + // at one bit a sample that changes nothing, and nothing is packed. + match mono(doc, s, f64::NAN, true, &settings.mono, budget) { + Some((new, _, _)) => Decision::Replace { image: r, before: s.raw.len(), new, resampled: false, smask: None, bits: None }, + None => Decision::Stays { refused: budget.refused() > refused }, + } +} + /// A picture's soft mask, if it names one. fn smask_of(doc: &Document, r: ObjRef) -> Option { match &*doc.get(r) { @@ -1494,12 +1634,21 @@ const PICTURE_MEMORY: u64 = 1 << 30; /// What a picture holds while it is worked out, for each pixel: decoded, and resampled or encoded. const PICTURE_BYTES_PER_PIXEL: u64 = 8; -/// A picture's pixels, if it is small enough to be decoded at all. +/// A picture's pixels, if it is small enough to be decoded at all. One of one bit a pixel as +/// stored is taken while the bytes its rows hold ([`one_bit_bytes`]) are no more than +/// `MAX_PIXELS`, and counts as those and one more a row: decoding allows each row's end a few +/// bytes besides its samples ([`allowance`], [`fax_code_allowance`]), which for an image a few +/// samples wide is as much again as its samples. fn picture_pixels(doc: &Document, r: ObjRef) -> Option { let object = doc.get(r); let Object::Stream(s) = &*object else { return None }; let (w, h) = (u64::try_from(s.dict.int(b"Width")?).ok()?, u64::try_from(s.dict.int(b"Height")?).ok()?); - w.checked_mul(h).filter(|pixels| *pixels <= MAX_PIXELS) + if one_bit_as_stored(doc, &s.dict) { + let bytes = one_bit_bytes(w, h); + return (bytes <= MAX_PIXELS).then(|| bytes.saturating_add(h)); + } + let pixels = w.checked_mul(h)?; + (pixels <= MAX_PIXELS).then_some(pixels) } /// How many pictures of `largest` pixels to work on at once: `workers`, or as many as @@ -1511,7 +1660,7 @@ fn threads_for(workers: usize, largest: u64) -> usize { /// What's to become of one picture. enum Decision { - /// Not an image stream after all. + /// Not an image stream after all, or a picture's mask the pass doesn't store again. NotAnImage, /// An image that stays as it is, `refused` if the allowance couldn't pay for it. Stays { refused: bool }, @@ -1750,6 +1899,83 @@ mod dimension_tests { assert_eq!(threads_for(8, 0), 8); } + /// A one-bit image counts as the bytes its rows hold and one more a row when planning how many + /// pictures run at once, while those it holds are up to `MAX_PIXELS`: about an eighth of its + /// pixels when wide, two bytes a row when a few samples wide. One of eight bits counts whole, + /// up to `MAX_PIXELS`. + #[test] + fn a_one_bit_picture_counts_as_the_bytes_of_its_rows() { + let mut doc = Document::new_empty(); + let mut add = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))) + }; + let (one_bit, past_one_bit, eight_bit) = (add(20_000, 32_000, 1), add(20_000, 32_001, 1), add(10_000, 8_001, 8)); + let (strip, past_strip, narrow) = (add(1, 80_000_000, 1), add(1, 80_000_001, 1), add(8, 10_000_000, 1)); + assert_eq!(picture_pixels(&doc, one_bit), Some(80_000_000 + 32_000)); + assert_eq!(picture_pixels(&doc, past_one_bit), None); + assert_eq!(picture_pixels(&doc, strip), Some(160_000_000), "a byte a row, and one more"); + assert_eq!(picture_pixels(&doc, past_strip), None); + assert_eq!(picture_pixels(&doc, narrow), Some(20_000_000)); + assert_eq!(threads_for(8, 20_000_000), 6); + assert_eq!(threads_for(8, 160_000_000), 1); + assert_eq!(picture_pixels(&doc, eight_bit), None); + assert_eq!(threads_for(8, 80_000_000), 1); + } + + /// A fax image's code may take about a byte a pixel and its rows' ends, but no more than + /// `MAX_PIXELS` bytes for its pixels however large the page, and four a row. + #[test] + fn a_fax_images_code_is_allowed_80_million_bytes_and_four_a_row() { + assert_eq!(fax_code_allowance(1_000, 1_000), Some(1_000_000 + 4_000 + 4_096)); + assert_eq!(fax_code_allowance(20_000, 32_000), Some(80_000_000 + 128_000 + 4_096)); + assert_eq!(fax_code_allowance(1, 80_000_000), Some(80_000_000 + 320_000_000 + 4_096)); + } + + /// The monochrome step decodes an image already of one bit a pixel while its rows hold no more + /// than `MAX_PIXELS` bytes, and one it would pack from eight bits up to `MAX_PIXELS` pixels. + /// Past those, nothing is decoded. + #[test] + fn one_bit_images_are_decoded_up_to_their_own_limit() { + let image = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + Stream::from_raw(dict, Vec::new()) + }; + let decoded = |s: &Stream| { + DECODED_PIXELS.with(|c| c.set(0)); + one_bit(&Document::new_empty(), s, true, false, &mut ImageBudget::with_work(u64::MAX)); + DECODED_PIXELS.with(|c| c.get()) + }; + assert_eq!(decoded(&image(20_000, 32_000, 1)), 640_000_000, "one bit, at the limit"); + assert_eq!(decoded(&image(20_000, 32_001, 1)), 0, "one bit, past it"); + assert_eq!(decoded(&image(1, 80_000_000, 1)), 80_000_000, "one bit a sample wide, a byte a row, at the limit"); + assert_eq!(decoded(&image(1, 80_000_001, 1)), 0, "one bit a sample wide, past it"); + assert_eq!(decoded(&image(10_000, 8_000, 8)), 80_000_000, "eight bits, at theirs"); + assert_eq!(decoded(&image(10_000, 8_001, 8)), 0, "eight bits, past theirs"); + } + + /// A zero-wide image isn't decoded: its rows hold no bytes, but its filters would be allowed + /// three a row. + #[test] + fn a_zero_wide_image_is_not_decoded() { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(0)); + dict.set(b"Height".to_vec(), Object::Int(1_000_000)); + dict.set(b"ImageMask".to_vec(), Object::Bool(true)); + let s = Stream::flate(dict, &vec![0u8; 1 << 20]); + let mut budget = ImageBudget::with_work(1 << 40); + assert!(one_bit(&Document::new_empty(), &s, true, false, &mut budget).is_none()); + assert_eq!(budget.work_left(), 1 << 40); + } + /// Only what redaction can store again without the dictionary shares one: an image mask, or /// DeviceGray named directly, of at most 2^26 pixels. #[test] @@ -1994,6 +2220,86 @@ mod dimension_tests { } } + /// A picture's mask past the run's allowance stays as it is, counted as refused, as an image + /// is; one the allowance paid for and that stays (its noise is no smaller as Group 4) isn't. The + /// progress counts the masks with the pictures. + #[test] + fn a_mask_past_the_runs_allowance_stays_and_is_counted() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let noise: Vec = (0..32 * 256) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + let page_of = |pictures: usize, masked: bool| { + let mut doc = Document::new_empty(); + let mut xobjects = Dict::new(); + let mut drawn = String::from("q 10 0 0 10 0 0 cm "); + let mut masks = Vec::new(); + for i in 0..pictures { + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(16)); + dict.set(b"Height".to_vec(), Object::Int(16)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceCMYK")); + if masked { + let mut mask = Dict::new(); + mask.set(b"Subtype".to_vec(), Object::name("Image")); + mask.set(b"ImageMask".to_vec(), Object::Bool(true)); + mask.set(b"Width".to_vec(), Object::Int(256)); + mask.set(b"Height".to_vec(), Object::Int(256)); + let mask = doc.add(Object::Stream(Stream::flate(mask, &noise))); + dict.set(b"Mask".to_vec(), Object::Ref(mask)); + masks.push(mask); + } + let picture = doc.add(Object::Stream(Stream::flate(dict, &[0; 16 * 16 * 4]))); + xobjects.set(format!("I{i}").into_bytes(), Object::Ref(picture)); + drawn.push_str(&format!("/I{i} Do ")); + } + drawn.push('Q'); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), drawn.as_bytes()))); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + (doc, page, masks) + }; + let keep = crate::ImageSettings { downsample: false, compression: crate::Compression::Retain, ..crate::ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let cost = |pictures: usize, masked: bool| { + let (mut doc, page, _) = page_of(pictures, masked); + let mut budget = ImageBudget::with_work(u64::MAX); + let mut report = Report::default(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + assert_eq!(report.images_refused, 0); + u64::MAX - budget.work_left() + }; + let mask = cost(1, true) - cost(1, false); + assert!(mask > 0); + let (mut doc, page, masks) = page_of(3, true); + let before: Vec<_> = masks.iter().map(|m| doc.get(*m)).collect(); + let mut budget = ImageBudget::with_work(cost(3, false) + 2 * mask); + let mut report = Report::default(); + let mut last = None; + let mut progress = |stage: Stage| { + if let Stage::Images { done, total } = stage { + last = Some((done, total)); + } + true + }; + run(&mut doc, &[page], &settings, &mut report, &mut progress, &mut budget).unwrap(); + assert_eq!((report.images, report.images_refused), (6, 1), "{report:?}"); + assert_eq!(last, Some((6, 6)), "three pictures and their three masks"); + assert!(masks.iter().zip(&before).all(|(m, b)| doc.get(*m) == *b), "every mask stays as it was"); + } + /// An image the allowance ran out on between being packed and being resampled is rewritten, /// as the packed one, so it isn't one that stayed as it was. #[test] diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 750f641a07..4aed8b0786 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -15,7 +15,8 @@ //! that use only palette entries 0 and 1, packed to one bit) above a threshold are averaged down to a //! target resolution when grey, image masks or indexed, halftones and dithers aside (see the README), and //! are stored as CCITT Group 4 or lossless JBIG2, or 1-bit -//! Flate once resampled or packed, when that is smaller. +//! Flate once resampled or packed, when that is smaller. So are pictures' stencil and one-bit soft +//! masks, at their size. //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings, form submission, import and reset actions. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted @@ -159,7 +160,8 @@ pub enum MonoCompression { /// How black-and-white images (1-bit, CCITT fax among them; and eight-bit grey and RGB of only black /// and white, and indexed images that use only palette entries 0 and 1, packed to one bit), and -/// image masks, are resampled and stored. +/// image masks, are resampled and stored. Pictures' stencil and one-bit soft masks are stored the +/// same way, at their size. #[derive(Clone, Copy, Debug, PartialEq)] pub struct MonoSettings { /// Average images whose effective resolution is above `above_ppi` down to `target_ppi`. @@ -206,7 +208,7 @@ pub struct Settings { pub merge_fragments: bool, pub color: ImageSettings, pub gray: ImageSettings, - /// Black-and-white images, and image masks. + /// Black-and-white images, image masks, and pictures' one-bit masks. pub mono: MonoSettings, pub discard_thumbnails: bool, pub discard_alternate_images: bool, @@ -314,6 +316,7 @@ pub struct Report { /// Images merged from fragments, and how many fragments went into them. pub merged_images: usize, pub fragments_merged: usize, + /// Images the image step read: those the pages draw, and their pictures' one-bit masks. pub images: usize, pub images_resampled: usize, pub images_recompressed: usize, diff --git a/crates/optimize/src/links.rs b/crates/optimize/src/links.rs index 5b74ad7023..bef69c43b7 100644 --- a/crates/optimize/src/links.rs +++ b/crates/optimize/src/links.rs @@ -52,8 +52,12 @@ fn tree_entries(doc: &Document, node: &Object) -> Vec<(Vec, Object)> { } } } - if let Some(kids) = d.get(b"Kids").map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) { - queue.extend(kids.into_iter().map(|k| (k, depth + 1))); + // A kids array shared by reference is read once, and only kids not yet seen are queued: + // nodes sharing one array can't multiply the queue. + if let Some(kids) = + d.get(b"Kids").filter(|k| !matches!(k, Object::Ref(r) if !seen.insert(*r))).map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) + { + queue.extend(kids.into_iter().filter(|k| !matches!(k, Object::Ref(r) if seen.contains(r))).map(|k| (k, depth + 1))); } } out diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7ab25262f8..eee2e9570c 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -423,8 +423,10 @@ fn black_and_white_images_become_ccitt_group_4() { /// Which 1-bit images qualify for Group 4: those of one colour component, an ICC profile of one /// component and a DeviceN of one colourant among them, but not one of three components or two /// colourants (more than a bit a pixel); image masks of one bit, as `/BitsPerComponent 1` says or -/// by default, but not one that says 8; and images of up to 80 million pixels, not one row more. -/// The byte totals count exactly the images stored again. +/// by default, but not one that says 8; and images past 80 million pixels too, where an image of +/// eight bits would be left alone (`one_bit_images_are_decoded_up_to_their_own_limit` checks the +/// bytes of rows a one-bit image may hold without decoding so much). The byte totals count exactly +/// the images stored again. #[test] fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mut doc = Document::new_empty(); @@ -457,7 +459,7 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mask = |bits: i64| [("ImageMask", Object::Bool(true)), ("BitsPerComponent", bits.into())]; let mask_1 = mono_image(&mut doc, w, h, &text, None, &mask(1)); let mask_8 = mono_image(&mut doc, w, h, &bilevel(8 * w, h, false), None, &mask(8)); - // 10,000 × 8,000 is the limit itself. + // 10,000 × 8,000 is the limit of an image of eight bits; one of one bit goes past it. let grey = one_bit(Object::name("DeviceGray")); let at_limit = mono_image(&mut doc, 10_000, 8_000, &bilevel(10_000, 8_000, false), None, &grey); let past_limit = mono_image(&mut doc, 10_000, 8_001, &bilevel(10_000, 8_001, false), None, &grey); @@ -474,8 +476,8 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { }) .collect(); page(&mut doc, &names.iter().map(String::as_str).zip(all).collect::>(), &content); - let converted = [icc_1, spot, mask_1, at_limit]; - let left = [icc_3, two_spots, mask_8, past_limit]; + let converted = [icc_1, spot, mask_1, at_limit, past_limit]; + let left = [icc_3, two_spots, mask_8]; let before = |doc: &Document, refs: &[ObjRef]| refs.iter().map(|r| stream(doc, *r).raw).collect::>(); let (old, untouched) = (before(&doc, &converted), before(&doc, &left)); let r = optimize(&mut doc, &Settings::default()).unwrap(); @@ -4925,15 +4927,22 @@ fn indexed_thin_rules_survive_averaging_for_both_palette_orders_and_decodes() { } } -/// An image past what the JBIG2 decoder takes on a side stays as it is: JBIG2 can state the size, -/// but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it drew nothing here. At 65,535 -/// on a side, the same bars do become JBIG2, so the fixture does reach that choice, and the coding -/// stored (symbols, here) reads back through that decoder. +/// An image past what the JBIG2 decoder takes on a side, or past 2^28 pixels in all, stays as it +/// is: JBIG2 can state the size, but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it +/// drew nothing here. At 65,535 on a side, and at 2^28 in all (past 80 million, which the +/// monochrome step takes of a one-bit image), the same bars do become JBIG2, so the fixture does +/// reach that choice, and the coding stored reads back through that decoder. Averaged, an image past +/// 2^28 becomes small enough and is stored as JBIG2 too. #[test] fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { - for (w, h, stored) in - [(65_535u32, 16u32, &b"JBIG2Decode"[..]), (16, 65_535, b"JBIG2Decode"), (65_536, 16, b"FlateDecode"), (16, 65_536, b"FlateDecode")] - { + for (w, h, stored) in [ + (65_535u32, 16u32, &b"JBIG2Decode"[..]), + (16, 65_535, b"JBIG2Decode"), + (65_536, 16, b"FlateDecode"), + (16, 65_536, b"FlateDecode"), + (16_384, 16_384, b"JBIG2Decode"), + (16_384, 16_385, b"FlateDecode"), + ] { let mut doc = Document::new_empty(); let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); page(&mut doc, &[("Im", image)], &format!("q {w} 0 0 {h} 0 0 cm /Im Do Q")); @@ -4958,6 +4967,15 @@ fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { assert!(decoded_image(&doc, &s).unwrap() == bits, "{w} × {h} reads back"); } } + let (w, h) = (16_384u32, 16_385u32); + let mut doc = Document::new_empty(); + let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("Im", image)], &format!("q {pw} 0 0 {ph} 0 0 cm /Im Do Q")); + let settings = Settings { mono: MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }, ..Settings::default() }; + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + assert_eq!(stream(&doc, image).dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "averaged, it is stored as JBIG2"); } /// The colour and grey steps leave an image over 80 million pixels as it is, whatever their settings @@ -5363,7 +5381,8 @@ fn stencil(doc: &mut Document, w: u32, h: u32) -> ObjRef { } /// A picture with a stencil mask is made smaller like any other: the mask has a resolution of its -/// own, so it stays exactly as it was and the hole it cuts stays where it was. +/// own, so it keeps its size and its bits (stored again as Group 4, as a picture's mask is) and the +/// hole it cuts stays where it was. #[test] fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let mut doc = Document::new_empty(); @@ -5381,7 +5400,9 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let after = stream(&doc, img); assert_eq!(after.dict.int(b"Width"), Some(300), "150 ppi"); assert_eq!(after.dict.reference(b"Mask"), Some(mask), "the mask stays on it"); - assert_eq!(stream(&doc, mask), mask_before, "and doesn't change"); + let mask_after = stream(&doc, mask); + assert_eq!((mask_after.dict.int(b"Width"), mask_after.dict.int(b"Height")), (Some(1200), Some(1200)), "the mask isn't resampled"); + assert_eq!(decoded_image(&doc, &mask_after).unwrap(), decoded_image(&doc, &mask_before).unwrap(), "and keeps its bits"); let rendered = render(&doc); assert_eq!(rendered.len(), before.len()); let changed = before.iter().zip(&rendered).filter(|(a, b)| a.abs_diff(**b) > 96).count(); @@ -5396,6 +5417,156 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { } } +/// A picture's masks of one bit a sample, which the page doesn't draw by name, are stored again as +/// the monochrome settings ask, losslessly: a stencil `/Mask` (with a decode array that inverts it) +/// and a one-bit `/SMask` become Group 4, keep their size and bits, and stay the pictures' masks; +/// the page looks the same to the pixel. A mask two pictures share is stored once, and a stencil +/// the page also draws by name is stored as that image. An eight-bit soft mask of only black and +/// white stays as it is, to be resampled with its picture. With `mono: Retain` nothing changes. +/// One-bit images past 80 million pixels are stored again, as images of eight bits that large are +/// not: their rows hold a byte for each 8 samples. A picture's stencil mask and one-bit soft mask +/// too. +#[test] +fn one_bit_images_past_80_million_pixels_are_stored_as_group_4() { + let mut doc = Document::new_empty(); + let (w, h) = (9_000, 8_900); + let text = bilevel(w, h, false); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let stencil = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let soft = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let (picture, shaded) = (masked(&mut doc, "Mask", stencil), masked(&mut doc, "SMask", soft)); + page( + &mut doc, + &[("I", one_bit), ("P", picture), ("S", shaded)], + "q 300 0 0 300 20 400 cm /I Do Q q 300 0 0 120 20 100 cm /P Do Q q 300 0 0 120 20 250 cm /S Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let mono = MonoSettings { downsample: false, ..MonoSettings::REDUCE }; + let before = [one_bit, stencil, soft].map(|r| stream(&doc, r)); + let r = optimize(&mut doc, &Settings { color: keep, gray: keep, mono, ..Settings::default() }).unwrap(); + assert_eq!(r.images_recompressed, 3, "{r:?}"); + for (r, old) in [one_bit, stencil, soft].into_iter().zip(&before) { + let new = stream(&doc, r); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{r:?}"); + assert!(new.raw.len() < old.raw.len(), "{r:?}: {} against {}", new.raw.len(), old.raw.len()); + assert_eq!(decoded_image(&doc, &new).unwrap(), text, "{r:?} keeps its bits"); + } +} + +/// With the default settings (Reduce File Size), a one-bit image past 80 million pixels drawn at +/// 600 ppi is averaged to 300 ppi, as a smaller one is: vertical stripes four samples wide. +#[test] +fn a_one_bit_image_past_80_million_pixels_is_averaged_with_the_default_settings() { + let (w, h) = (9_000u32, 8_900u32); + let mut doc = Document::new_empty(); + let stripes = vec![0x0F; (w as usize).div_ceil(8) * h as usize]; + let one_bit = mono_image(&mut doc, w, h, &stripes, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("I", one_bit)], &format!("q {pw} 0 0 {ph} 0 0 cm /I Do Q")); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + let s = stream(&doc, one_bit); + assert_eq!((s.dict.int(b"Width"), s.dict.int(b"Height")), (Some(4_500), Some(4_450))); +} + +#[test] +fn a_pictures_one_bit_masks_are_stored_again() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let decode = Object::Array(vec![1.into(), 0.into()]); + let stencil_mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true)), ("Decode", decode.clone())]); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let wide: Vec = text.iter().flat_map(|b| (0..8).rev().map(move |i| if b >> i & 1 == 1 { 255 } else { 0 })).collect(); + let eight_bit = mono_image(&mut doc, w, h, &wide, None, &[("BitsPerComponent", 8.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let drawn = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let a = masked(&mut doc, "Mask", stencil_mask); + let b = masked(&mut doc, "Mask", stencil_mask); + let c = masked(&mut doc, "SMask", one_bit); + let d = masked(&mut doc, "SMask", eight_bit); + let e = masked(&mut doc, "Mask", drawn); + page( + &mut doc, + &[("A", a), ("B", b), ("C", c), ("D", d), ("E", e), ("M", drawn)], + "q 300 0 0 120 20 660 cm /A Do Q q 300 0 0 120 320 660 cm /B Do Q q 300 0 0 120 20 520 cm /C Do Q \ + q 300 0 0 120 320 520 cm /D Do Q q 300 0 0 120 20 380 cm /E Do Q q 0 0.4 0.8 rg 300 0 0 120 320 380 cm /M Do Q", + ); + // Pictures in colour and grey are left as they are, so only the masks can change. + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let before = render(&doc); + let masks = [stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&doc, r)); + + let mut kept = doc.clone(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..settings.clone() }).unwrap(); + assert_eq!(r.images_recompressed, 0, "Retain keeps every mask as it is"); + assert_eq!([stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&kept, r)), masks); + + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_recompressed, 3, "the shared stencil once, the soft mask, the stencil drawn by name once: {r:?}"); + for (mask, old) in [(stencil_mask, &masks[0]), (one_bit, &masks[1]), (drawn, &masks[3])] { + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{mask:?}"); + assert_eq!((new.dict.int(b"Width"), new.dict.int(b"Height")), (Some(w as i64), Some(h as i64))); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, old).unwrap(), "{mask:?} keeps its bits"); + } + assert_eq!(stream(&doc, stencil_mask).dict.get(b"Decode"), Some(&decode), "the decode array stays"); + assert_eq!(stream(&doc, eight_bit), masks[2], "an eight-bit soft mask stays"); + for (picture, key, mask) in + [(a, "Mask", stencil_mask), (b, "Mask", stencil_mask), (c, "SMask", one_bit), (d, "SMask", eight_bit), (e, "Mask", drawn)] + { + assert_eq!(stream(&doc, picture).dict.reference(key.as_bytes()), Some(mask), "{picture:?} keeps its mask"); + } + assert!(render(&doc) == before, "the page looks the same"); +} + +/// Stored as JBIG2, a picture's mask is coded on its own: it shares no symbol dictionary with the +/// images the page draws by name, which do share one. +#[test] +fn a_pictures_mask_stored_as_jbig2_shares_no_dictionary() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let picture = image(&mut doc, 200, 80, 3, false, None); + let mut s = stream(&doc, picture); + s.dict.set(b"Mask".to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + let named: Vec = (0..3) + .map(|_| mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))])) + .collect(); + page( + &mut doc, + &[("P", picture), ("G0", named[0]), ("G1", named[1]), ("G2", named[2])], + "q 300 0 0 120 20 660 cm /P Do Q q 300 0 0 120 320 660 cm /G0 Do Q q 300 0 0 120 20 520 cm /G1 Do Q q 300 0 0 120 320 520 cm /G2 Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let jbig2 = MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }; + let before = render(&doc); + let old = stream(&doc, mask); + optimize(&mut doc, &Settings { color: keep, gray: keep, mono: jbig2, ..Settings::default() }).unwrap(); + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..])); + assert!(!new.dict.contains(b"DecodeParms"), "no /JBIG2Globals"); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, &old).unwrap()); + assert!(named.iter().any(|r| stream(&doc, *r).dict.get(b"DecodeParms").is_some()), "the images drawn by name share one"); + assert!(render(&doc) == before, "the page looks the same"); +} + /// A colour-key mask names sample values: resampling or JPEG would make holes where there were none /// and fill those there were, so the picture stays as it is, however the array is written. A mask /// that can't be read as a stencil is no reason to change the picture either. @@ -5442,6 +5613,70 @@ fn a_picture_whose_mask_is_not_a_stencil_is_left_alone() { } } +/// A resource dictionary shared by reference between thousands of pages is walked once by the +/// space audit, not once per page. +#[test] +fn space_audit_walks_a_shared_resource_dictionary_once() { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 4, 4, 3, false, None); + let mut x = Dict::new(); + for i in 0..5_000 { + x.set(format!("Im{i}").into_bytes(), Object::Ref(img)); + } + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + let shared = doc.add(Object::Dict(res)); + let pages = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|d| d.reference(b"Pages"))).unwrap(); + let mut kids = Vec::new(); + for _ in 0..2_000 { + let mut p = Dict::new(); + p.set(b"Type".to_vec(), Object::name("Page")); + p.set(b"Parent".to_vec(), Object::Ref(pages)); + p.set(b"MediaBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 612.into(), 792.into()])); + p.set(b"Resources".to_vec(), Object::Ref(shared)); + kids.push(Object::Ref(doc.add(Object::Dict(p)))); + } + doc.update_dict(pages, |d| { + d.set(b"Count".to_vec(), Object::Int(kids.len() as i64)); + d.set(b"Kids".to_vec(), Object::Array(kids)); + }) + .unwrap(); + let started = std::time::Instant::now(); + let audit = audit_space(&doc, 1 << 20); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + assert!(audit.iter().any(|u| u.category == SpaceCategory::Images && u.bytes > 0), "{audit:?}"); +} + +/// A name tree whose nodes all share one kids array (by reference) is read once per node, not +/// once per node per kid: the queue can't grow to the nodes times the kids. +#[test] +fn a_name_tree_sharing_one_kids_array_is_walked_once() { + let mut doc = Document::new_empty(); + let p = page(&mut doc, &[], "BT ET"); + let s = |t: &str| Object::String(pdfcraft_cos::PdfString::literal(t.as_bytes().to_vec())); + let fit = Object::Array(vec![Object::Ref(p), Object::name("Fit")]); + let kids = doc.add(Object::Array(Vec::new())); + let mut nodes = Vec::new(); + for i in 0..3_000 { + let mut node = Dict::new(); + node.set(b"Kids".to_vec(), Object::Ref(kids)); + node.set(b"Names".to_vec(), Object::Array(vec![s(&format!("d{i}")), fit.clone()])); + nodes.push(Object::Ref(doc.add(Object::Dict(node)))); + } + doc.set(kids, Object::Array(nodes)); + let mut tree = Dict::new(); + tree.set(b"Kids".to_vec(), Object::Ref(kids)); + let tree = doc.add(Object::Dict(tree)); + let mut names = Dict::new(); + names.set(b"Dests".to_vec(), Object::Ref(tree)); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Names".to_vec(), Object::Dict(names))).unwrap(); + let started = std::time::Instant::now(); + let report = optimize(&mut doc, &Settings::default()).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed()); + assert_eq!(report.unreferenced_dests, 3_000, "every node's destination was found, once"); +} + /// Pictures run on several threads: what comes out doesn't depend on how many. #[test] fn the_same_file_comes_out_with_any_number_of_threads() { diff --git a/crates/prc-render/tests/model.rs b/crates/prc-render/tests/model.rs index 80e84252c6..ce94ceb4ce 100644 --- a/crates/prc-render/tests/model.rs +++ b/crates/prc-render/tests/model.rs @@ -78,6 +78,30 @@ fn tess_section(points: &[[f64; 3]], normals: &[[f64; 3]], corners: &[(u32, u32) w.bytes() } +/// A tessellation section of one `TESS_3D_Compressed` (authoring version 8137, origin 0, +/// tolerance 0.5) whose normals are recalculated: the seed triangle `seed`, in tolerances +/// [WD 7.8.9.7]. +fn compressed_section(seed: [i64; 9]) -> Vec { + let mut w = W::default(); + w.uint(305).uint(0).bit(true).uint(1); + w.uint(173).bit(false).bit(true).double(0.5).put(0, 32).put(0, 32).put(0, 32); + // point_array: 4-bit sign-magnitude integers. + w.bit(false).uint(9); + seed.iter().for_each(|_| _ = w.put(4, 8)); + seed.iter().for_each(|&p| _ = w.bit(p < 0).put(p.unsigned_abs(), 3)); + // Three edge statuses, none set; one triangle on face 0; three slots, none a reference. + w.bit(false).uint(3).put(0, 24); + w.bit(false).uint(1).put(1, 8).bit(false); + w.uint(3).put(0, 3).uint(0); + // Normals recalculated; no colours or multiple line attributes; no style; no texture or + // behaviours. + w.bit(true).bit(false).double(0.5).put(0, 8).bit(false).bit(false); + w.bit(false).uint(0).bit(true).bit(false); + // The section's user data. + w.uint(0); + w.bytes() +} + /// A one-file-structure PRC stream whose tessellation section is `tess`, with no product tree. fn prc_file(tess: &[u8]) -> Vec { const HEADER_LEN: usize = 107; @@ -133,6 +157,30 @@ fn a_synthetic_prc_renders_its_nearer_triangle_over_the_farther() { assert_eq!(pixel(&image, 20, 34), [57, 58, 60, 255], "the farther triangle, lit edge on"); } +/// The seed (-1, -1, 0), (1, -1, 0), then the midpoint (0, -1, 0) raised by 2 to (0, 1, 0). +const COMPRESSED_TRIANGLE: [i64; 9] = [-2, -2, 0, 4, 0, 0, 0, 4, 0]; + +#[test] +fn a_compressed_tessellation_is_rebuilt_and_rendered() { + let prc = PrcFile::parse(&prc_file(&compressed_section(COMPRESSED_TRIANGLE))).unwrap(); + let meshes = assemble(&prc).unwrap(); + assert_eq!(meshes.len(), 1); + assert_eq!(meshes[0].mesh.triangles.len(), 1); + let camera = Camera::fit_meshes(&meshes, [0.0, 0.0, -1.0], [0.0, 1.0, 0.0], false, 1.0).unwrap(); + let image = render(&meshes, &camera, &RenderOptions { width: 40, height: 40, ..RenderOptions::default() }).unwrap(); + assert_eq!(pixel(&image, 1, 1), [255, 255, 255, 255], "background corner"); + assert_ne!(pixel(&image, 20, 24), [255, 255, 255, 255], "the triangle"); +} + +/// `model_render`'s end-to-end test embeds this builder's compressed triangle as hexadecimal; +/// this keeps that fixture the builder's output. +#[test] +fn the_automation_fixture_is_the_compressed_triangle() { + let hex: String = prc_file(&compressed_section(COMPRESSED_TRIANGLE)).iter().map(|b| format!("{b:02x}")).collect(); + let fixture = include_str!("../../automation/tests/it/model_render.rs"); + assert!(fixture.contains(&format!("const COMPRESSED_TRIANGLE_PRC_HEX: &str = \"{hex}\";")), "{hex}"); +} + #[test] fn damaged_streams_are_errors_not_panics() { let points = [[0., 0., 0.], [1., 0., 0.], [0., 1., 0.]]; diff --git a/crates/prc/src/bits.rs b/crates/prc/src/bits.rs index 837d281f3b..b6897e56f6 100644 --- a/crates/prc/src/bits.rs +++ b/crates/prc/src/bits.rs @@ -69,7 +69,6 @@ impl<'a> BitReader<'a> { } /// Array elements this reader may still decode. - #[cfg(test)] pub(crate) fn elements_left(&self) -> usize { self.budget.elements } diff --git a/crates/prc/src/tess.rs b/crates/prc/src/tess.rs index c9bffb30f4..a7614b0a5c 100644 --- a/crates/prc/src/tess.rs +++ b/crates/prc/src/tess.rs @@ -2,12 +2,14 @@ //! (172) with its `TESS_Face`s (174), `TESS_3D_Wire` (175), `TESS_Markup` (176) and //! `TESS_3D_Compressed` (173) [WD 7.3.7, 7.8]. //! -//! `TESS_3D_Compressed` is read field by field to its end [WD 7.8.9.7] so the entities after -//! it decode; its triangles are not rebuilt. +//! `TESS_3D_Compressed` is read field by field to its end [WD 7.8.9.7]. One whose normals are +//! recalculated is rebuilt into triangles ([`compressed`]); one that stores its normals, or +//! whose arrays do not fit the traversal, is not. //! //! The schema runs after the fields of each concrete type read here (201, 172, 173, 174, 175, //! 176, 305), not for the abstract levels (`PRCBase`, `TESS`). +mod compressed; mod triangulate; use crate::arrays; @@ -49,7 +51,8 @@ pub enum Tessellation { Wire(Vec>), /// A `TESS_Markup` (PMI drawing), read past but not decoded. Markup, - /// A `TESS_3D_Compressed` mesh, read past but not rebuilt. + /// A `TESS_3D_Compressed` mesh, read past but not rebuilt: it stores its normals, or its + /// arrays do not fit the traversal. One that is rebuilt is a [`Tessellation::Mesh`]. #[non_exhaustive] Compressed { /// The entity's triangle count. @@ -65,7 +68,8 @@ pub struct TriangleMesh { /// The coordinate array as points. pub positions: Vec<[f64; 3]>, /// Counter-clockwise triangles seen from the outside, as indices into - /// [`Self::positions`] [WD 7.8.5.2]. + /// [`Self::positions`] [WD 7.8.5.2]. A rebuilt `TESS_3D_Compressed` keeps its traversal's + /// winding, which is not oriented. pub triangles: Vec<[u32; 3]>, /// `triangles` split by `TESS_Face`: face `i` is `triangles[faces[i].clone()]`. pub faces: Vec>, @@ -136,7 +140,7 @@ impl Ctx<'_, '_> { self.markup()?; Tessellation::Markup } - TESS_3D_COMPRESSED => Tessellation::Compressed { triangles: self.tess_3d_compressed()? }, + TESS_3D_COMPRESSED => self.tess_3d_compressed()?, t => return Err(malformed(format!("entity type {t} in the tessellation array"))), }); } @@ -295,23 +299,25 @@ impl Ctx<'_, '_> { self.schema.skip_added_fields(TESS_MARKUP, &mut self.r) } - /// `TESS_3D_Compressed` (173) after its type code [WD 7.8.9.7], read past; returns its - /// triangle count. prc-rs: the field order and version gates follow its schema. + /// `TESS_3D_Compressed` (173) after its type code [WD 7.8.9.7]: rebuilt into a mesh when + /// its normals are recalculated and its arrays fit, else read past to its end. prc-rs: the + /// field order and version gates follow its schema. /// /// Sizes the WD leaves unstated: the face count is the largest face index + 1, /// `normal_is_reversed` is one bit per triangle, and the point-reference array's /// compressed flag is implicit (more than three references). - fn tess_3d_compressed(&mut self) -> Result { + fn tess_3d_compressed(&mut self) -> Result { let r = &mut self.r; r.bit()?; // is_calculated r.bit()?; // has_faces - r.double()?; // tolerance + let tolerance = r.double()?; + let mut origin = [0.0f64; 3]; if self.version >= ORIGIN_FROM { - for _ in 0..3 { - r.float_as_bytes()?; + for o in &mut origin { + *o = f64::from(r.float_as_bytes()?); } } - arrays::compressed_integer_array(r)?; // point_array + let points = arrays::compressed_integer_array(r)?; // point_array let edge_status = arrays::character_array(r, 2, None, false)?; let face_of = arrays::compressed_indice_array(r, None)?; let t = face_of.len(); @@ -320,42 +326,50 @@ impl Ctx<'_, '_> { } let faces = face_of.iter().max().map_or(0, |&m| (m as usize).saturating_add(1)); let n = r.unsigned_integer()? as usize; - let references = arrays::bool_array(r, n)?.iter().filter(|&&b| b).count(); - let refs = arrays::compressed_indice_array(r, Some(references > 3))?; - if refs.len() != references { - return Err(malformed(format!("{} point references for {references} flagged points", refs.len()))); + let is_reference = arrays::bool_array(r, n)?; + let flagged = is_reference.iter().filter(|&&b| b).count(); + let references = arrays::compressed_indice_array(r, Some(flagged > 3))?; + if references.len() != flagged { + return Err(malformed(format!("{} point references for {flagged} flagged points", references.len()))); } - self.compressed_normals(t, faces)?; - let r = &mut self.r; - if r.bit()? { - arrays::bool_array(r, faces)?; // is_point_color_on_face - arrays::character_array(r, 8, None, false)?; // point_color_array + let recalc = self.compressed_normals(t, faces)?; + let attributes = self.compressed_attributes(faces)?; + if !recalc { + return Ok(Tessellation::Compressed { triangles: t }); } - if r.bit()? { - arrays::bool_array(r, faces)?; // is_multiple_line_attribute_on_face - } - arrays::short_array(r, 16)?; // line_attributes - if !r.bit()? { - self.compressed_texture_parameter()?; - if !self.r.bit()? { - arrays::bool_array(&mut self.r, faces)?; // face_has_texture - } - } - if self.r.bit()? { - arrays::character_array(&mut self.r, 8, None, false)?; // behaviours - } - self.schema.skip_added_fields(TESS_3D_COMPRESSED, &mut self.r)?; - Ok(t) + let arrays = compressed::Arrays { + tolerance, + origin, + points: &points, + edge_status: &edge_status, + triangles: t, + is_reference: &is_reference, + references: &references, + limit: self.r.elements_left(), + }; + // A walk that would outgrow the allowance left stops there, and the arrays stay + // compressed, as do arrays the walk does not fit. + let Ok(m) = compressed::rebuild(&arrays) else { + return Ok(Tessellation::Compressed { triangles: t }); + }; + // The rebuilt triangles, their styles and every face range are charged once the arrays + // fit and before any of them is made, so arrays that do not fit spend nothing. + let styled = attributes.styled(faces); + let kept = m.triangles.len().saturating_mul(triangle_elements(false, 0, styled)); + self.r.take_elements(kept.saturating_add(faces.saturating_mul(RANGE_ELEMENTS)))?; + let styles = if styled { attributes.styles(&face_of, faces) } else { Vec::new() }; + Ok(Tessellation::Mesh(by_face(m, &face_of, faces, &styles))) } - /// A compressed tessellation's normals, from `must_recalculate_normals` on, read past. - fn compressed_normals(&mut self, triangles: usize, faces: usize) -> Result<(), PrcError> { + /// A compressed tessellation's normals, from `must_recalculate_normals` on, read past; + /// returns whether they are recalculated. + fn compressed_normals(&mut self, triangles: usize, faces: usize) -> Result { let r = &mut self.r; if r.bit()? { arrays::bool_array(r, triangles)?; // normal_is_reversed r.double()?; // crease_angle r.character()?; // normal_recalculation_flags - return Ok(()); + return Ok(true); } let angle_bits = u32::from(r.character()?); if angle_bits > MAX_ANGLE_BITS { @@ -365,7 +379,29 @@ impl Ctx<'_, '_> { arrays::bool_array(r, n)?; // is_binary_normal arrays::short_array(r, angle_bits)?; // normal angles arrays::bool_array(r, faces)?; // is_face_planar - Ok(()) + Ok(false) + } + + /// A compressed tessellation's colours, line attributes, texture and behaviours, through + /// its added schema fields; returns what its faces' styles are made from. + fn compressed_attributes(&mut self, faces: usize) -> Result { + let r = &mut self.r; + if r.bit()? { + arrays::bool_array(r, faces)?; // is_point_color_on_face + arrays::character_array(r, 8, None, false)?; // point_color_array + } + // is_multiple_line_attribute_on_face + let multi = if r.bit()? { arrays::bool_array(r, faces)? } else { Vec::new() }; + let line_attributes = arrays::short_array(r, 16)?; + if !r.bit()? { + self.compressed_texture_parameter()?; + if !self.r.bit()? { + arrays::bool_array(&mut self.r, faces)?; // face_has_texture + } + } + let behaviours = if self.r.bit()? { arrays::character_array(&mut self.r, 8, None, false)? } else { Vec::new() }; + self.schema.skip_added_fields(TESS_3D_COMPRESSED, &mut self.r)?; + Ok(FaceAttributes { multi, line_attributes, behaviours }) } /// `CompressedTextureParameter` [WD 7.8.9.8-7.8.9.9; pdf-issues #729, #749], read past. @@ -387,6 +423,71 @@ impl Ctx<'_, '_> { } } +/// Budget elements one face range (`Range`) takes. +const RANGE_ELEMENTS: usize = std::mem::size_of::>().div_ceil(8); + +/// What a compressed tessellation's faces are styled from [WD 7.8.9.5]: with no face flagged +/// `is_multiple_line_attribute_on_face` (`multi`), `line_attributes` holds one style index + 1 +/// per face (0 keeps the owner's graphics) and `behaviours` one behaviour byte per face. +struct FaceAttributes { + multi: Vec, + line_attributes: Vec, + behaviours: Vec, +} + +impl FaceAttributes { + /// Whether the faces give their triangles styles. The one-per-triangle layout of a flagged + /// face is not established, so such a tessellation keeps its owner's graphics, as does one + /// whose attributes and behaviours are all 0. + fn styled(&self, faces: usize) -> bool { + let behaviours = self.behaviours.len() == faces && self.behaviours.iter().any(|&b| b != 0); + !self.multi.iter().any(|&b| b) && self.line_attributes.len() == faces && (behaviours || self.line_attributes.iter().any(|&a| a != 0)) + } + + /// Per triangle, the style its face gives it; call only when [`Self::styled`]. + fn styles(&self, face_of: &[u32], faces: usize) -> Vec { + let behaviour = |f: usize| { + if self.behaviours.len() == faces { self.behaviours.get(f).and_then(|&b| u16::try_from(b).ok()).unwrap_or(0) } else { 0 } + }; + face_of + .iter() + .map(|&f| { + let f = f as usize; + TriangleStyle { style: self.line_attributes.get(f).and_then(|&a| u32::try_from(a).ok()).unwrap_or(0), behaviour: behaviour(f) } + }) + .collect() + } +} + +/// A rebuilt compressed mesh with its triangles grouped by face, in walk order within each, +/// so face `i` is `triangles[faces[i].clone()]` as for a `TESS_3D`. `faces` is charged to the +/// element budget by the caller. +fn by_face(m: compressed::Rebuilt, face_of: &[u32], faces: usize, styles: &[TriangleStyle]) -> TriangleMesh { + let mut order: Vec = (0..m.triangles.len()).collect(); + order.sort_by_key(|&i| face_of.get(i).copied().unwrap_or(0)); + let mut mesh = TriangleMesh { positions: m.positions, normals_recalculated: true, ..TriangleMesh::default() }; + mesh.faces = Vec::with_capacity(faces); + for &i in &order { + let f = face_of.get(i).map_or(0, |&f| f as usize); + while mesh.faces.len() <= f { + let at = mesh.triangles.len(); + mesh.faces.push(at..at); + } + if let (Some(&t), Some(r)) = (m.triangles.get(i), mesh.faces.last_mut()) { + mesh.triangles.push(t); + r.end = mesh.triangles.len(); + } + if let Some(&s) = styles.get(i) { + mesh.triangle_styles.push(s); + } + } + while mesh.faces.len() < faces { + let at = mesh.triangles.len(); + mesh.faces.push(at..at); + } + mesh +} + /// The normal array and each triangle's corner normals, or both empty when any slot fails to /// name a normal: a mesh shades from its stored normals wholly or not at all. fn stored_normals(coords: &[f64], slots: &[[u32; 3]], triangles: usize) -> (Vec<[f64; 3]>, Vec<[u32; 3]>) { diff --git a/crates/prc/src/tess/compressed.rs b/crates/prc/src/tess/compressed.rs new file mode 100644 index 0000000000..53e082b4d2 --- /dev/null +++ b/crates/prc/src/tess/compressed.rs @@ -0,0 +1,353 @@ +//! Triangle reconstruction for a `TESS_3D_Compressed` (173) whose normals are recalculated +//! [WD 7.8.9, 7.8.9.2, 7.8.9.7]. +//! +//! The WD names the arrays but not the traversal that wrote them; the rules below are the ones +//! the arrays fit when every slot, reference and point is consumed exactly. They are inferred +//! from compressed tessellations and are not WD text. +//! +//! Traversal. A component starts with a seed triangle of three slots; every later triangle +//! takes one slot, its apex. A slot flagged in `point_is_reference` takes the next +//! `point_reference_array` value, a vertex number; any other slot makes the next vertex from +//! the next `point_array` triple. Triangle `[A B C]`, entered across `[A B]`, reads +//! `edge_status & 3`: bit 1 continues across `[C B]`, bit 0 across `[A C]`; with both, +//! `[C B]` is taken and `[A C]` pushed. With neither, pushed edges are popped, skipping any +//! already shared by two triangles; an empty stack starts the next component. A triangle +//! whose continuation edge is already shared by two triangles, while the other edge is open, +//! continues across the open one. +//! +//! Geometry, in units of `tolerance`: +//! - seed: `V0 = origin + d`, `V1 = V0 + d`, `V2 = mid(V0, V1) + d`; +//! - apex across `[P Q]` with the triangle's third vertex `W` [WD 7.8.9.2]: with +//! `O = (P + Q) * 0.5`, `L` and `H` the lower- and higher-numbered of `P`, `Q`, +//! `X = unitize(L - H)`, `Z = unitize((W - O) x X)` and `Y = unitize(Z x X)`, the apex is +//! `((O + d.x X) - d.y Y) - d.z Z`. +//! +//! Arithmetic. The encoder frames each apex on the points it already approximated [WD 7.8.9], +//! so the decoder repeats the encoder's floating point: a last-bit difference compounds from +//! apex to apex. `unitize` therefore matches the WD's `PrcPt::Unitize` pseudocode [WD 12.3] as +//! read, whose length is not exact, and the operations keep the order above. +//! +//! Fold. A triangle folded against its parent swaps bits 1 and 0 for its own continuation. +//! Without stored normals to orient it, a new apex lying in its parent's plane on the parent's +//! side (`d.z == 0`, `d.y > 0`) is folded. A component that does not fit is rewound and walked +//! again with the fold inverted at its second triangle, where an apex that is a reference +//! leaves no other trace of its fold. +//! +//! Only the three-statuses-per-triangle `edge_status` form is reconstructed (the first `T` +//! are read); the one-per-triangle form walks differently. + +use std::collections::HashMap; + +type V = [f64; 3]; + +/// The arrays of one `TESS_3D_Compressed` the walk reads. +pub(super) struct Arrays<'a> { + pub(super) tolerance: f64, + pub(super) origin: V, + pub(super) points: &'a [i64], + pub(super) edge_status: &'a [i32], + pub(super) triangles: usize, + pub(super) is_reference: &'a [bool], + pub(super) references: &'a [u32], + /// Budget elements the walk may hold at once (see [`Walk::held`]). + pub(super) limit: usize, +} + +/// The rebuilt vertices and triangles, in walk order. +#[derive(Debug)] +pub(super) struct Rebuilt { + pub(super) positions: Vec, + pub(super) triangles: Vec<[u32; 3]>, +} + +/// Rebuilds the mesh, or says why the arrays do not fit the traversal. +pub(super) fn rebuild(a: &Arrays<'_>) -> Result { + if a.edge_status.len() != a.triangles.saturating_mul(3) { + return Err("the one-status-per-triangle edge form is not reconstructed"); + } + let mut w = Walk::new(a.triangles, a.limit); + w.walk(a)?; + if w.slot != a.is_reference.len() || w.ri != a.references.len() || w.pi != a.points.len() { + return Err("the decoded triangles do not use up the stored arrays"); + } + Ok(Rebuilt { positions: w.pos, triangles: w.tris }) +} + +fn add(a: V, b: V) -> V { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +fn sub(a: V, b: V) -> V { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +fn scale(a: V, s: f64) -> V { + a.map(|c| c * s) +} +fn dot(a: V, b: V) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +fn cross(a: V, b: V) -> V { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +fn key(a: u32, b: u32) -> (u32, u32) { + if a < b { (a, b) } else { (b, a) } +} + +/// `PrcPt::Length` [WD 12.3]: the squared length summed in `f64` and rounded to `f32`, then +/// Newton's square root in `f64`, seeded with that `f32` with its exponent halved. The printed +/// loop never advances its iterate; it is read as iterating until two successive values are +/// equal, `None` (the WD's -1) when 100 steps do not settle. +fn wd_length(a: V) -> Option { + let squared = (a[0] * a[0] + a[1] * a[1] + a[2] * a[2]) as f32; + let bits = squared.to_bits(); + let exponent = (bits >> 23) & 0xff; + let seed = if exponent > 127 { f32::from_bits((bits & !(0xff << 23)) | (((exponent - 127) / 2 + 127) << 23)) } else { squared }; + let squared = f64::from(squared); + let mut x = f64::from(seed); + for _ in 0..100 { + let next = 0.5 * (x + squared / x); + if next == x { + return Some(next); + } + x = next; + } + None +} + +/// `PrcPt::Unitize` [WD 12.3]: each component divided by [`wd_length`]; `None` below +/// `FLT_EPSILON`. +fn unitize(a: V) -> Option { + let l = wd_length(a).filter(|&l| l >= f64::from(f32::EPSILON))?; + Some([a[0] / l, a[1] / l, a[2] / l]) +} + +/// The `Z` of `PrcPt::MakeOrthoRep` [WD 12.3] on `X`: `unitize(X x (0, 1, 0))`, or +/// `X x (1, 0, 0)` when that is null, from `X` unitized again as the pseudocode does. +fn make_ortho_rep_z(x: V) -> Option { + let x = unitize(x)?; + unitize(cross(x, [0.0, 1.0, 0.0])).or_else(|| unitize(cross(x, [1.0, 0.0, 0.0]))) +} + +/// The apex across `[lo hi]` (`lo` the lower-numbered vertex) opposite `w`, `d` already scaled +/// by the tolerance [WD 7.8.9.2]. A null `Z` or `Y` takes `MakeOrthoRep(X)`'s `Z` as the WD +/// prescribes, with `Y = unitize(Z x X)` from the frame's own `X` (not the copy `MakeOrthoRep` +/// unitized again, whose last bits differ), turned a half-turn about `X` when the residue of +/// `(W - O) x X` points against that `Z`. A null `X`, an edge the WD's non-degeneracy rule +/// excludes, is used as it is. +fn apex(lo: V, hi: V, w: V, d: V) -> V { + let o = scale(add(lo, hi), 0.5); + let x = sub(lo, hi); + let x = unitize(x).unwrap_or(x); + let residue = cross(sub(w, o), x); + let (y, z) = unitize(residue) + .and_then(|z| Some((unitize(cross(z, x))?, z))) + .or_else(|| { + let z = make_ortho_rep_z(x)?; + let y = unitize(cross(z, x))?; + Some(if dot(residue, z) < 0.0 { (scale(y, -1.0), scale(z, -1.0)) } else { (y, z) }) + }) + .unwrap_or_default(); + sub(sub(add(o, scale(x, d[0])), scale(y, d[1])), scale(z, d[2])) +} + +/// Traversal state; `log` records every edge-count increment so a component can be rewound. +/// Everything it holds is checked against [`Arrays::limit`] after each triangle. +#[derive(Default)] +struct Walk { + pos: Vec, + tris: Vec<[u32; 3]>, + edges: HashMap<(u32, u32), u8>, + log: Vec<(u32, u32)>, + stack: Vec<(u32, u32, u32)>, + next: Option<(u32, u32, u32)>, + slot: usize, + ri: usize, + pi: usize, +} + +/// Where a component starts, to rewind to. +#[derive(Clone, Copy)] +struct Mark { + tri: usize, + pos: usize, + log: usize, + slot: usize, + ri: usize, + pi: usize, +} + +impl Walk { + /// Reserves room for `t` triangles, no more than `limit` elements' worth (two each). + fn new(t: usize, limit: usize) -> Self { + Walk { tris: Vec::with_capacity(t.min(limit / 2).min(1 << 20)), ..Walk::default() } + } + + fn mark(&self, tri: usize) -> Mark { + Mark { tri, pos: self.pos.len(), log: self.log.len(), slot: self.slot, ri: self.ri, pi: self.pi } + } + + fn rewind(&mut self, m: Mark) { + for k in self.log.drain(m.log..) { + if let Some(n) = self.edges.get_mut(&k) { + *n = n.saturating_sub(1); + } + } + self.pos.truncate(m.pos); + self.tris.truncate(m.tri); + self.stack.clear(); + self.next = None; + (self.slot, self.ri, self.pi) = (m.slot, m.ri, m.pi); + } + + /// One slot: `Some(v)` a reference to vertex `v`, `None` a new point. + fn take(&mut self, a: &Arrays<'_>) -> Result, &'static str> { + let flagged = *a.is_reference.get(self.slot).ok_or("slots run out")?; + self.slot = self.slot.saturating_add(1); + if !flagged { + return Ok(None); + } + let v = *a.references.get(self.ri).ok_or("references run out")?; + self.ri = self.ri.saturating_add(1); + self.get(v)?; + Ok(Some(v)) + } + + fn point(&mut self, a: &Arrays<'_>) -> Result { + let Some(&[x, y, z]) = a.points.get(self.pi..self.pi.saturating_add(3)) else { + return Err("points run out"); + }; + self.pi = self.pi.saturating_add(3); + let tol = a.tolerance; + Ok([x as f64 * tol, y as f64 * tol, z as f64 * tol]) + } + + fn get(&self, v: u32) -> Result { + self.pos.get(v as usize).copied().ok_or("a triangle refers to a vertex not yet decoded") + } + + fn push(&mut self, p: V) -> Result { + let id = u32::try_from(self.pos.len()).map_err(|_| "too many vertices")?; + self.pos.push(p); + Ok(id) + } + + /// The next triangle and its default fold: the seed, or the apex across the edge to + /// continue from. + fn triangle(&mut self, a: &Arrays<'_>) -> Result<([u32; 3], bool), &'static str> { + if let Some((p, q, w)) = self.next.take() { + if let Some(r) = self.take(a)? { + return Ok(([p, q, r], false)); + } + let (pp, qq, ww) = (self.get(p)?, self.get(q)?, self.get(w)?); + let (lo, hi) = if p < q { (pp, qq) } else { (qq, pp) }; + let fold = matches!(a.points.get(self.pi..self.pi.saturating_add(3)), Some(&[_, dy, 0]) if dy > 0); + let d = self.point(a)?; + let r = self.push(apex(lo, hi, ww, d))?; + return Ok(([p, q, r], fold)); + } + let mut v = [0u32; 3]; + for k in 0..3 { + let id = match self.take(a)? { + Some(r) => r, + None => { + let d = self.point(a)?; + let p = match k { + 0 => add(a.origin, d), + 1 => add(self.get(v[0])?, d), + _ => add(scale(add(self.get(v[0])?, self.get(v[1])?), 0.5), d), + }; + self.push(p)? + } + }; + if let Some(s) = v.get_mut(k) { + *s = id; + } + } + Ok((v, false)) + } + + /// Budget elements (8 bytes each) the walk's arrays hold, by length: growth can add up to + /// as much again in spare capacity. + fn held(&self) -> usize { + let pos = self.pos.len().saturating_mul(3); + let tris = self.tris.len().saturating_mul(2); + let edges = self.edges.len().saturating_mul(2); + let stack = self.stack.len().saturating_mul(2); + pos.saturating_add(tris).saturating_add(edges).saturating_add(self.log.len()).saturating_add(stack) + } + + fn closed(&self, e: (u32, u32, u32)) -> bool { + self.edges.get(&key(e.0, e.1)).copied().unwrap_or(0) >= 2 + } + + /// Adds one triangle; `flip` inverts its fold. + fn step(&mut self, a: &Arrays<'_>, status: i32, flip: bool) -> Result<(), &'static str> { + let ([ta, tb, tc], fold) = self.triangle(a)?; + for (x, y) in [(ta, tb), (tb, tc), (tc, ta)] { + let n = self.edges.entry(key(x, y)).or_insert(0); + *n = n.saturating_add(1); + let shared = *n; + self.log.push(key(x, y)); + if shared > 2 { + return Err("an edge is shared by more than two triangles"); + } + } + self.tris.push([ta, tb, tc]); + if self.held() > a.limit { + return Err("the rebuild needs more than the decoded-element allowance left"); + } + let (mut left, mut right) = ((tc, tb, ta), (ta, tc, tb)); + if fold != flip { + std::mem::swap(&mut left, &mut right); + } + let (taken, other) = if status & 2 != 0 { (left, right) } else { (right, left) }; + if status & 3 != 0 && self.closed(taken) && !self.closed(other) { + std::mem::swap(&mut left, &mut right); + } + match (status & 2 != 0, status & 1 != 0) { + (true, true) => { + self.next = Some(left); + self.stack.push(right); + } + (true, false) => self.next = Some(left), + (false, true) => self.next = Some(right), + (false, false) => {} + } + while self.next.is_none() { + let Some(e) = self.stack.pop() else { break }; + if !self.closed(e) { + self.next = Some(e); + } + } + Ok(()) + } + + /// Walks every component. One that fails is rewound and walked once more with the fold + /// inverted at its second triangle, so the walk is at most twice the triangle count. + fn walk(&mut self, a: &Arrays<'_>) -> Result<(), &'static str> { + let mut start = self.mark(0); + let mut flip_at = None; + let mut i = 0; + while i < a.triangles { + if self.next.is_none() && !(flip_at.is_some() && i == start.tri) { + start = self.mark(i); + flip_at = None; + } + let status = a.edge_status.get(i).copied().unwrap_or(0); + match self.step(a, status, flip_at == Some(i)) { + Ok(()) => i = i.saturating_add(1), + Err(e) => { + if flip_at.is_some() || i == start.tri { + return Err(e); + } + self.rewind(start); + flip_at = Some(start.tri.saturating_add(1)); + i = start.tri; + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/prc/src/tess/compressed/tests.rs b/crates/prc/src/tess/compressed/tests.rs new file mode 100644 index 0000000000..d174ff657e --- /dev/null +++ b/crates/prc/src/tess/compressed/tests.rs @@ -0,0 +1,299 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::*; + +const S: f64 = std::f64::consts::SQRT_2; + +fn run(points: &[i64], status: &[i32], t: usize, is_ref: &[bool], refs: &[u32]) -> Result { + rebuild(&Arrays { + tolerance: 0.5, + origin: [10.0, 0.0, 0.0], + points, + edge_status: status, + triangles: t, + is_reference: is_ref, + references: refs, + limit: usize::MAX, + }) +} + +fn close(a: V, b: V) -> bool { + (0..3).all(|i| (a[i] - b[i]).abs() < 1e-9) +} + +/// Seed `[0 1 2]`, then the apex across `[2 1]`: `X` runs from vertex 2 to vertex 1, and +/// each of `d`'s three components lands on its own axis with its own sign. Tolerance 0.5 +/// halves every step. +#[test] +fn a_seed_and_one_apex_rebuild() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 2, -6, 2]; + let m = run(&pts, &[2, 0, 0, 0, 0, 0], 2, &[false; 4], &[]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3]]); + let p: Vec = m.positions.iter().map(|v| [v[0] - 10.0, v[1], v[2]]).collect(); + assert!(close(p[0], [0.0, 0.0, 0.0]) && close(p[1], [2.0, 0.0, 0.0])); + assert!(close(p[2], [0.0, 2.0, 0.0]), "{:?}", p[2]); + assert!(close(p[3], [1.0 + 2.0 * S, 1.0 + S, -1.0]), "{:?}", p[3]); +} + +/// The walk stops once its arrays would hold more than `limit` elements: the seed-and-apex +/// pair ends holding four vertices (12), two triangles (4), five edges (10) and six logged +/// edges (6), 32 in all. +#[test] +fn a_walk_past_its_limit_stops() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 2, -6, 2]; + let with = |limit| { + rebuild(&Arrays { + tolerance: 0.5, + origin: [0.0; 3], + points: &pts, + edge_status: &[2, 0, 0, 0, 0, 0], + triangles: 2, + is_reference: &[false; 4], + references: &[], + limit, + }) + }; + assert_eq!(with(32).unwrap().triangles.len(), 2); + assert_eq!(with(31).unwrap_err(), "the rebuild needs more than the decoded-element allowance left"); +} + +/// The triangles reserved up front fit the limit too. +#[test] +fn the_walk_reserves_within_its_limit() { + assert!(Walk::new(1 << 20, 10).tris.capacity() < 1 << 20); + assert!(Walk::new(1 << 20, 10).tris.capacity() <= 5); + assert!(Walk::new(4, usize::MAX).tris.capacity() >= 4); +} + +/// Bit 0 alone continues across `[A C]`, keeping `W = B`. +#[test] +fn the_right_edge_is_entered_from_a_to_c() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, -2, 0]; + let m = run(&pts, &[1, 0, 0, 0, 0, 0], 2, &[false; 4], &[]).unwrap(); + assert_eq!(m.triangles[1], [0, 2, 3]); +} + +/// A reference slot reuses a vertex; a closed pushed edge is skipped and the next component +/// starts from a fresh seed. +#[test] +fn references_and_components() { + let pts = [0; 9 + 9]; + let is_ref = [false, false, false, true, false, false, false]; + let m = run(&pts, &[3, 0, 0, 0, 0, 0, 0, 0, 0], 3, &is_ref, &[0]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 0], [3, 4, 5]]); +} + +/// A double-sided panel: four coplanar vertices, four triangles. The second triangle's apex +/// folds back onto the seed's side, so its own continuation takes `[A C]` first; the last two +/// apexes are the seed's first two vertices. +#[test] +fn a_folded_apex_swaps_its_continuation() { + let is_ref = [false, false, false, false, true, true]; + let status = [3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let m = run(&[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], &status, 4, &is_ref, &[0, 1]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [2, 3, 0], [0, 3, 1]]); + // The same panel unfolded is rescued by the component retry. + let m = run(&[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, -3, 0], &status, 4, &is_ref, &[0, 1]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [2, 3, 0], [0, 3, 1]]); +} + +/// The fourth apex `(1, 3, 0)` lies in its parent's plane on the parent's side, so its +/// triangle folds and continues across `[A C]` to the reference. +#[test] +fn a_fold_past_the_retry_point_is_detected() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, -3, 1, 1, 3, 0]; + let is_ref = [false, false, false, false, false, true]; + let m = run(&pts, &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], 4, &is_ref, &[0]).unwrap(); + assert_eq!(m.triangles[3], [3, 4, 0]); +} + +/// A panel whose fold apex is a reference carries no fold signal: the unfolded walk closes an +/// edge it still needs, so the component is rewound and walked again folded at its second +/// triangle. +#[test] +fn a_reference_fold_is_found_by_retrying_the_component() { + let pts: Vec = (1..=30).collect(); + let mut is_ref = vec![false; 9]; + is_ref.extend([true, true, false, true, true, true]); + let mut status = vec![0; 21]; + status[3..7].copy_from_slice(&[3, 3, 2, 0]); + let m = run(&pts, &status, 7, &is_ref, &[0, 3, 6, 0, 3]).unwrap(); + assert_eq!(m.triangles[3..], [[0, 3, 9], [9, 3, 6], [9, 6, 0], [0, 6, 3]]); +} + +/// A tetrahedron closed by two references: after `[3 1 0]` the continuation edge `[0 1]` is +/// already shared twice, so the walk takes the open `[3 0]` instead. +#[test] +fn a_closed_continuation_edge_gives_way_to_the_open_one() { + let pts = [0, 0, 0, 4, 0, 0, -2, 4, 0, 0, 0, 4]; + let mut status = [0; 12]; + status[..3].copy_from_slice(&[2, 2, 2]); + let mut is_ref = [false; 6]; + is_ref[4..].copy_from_slice(&[true, true]); + let m = run(&pts, &status, 4, &is_ref, &[0, 2]).unwrap(); + assert_eq!(m.triangles, [[0, 1, 2], [2, 1, 3], [3, 1, 0], [3, 0, 2]]); +} + +#[test] +fn arrays_that_do_not_fit_are_refused() { + let pts = [0; 9]; + let ok = [false; 3]; + assert!(run(&pts, &[0, 0, 0], 1, &ok, &[]).is_ok()); + assert!(run(&pts, &[0], 1, &ok, &[]).is_err(), "T form"); + assert!(run(&[0; 12], &[0, 0, 0], 1, &ok, &[]).is_err(), "points left"); + assert!(run(&[0; 6], &[0, 0, 0], 1, &ok, &[]).is_err(), "points short"); + assert!(run(&pts, &[0, 0, 0], 1, &[false; 4], &[]).is_err(), "slots left"); + assert!(run(&[0; 6], &[0, 0, 0], 1, &[false, false, true], &[7]).is_err(), "forward reference"); + assert!(run(&[0; 9], &[0, 0, 0], 1, &[false, false, false, true], &[0]).is_err(), "references left"); + // A third triangle on the seed's edge [0 1], each apex a reference. + let three = [false, false, false, true, true]; + let shared = run(&[0; 9], &[2, 2, 0, 0, 0, 0, 0, 0, 0], 3, &three, &[0, 0]).map(|_| ()); + assert_eq!(shared, Err("an edge is shared by more than two triangles")); +} + +/// `PrcPt::Length` rounds the squared length to `f32` and takes Newton's root from a halved +/// exponent, which can settle an ulp off the correctly rounded root; `Unitize` refuses +/// lengths below `FLT_EPSILON` [WD 12.3]. +#[test] +fn length_and_unitize_follow_the_wd_pseudocode() { + assert_eq!(wd_length([3.0, 4.0, 0.0]), Some(5.0)); + let newton = f64::from_bits(S.to_bits() - 1); + assert_eq!(wd_length([1.0, 1.0, 0.0]), Some(newton)); + let long = 1.0 + 2f64.powi(-30); + assert_eq!(wd_length([long, 0.0, 0.0]), Some(1.0)); + assert_eq!(unitize([0.0, long, 0.0]), Some([0.0, long, 0.0])); + assert_eq!(unitize([1e-8, 0.0, 0.0]), None); + assert_eq!(unitize([0.0; 3]), None); +} + +/// A `W` on the edge's line leaves `Z` null, so the frame comes from `MakeOrthoRep(X)`: `Z` +/// from `X x (0, 1, 0)`, or from `X x (1, 0, 0)` when `X` is along `Y` [WD 7.8.9.2, 12.3]. +#[test] +fn a_null_z_takes_the_ortho_rep_of_x() { + let d = [1.0, 2.0, 3.0]; + assert_eq!(apex([0.0; 3], [-2.0, 0.0, 0.0], [5.0, 0.0, 0.0], d), [0.0, -2.0, -3.0]); + assert_eq!(apex([0.0; 3], [0.0, -2.0, 0.0], [0.0, 5.0, 0.0], d), [-2.0, 0.0, 3.0]); +} + +/// A degenerate frame is turned a half-turn about `X` when the sub-epsilon residue of +/// `(W - O) x X` points against `MakeOrthoRep`'s `Z`, and kept when it points along it. +#[test] +fn a_degenerate_frame_follows_its_residue() { + let (lo, hi, d) = ([0.0; 3], [-2.0, 0.0, 0.0], [1.0, 2.0, 3.0]); + assert_eq!(apex(lo, hi, [5.0, 1e-9, 0.0], d), [0.0, 2.0, 3.0]); + assert_eq!(apex(lo, hi, [5.0, -1e-9, 0.0], d), [0.0, -2.0, -3.0]); +} + +/// A degenerate frame's `Y` comes from the frame's own `X`, whose length `unitize` left off by +/// a few 1e-8, not from the copy `MakeOrthoRep` unitized again: the two differ by that much, +/// and the walk would carry the difference into later apexes. +#[test] +fn a_degenerate_frame_takes_y_from_its_own_x() { + let (lo, hi) = ([0.1, 0.2, 0.3], [0.7, -0.4, 0.9]); + let o = scale(add(lo, hi), 0.5); + let x = unitize(sub(lo, hi)).unwrap(); + let w = add(o, scale(x, 0.37)); + let d = [1.0, 2.0, 3.0]; + let z = make_ortho_rep_z(x).unwrap(); + let p = apex(lo, hi, w, d); + let own = unitize(cross(z, x)).unwrap(); + let again = unitize(cross(z, unitize(x).unwrap())).unwrap(); + assert_ne!(own, again, "the probe must tell the two Ys apart"); + let sign = if dot(cross(sub(w, o), x), z) < 0.0 { -1.0 } else { 1.0 }; + let expect = |y: V| sub(sub(add(o, scale(x, d[0])), scale(y, sign * d[1])), scale(z, sign * d[2])); + assert_eq!(p, expect(own)); + assert_ne!(p, expect(again)); +} + +/// A strip of thin, non-planar triangles compressed by an encoder written out from the WD: +/// each point is expressed in the frame of the working mesh, rounded to the tolerance and +/// reinjected [WD 7.8.9, 7.8.9.2], with `PrcPt::Length` and `Unitize` transcribed from the +/// pseudocode [WD 12.3] as read. The decoder must land exactly on that working mesh, which +/// keeps every vertex within the rounding bound of its source. +#[test] +fn a_strip_decodes_onto_the_encoders_working_mesh() { + fn length(v: V) -> f64 { + let f = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) as f32; + let mut bits = f.to_bits(); + let exponent = (bits >> 23) & 0xff; + if exponent > 127 { + bits = (bits & 0x807f_ffff) | (((exponent - 127) / 2 + 127) << 23); + } + let (squared, mut x0) = (f64::from(f), f64::from(f32::from_bits(bits))); + for _ in 0..100 { + let xi = 0.5 * (x0 + squared / x0); + if xi == x0 { + return xi; + } + x0 = xi; + } + -1.0 + } + fn unitized(v: V) -> V { + let l = length(v); + assert!(l >= f64::from(f32::EPSILON), "null axis"); + [v[0] / l, v[1] / l, v[2] / l] + } + let (tol, n) = (1e-4, 48); + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + let mut jitter = |scale: f64| { + state = state.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + ((state >> 11) as f64 / (1u64 << 53) as f64 - 0.5) * scale + }; + // Two rails 4 apart, 0.05 between a rail's points, 0.02 of relief. + let source: Vec = (0..n) + .map(|i| { + let (along, rail) = (i as f64 * 0.025, (i % 2) as f64 * 4.0); + [along + jitter(0.01), rail + jitter(0.01), jitter(0.02)] + }) + .collect(); + let round = |v: f64| (v / tol).round() as i64; + let (mut points, mut work) = (Vec::new(), Vec::::new()); + for (i, target) in source.iter().take(3).enumerate() { + let base = match i { + 0 => [0.0; 3], + 1 => work[0], + _ => scale(add(work[0], work[1]), 0.5), + }; + let d = [0, 1, 2].map(|c| round(target[c] - base[c])); + points.extend(d); + work.push(add(base, d.map(|c| c as f64 * tol))); + } + // Triangle k enters across vertices k and k + 1, opposite k - 1, and continues right on + // odd k, left on even k, unless it folds. + let mut status = vec![0; 3 * (n - 2)]; + status[0] = 2; + for k in 1..n - 2 { + let (lo, hi, w) = (work[k], work[k + 1], work[k - 1]); + let o = scale(add(hi, lo), 0.5); + let x = unitized(sub(lo, hi)); + let z = unitized(cross(sub(w, o), x)); + let y = unitized(cross(z, x)); + let r = sub(source[k + 2], o); + let d = [round(dot(r, x)), round(-dot(r, y)), round(-dot(r, z))]; + points.extend(d); + let [dx, dy, dz] = d.map(|c| c as f64 * tol); + work.push(sub(sub(add(o, scale(x, dx)), scale(y, dy)), scale(z, dz))); + if k < n - 3 { + let fold = d[2] == 0 && d[1] > 0; + status[k] = if (k % 2 == 1) != fold { 1 } else { 2 }; + } + } + let m = rebuild(&Arrays { + tolerance: tol, + origin: [0.0; 3], + points: &points, + edge_status: &status, + triangles: n - 2, + is_reference: &vec![false; n], + references: &[], + limit: usize::MAX, + }) + .unwrap(); + assert_eq!(m.triangles[1..4], [[2, 1, 3], [2, 3, 4], [4, 3, 5]]); + assert_eq!(m.positions, work); + for (p, s) in m.positions.iter().zip(&source) { + let off = dot(sub(*p, *s), sub(*p, *s)).sqrt() / tol; + assert!(off < 0.87, "{off} tolerances off"); + } +} diff --git a/crates/prc/src/tess/tests.rs b/crates/prc/src/tess/tests.rs index 26219f2865..253e1c9d5e 100644 --- a/crates/prc/src/tess/tests.rs +++ b/crates/prc/src/tess/tests.rs @@ -529,12 +529,281 @@ fn a_compressed_mesh_is_read_to_its_end() { let schema = Schema::default(); let mut ctx = Ctx::new(BitReader::new(&bytes), &schema, v); let t = ctx.file_structure_tessellation().unwrap(); - assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }, Tessellation::Mesh(_)]), "full={full} v={v}: {t:?}"); assert_eq!(ctx.r.position(), w.len(), "full={full} v={v}"); + assert!(matches!(t[1], Tessellation::Mesh(_)), "full={full} v={v}: {t:?}"); + // Stored normals and the one-status-per-triangle form are not rebuilt; the + // recalculated 3T seed is. + if full || !three_t { + assert!(matches!(t[0], Tessellation::Compressed { triangles: 1, .. }), "full={full} v={v}: {t:?}"); + } else { + let m = mesh(&t[0]); + assert_eq!((m.triangles.as_slice(), m.faces.as_slice()), ([[0, 1, 2]].as_slice(), std::slice::from_ref(&(0..1))), "v={v}"); + assert!(m.normals_recalculated && m.normals.is_empty()); + assert_eq!(m.triangle_styles, [TriangleStyle { style: 1, behaviour: 0 }]); + // Each point component is 1, in units of the 0.001 tolerance, from the origin 0. + let want = [0.001, 0.002, 0.0025]; + assert!(m.positions.iter().zip(want).all(|(p, c)| p.iter().all(|x| (x - c).abs() < 1e-12)), "{:?}", m.positions); + } } } } +/// A `TESS_3D_Compressed` (authoring version 8137, origin 0) whose normals are recalculated, +/// with `points` in units of `tolerance`, the 3T `status`, each triangle's face, the slots' +/// reference flags, the references and one line attribute per face; every other optional +/// branch absent. +struct Packed<'a> { + tolerance: f64, + points: &'a [i64], + status: &'a [u8], + face_of: &'a [u32], + /// The width of each face index; 8 unless an index needs more. + face_bits: u32, + is_ref: &'a [bool], + refs: &'a [u32], + line_attributes: &'a [u16], +} + +/// A `CompressedIndiceArray` [WD 11.9] of `bits`-wide deltas, `implicit` with no compressed +/// flag. +fn indices(w: &mut W, v: &[u32], implicit: bool, bits: u32) { + if !implicit { + w.bit(false); + } + w.uint(u32::try_from(v.len()).unwrap()); + for i in 0..v.len() { + w.put(if i == 0 { u64::from(bits) } else { 0 }, 8); + } + let mut prev = 0i64; + for &x in v { + let d = i64::from(x) - prev; + w.bit(d < 0).put(d.unsigned_abs(), bits - 1); + prev = i64::from(x); + } +} + +fn packed(w: &mut W, c: &Packed<'_>) { + w.uint(173).bit(false).bit(true).double(c.tolerance); + w.put(0, 32).put(0, 32).put(0, 32); + // point_array: 8-bit sign-magnitude integers. + w.bit(false).uint(u32::try_from(c.points.len()).unwrap()); + for _ in c.points { + w.put(8, 8); + } + for &p in c.points { + w.bit(p < 0).put(p.unsigned_abs(), 7); + } + w.bit(false).uint(u32::try_from(c.status.len()).unwrap()); + for &e in c.status { + w.put(u64::from(e), 8); + } + indices(w, c.face_of, false, c.face_bits); + w.uint(u32::try_from(c.is_ref.len()).unwrap()); + for &b in c.is_ref { + w.bit(b); + } + assert!(c.refs.len() <= 3, "more than three references are Huffman-coded"); + indices(w, c.refs, true, 8); + // must_recalculate_normals, normal_is_reversed, crease angle, flags; no colours, no + // multiple line attributes. + w.bit(true); + for _ in c.face_of { + w.bit(false); + } + w.double(0.5).put(0, 8).bit(false).bit(false); + w.bit(false).uint(u32::try_from(c.line_attributes.len()).unwrap()); + for &a in c.line_attributes { + w.put(u64::from(a & 0xff), 8).put(u64::from(a >> 8), 8); + } + // No texture, no behaviours. + w.bit(true).bit(false); +} + +/// A double-sided panel of two faces packed in walk order, its fold apex and its last two +/// apexes references: rebuilt into a mesh grouped by face, each triangle styled by its face. +#[test] +fn a_recalculated_compressed_mesh_is_rebuilt_by_face() { + let mut b = W::default(); + packed( + &mut b, + &Packed { + tolerance: 0.5, + points: &[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], + status: &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], + face_of: &[1, 0, 1, 0], + face_bits: 8, + is_ref: &[false, false, false, false, true, true], + refs: &[0, 1], + line_attributes: &[3, 0], + }, + ); + let w = section(1, &b); + let bytes = w.bytes(); + let schema = Schema::default(); + let mut ctx = Ctx::new(BitReader::new(&bytes), &schema, 8137); + let t = ctx.file_structure_tessellation().unwrap(); + assert_eq!(ctx.r.position(), w.len()); + let m = mesh(&t[0]); + // Walk order [0 1 2] [2 1 3] [2 3 0] [0 3 1]; faces 1, 0, 1, 0. + assert_eq!(m.triangles, [[2, 1, 3], [0, 3, 1], [0, 1, 2], [2, 3, 0]]); + assert_eq!(m.faces, [0..2, 2..4]); + assert_eq!(m.positions[..3], [[0.0, 0.0, 0.0], [2.0, 0.0, 0.0], [0.0, 2.0, 0.0]]); + let style = |style| TriangleStyle { style, behaviour: 0 }; + assert_eq!(m.triangle_styles, [style(3), style(3), style(0), style(0)]); + assert!(m.normals_recalculated); +} + +/// Arrays the walk cannot fit stay a [`Tessellation::Compressed`], read to their end. +#[test] +fn a_compressed_mesh_that_does_not_fit_is_left_compressed() { + let mut b = W::default(); + // One point too many for a lone seed. + let c = Packed { + tolerance: 0.5, + points: &[0; 12], + status: &[0, 0, 0], + face_of: &[0], + face_bits: 8, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[0], + }; + packed(&mut b, &c); + let w = section(1, &b); + let t = decode(&w, &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }]), "{t:?}"); +} + +/// A face index near `u32::MAX` claims four billion faces: once the triangles rebuild, they are +/// charged before any face range is made, so the decode stops at the element allowance instead +/// of allocating them. +#[test] +fn a_huge_face_index_is_charged_not_allocated() { + let mut b = W::default(); + let c = Packed { + tolerance: 0.5, + points: &[0; 9], + status: &[0, 0, 0], + face_of: &[u32::MAX - 1], + face_bits: 33, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[], + }; + packed(&mut b, &c); + let r = decode(§ion(1, &b), &Schema::default(), 8137); + assert!(matches!(r, Err(PrcError::TooManyElements { .. })), "{r:?}"); +} + +/// The same huge face index on arrays the walk cannot fit charges nothing for triangles or +/// faces: the tessellation stays compressed and the file reads on. +#[test] +fn arrays_that_do_not_fit_spend_no_triangles_or_faces() { + let mut b = W::default(); + let c = Packed { + tolerance: 0.5, + points: &[0; 12], + status: &[0, 0, 0], + face_of: &[u32::MAX - 1], + face_bits: 33, + is_ref: &[false; 3], + refs: &[], + line_attributes: &[], + }; + packed(&mut b, &c); + let t = decode(§ion(1, &b), &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 1, .. }]), "{t:?}"); +} + +/// The two-face panel of [`a_recalculated_compressed_mesh_is_rebuilt_by_face`], its second +/// reference `last_ref`. +fn panel(last_ref: u32) -> W { + let mut b = W::default(); + packed( + &mut b, + &Packed { + tolerance: 0.5, + points: &[0, 0, 0, 4, 0, 0, -2, 4, 0, 1, 3, 0], + status: &[3, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], + face_of: &[1, 0, 1, 0], + face_bits: 8, + is_ref: &[false, false, false, false, true, true], + refs: &[0, last_ref], + line_attributes: &[3, 0], + }, + ); + section(1, &b) +} + +/// A rebuilt mesh is charged its triangles, each with its style, and each face range at its +/// size; the same arrays left compressed (a reference to a vertex not yet made) are charged +/// neither. +#[test] +fn a_rebuilt_compressed_mesh_is_charged_its_triangles_and_faces() { + let fits = panel(1); + let t = decode(&fits, &Schema::default(), 8137).unwrap(); + assert_eq!(mesh(&t[0]).triangles.len(), 4); + let left = panel(9); + let t = decode(&left, &Schema::default(), 8137).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 4, .. }]), "{t:?}"); + assert_eq!(spent(&fits) - spent(&left), 4 * triangle_elements(false, 0, true) + 2 * RANGE_ELEMENTS); + assert_eq!(triangle_elements(false, 0, true), 3); + #[cfg(target_pointer_width = "64")] + assert_eq!(RANGE_ELEMENTS, 2); +} + +/// A rebuild whose walk would outgrow the allowance left keeps the arrays compressed, even +/// when the rebuilt mesh alone would have fit, and the stream reads on. +#[test] +fn a_compressed_rebuild_is_bounded_by_the_allowance_left() { + let fits = panel(1); + let bytes = fits.bytes(); + let schema = Schema::default(); + let read = |n| Ctx::new(BitReader::with_elements(&bytes, n), &schema, 8137).file_structure_tessellation(); + let arrays = spent(&panel(9)); + let mesh_alone = spent(&fits) - arrays; + let t = read(arrays + mesh_alone).unwrap(); + assert!(matches!(t[..], [Tessellation::Compressed { triangles: 4, .. }]), "{t:?}"); + assert!(matches!(read(crate::MAX_DECODED_ELEMENTS).unwrap()[..], [Tessellation::Mesh(_)])); +} + +/// A face's nonzero behaviour styles its triangles even when its line attribute is 0 (the +/// owner's graphics); all zero, the faces give no styles. +#[test] +fn a_compressed_face_behaviour_alone_styles_its_triangles() { + let attributes = |behaviours: Vec| FaceAttributes { multi: Vec::new(), line_attributes: vec![0], behaviours }; + let a = attributes(vec![0x30]); + assert!(a.styled(1)); + assert_eq!(a.styles(&[0, 0], 1), [TriangleStyle { style: 0, behaviour: 0x30 }; 2]); + assert!(!attributes(vec![0]).styled(1)); + assert!(!attributes(Vec::new()).styled(1)); +} + +fn rebuilt(triangles: usize) -> compressed::Rebuilt { + compressed::Rebuilt { positions: vec![[0.0; 3]; 3], triangles: (0..triangles).map(|i| [0, 1, u32::try_from(i).unwrap()]).collect() } +} + +/// Triangles are grouped by face in walk order within each, faces no triangle names (between +/// and after) get empty ranges, and each style moves with its triangle. +#[test] +fn rebuilt_triangles_are_grouped_by_face_with_their_styles() { + let face_of = [2, 0, 2, 1, 0]; + let m = by_face(rebuilt(5), &face_of, 5, &[]); + let apex: Vec = m.triangles.iter().map(|t| t[2]).collect(); + assert_eq!(apex, [1, 4, 3, 0, 2]); + assert_eq!(m.faces, [0..2, 2..3, 3..5, 5..5, 5..5]); + assert!(m.triangle_styles.is_empty()); + assert!(m.normals_recalculated); + + let style = |style| TriangleStyle { style, behaviour: u16::try_from(style).unwrap() }; + let styles: Vec = face_of.iter().map(|&f| style(f + 1)).collect(); + let m = by_face(rebuilt(5), &[2, 0, 2, 3, 0], 4, &styles); + let apex: Vec = m.triangles.iter().map(|t| t[2]).collect(); + assert_eq!(apex, [1, 4, 0, 2, 3]); + assert_eq!(m.faces, [0..2, 2..2, 2..4, 4..5]); + assert_eq!(m.triangle_styles, [style(1), style(1), style(3), style(3), style(2)]); +} + #[test] fn refusals_and_damage_are_errors() { // Edge statuses neither T nor 3T long. diff --git a/crates/preflight/src/lib.rs b/crates/preflight/src/lib.rs index 7b6e960006..90c6d9db83 100644 --- a/crates/preflight/src/lib.rs +++ b/crates/preflight/src/lib.rs @@ -203,6 +203,7 @@ pub const WORDINGS: &[&str] = &[ "A graphics state has the rendering intent {}, which isn't a standard one", "A graphics state has a rendering intent that isn't a name", "The form asks viewers to make appearances (NeedAppearances)", + "Some content could not be read for device colour", // Pages, actions, annotations and fields. "Page {} has additional actions", "A {} action isn't allowed", @@ -229,7 +230,8 @@ pub struct Declared { /// `pdfaid:part` and `pdfaid:conformance`, e.g. (2, "B"). pub pdfa: Option<(u8, String)>, pub pdfua: Option, - /// Output intents' condition identifiers. + /// Output intents' condition identifiers: every intent the document lists (each object + /// once), each identifier cut to [`MAX_IDENTIFIER_BYTES`]. pub output_intents: Vec, } @@ -276,10 +278,27 @@ pub fn declared(doc: &Document) -> Declared { d.pdfua = xmp::identifier(&x, xmp::PDFUA_ID, "part").and_then(|p| p.parse().ok()); } let cat = catalog(doc); - if let Some(list) = cat.get(b"OutputIntents").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()) { - for oi in list { - if let Some(od) = doc.resolve(&oi).as_dict() { - let id = od.get(b"OutputConditionIdentifier").and_then(|v| doc.resolve(v).as_string().map(|s| s.to_text())).unwrap_or_default(); + let list = cat.get(b"OutputIntents").map(|o| doc.resolve(o)); + if let Some(list) = list.as_deref().and_then(Object::as_array) { + // One intent object listed again is one declaration: its identifier is not copied once + // per listing. Every distinct one is kept (verification reads them all too), and only + // the start of an identifier longer than any real one. + let mut seen = std::collections::HashSet::new(); + for oi in list.iter().filter(|oi| oi.as_ref().is_none_or(|r| seen.insert(r))) { + if let Some(od) = doc.resolve(oi).as_dict() { + let id = od + .get(b"OutputConditionIdentifier") + .and_then(|v| { + doc.resolve(v).as_string().map(|s| { + if s.bytes.len() <= MAX_IDENTIFIER_BYTES { + s.to_text() + } else { + // An even cut keeps UTF-16 text whole code units. + pdfcraft_cos::PdfString { bytes: s.bytes[..MAX_IDENTIFIER_BYTES & !1].to_vec(), hex: s.hex }.to_text() + } + }) + }) + .unwrap_or_default(); d.output_intents.push(id); } } @@ -287,6 +306,14 @@ pub fn declared(doc: &Document) -> Declared { d } +/// The most bytes of an output condition identifier [`declared`] keeps (a real one is a few +/// dozen). +const MAX_IDENTIFIER_BYTES: usize = 1024; + +/// Decoded bytes one verification reads looking for device colour, all content streams +/// together. Past it, content is not inspected, and the report says so. +const MAX_COLOUR_DECODED: usize = 1 << 30; + /// Every indirect object, with its reference. fn objects(doc: &Document) -> Vec<(ObjRef, std::sync::Arc)> { doc.object_numbers() @@ -361,6 +388,9 @@ struct DeviceColour { /// space, which PDF/A accepts whatever the output intent (ISO 19005-2 6.2.4.3). default_rgb: bool, default_cmyk: bool, + /// Content that could not be read (undecodable, or past [`MAX_COLOUR_DECODED`]) may use + /// device colour that was not seen. + incomplete: bool, } impl DeviceColour { @@ -388,27 +418,35 @@ const MAX_DIRECT_DEPTH: u8 = 8; /// The glyph procedures of every Type 3 font in `d`: `d` itself, or a font held directly inside /// it (a font can sit inline in a resource dictionary). -fn type3_procs(doc: &Document, d: &Dict, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs(doc: &Document, d: &Dict, depth: u8, read: &mut std::collections::HashSet, out: &mut std::collections::HashSet) { if d.name(b"Subtype") == Some(b"Type3") - && let Some(procs) = d.get(b"CharProcs").map(|c| doc.resolve(c)) - && let Some(procs) = procs.as_dict() + && let Some(c) = d.get(b"CharProcs") + // One glyph dictionary shared by many fonts is read once: its entries are the same. + && c.as_ref().is_none_or(|r| read.insert(r)) + && let Some(procs) = doc.resolve(c).as_dict() { out.extend(procs.iter().filter_map(|(_, p)| p.as_ref())); } for (_, v) in d.iter() { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } -fn type3_procs_within(doc: &Document, o: &Object, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs_within( + doc: &Document, + o: &Object, + depth: u8, + read: &mut std::collections::HashSet, + out: &mut std::collections::HashSet, +) { if depth > MAX_DIRECT_DEPTH { return; } match o { - Object::Dict(d) => type3_procs(doc, d, depth, out), + Object::Dict(d) => type3_procs(doc, d, depth, read, out), Object::Array(a) => { for v in a { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } _ => {} @@ -430,9 +468,10 @@ fn referenced_content(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) }; out.extend(contents.iter().filter_map(Object::as_ref)); } + let mut read = std::collections::HashSet::new(); for (_, o) in objs { if let Some(d) = dict_of(o) { - type3_procs(doc, d, 0, &mut out); + type3_procs(doc, d, 0, &mut read, &mut out); } } out @@ -560,6 +599,7 @@ fn mark_cs_object(doc: &Document, cs: &Object, used: &mut DeviceColour, depth: u fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) -> DeviceColour { let mut used = DeviceColour::default(); let referenced = referenced_content(doc, objs); + let mut budget = pdfcraft_cos::DecodeBudget::new(MAX_COLOUR_DECODED, usize::MAX); for (r, o) in objs { let Some(d) = dict_of(o) else { continue }; // A resource dictionary's /ColorSpace, whether it is this object or the page's inline @@ -586,9 +626,14 @@ fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) if !(used.gray && used.rgb && used.cmyk) && let Object::Stream(s) = &**o && (referenced.contains(r) || is_content(&s.dict)) - && let Ok(data) = s.decoded() { - let _ = content_colour(&data, &mut used); + match budget.decode(s) { + Ok(data) => { + let _ = content_colour(&data, &mut used); + } + // Not read: it may draw in a device space nothing above saw. + Err(_) => used.incomplete = true, + } } } used @@ -830,6 +875,9 @@ fn check_output_intent( issue: &mut impl FnMut(&'static str, &'static str, Option, Option, bool), ) { let used = uses_device_colour(doc, objs); + if used.incomplete { + issue("6.2.3", "Some content could not be read for device colour", None, None, false); + } match pdfa_intent_profile(doc) { None if used.any() => { issue("6.2.3", "Device colour is used, but there is no PDF/A output intent", None, None, !used.cmyk); @@ -1215,22 +1263,56 @@ const MAX_ACTION_CHAIN: usize = 1024; fn chain_forbidden(doc: &Document, a: &Dict) -> Option { let mut read = 0; let mut seen = std::collections::HashSet::new(); - // Actions written in an indirect `/Next` array, copied out of it. - let mut held: Vec = Vec::new(); + // Objects `/Next` refers to (an action, or an array of them), each read once and borrowed: + // one array shared by many chains is not copied per chain. + let mut held: Vec> = Vec::new(); if let Some(s) = walk_chain(doc, a, &mut read, &mut seen, &mut held) { return Some(s); } - while let Some(d) = held.pop() { - if let Some(s) = walk_chain(doc, &d, &mut read, &mut seen, &mut held) { - return Some(s); + while let Some(o) = held.pop() { + let dicts: Vec<&Dict> = match &*o { + Object::Dict(d) => vec![d], + Object::Array(list) => { + // An array read is work too, with its entries: a chain of arrays alone must + // still reach the limit. + read = read.saturating_add(1 + list.len().min(MAX_ACTION_CHAIN)); + if read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); + } + // More than the chain limit can't be read anyway. + hold_referenced(doc, list, &mut seen, &mut held); + list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).collect() + } + _ => continue, + }; + for d in dicts { + if let Some(s) = walk_chain(doc, d, &mut read, &mut seen, &mut held) { + return Some(s); + } } } None } -/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. The -/// actions of each indirect `/Next` array (each read once) go to `held`. -fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collections::HashSet, held: &mut Vec) -> Option { +/// Queue the objects the references in a `/Next` array name, each once. +fn hold_referenced(doc: &Document, list: &[Object], seen: &mut std::collections::HashSet, held: &mut Vec>) { + for r in list.iter().filter_map(Object::as_ref).take(MAX_ACTION_CHAIN) { + if seen.insert(r) { + // Resolved through any alias: a reference to a reference names its object's value. + held.push(doc.resolve(&Object::Ref(r))); + } + } +} + +/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. What +/// an indirect `/Next`, or a reference inside a `/Next` array, names goes to `held`. +fn walk_chain( + doc: &Document, + a: &Dict, + read: &mut usize, + seen: &mut std::collections::HashSet, + held: &mut Vec>, +) -> Option { let mut stack = vec![a]; while let Some(d) = stack.pop() { *read += 1; @@ -1242,13 +1324,16 @@ fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collec } match d.get(b"Next") { Some(Object::Dict(next)) => stack.push(next), - Some(Object::Array(list)) => stack.extend(list.iter().filter_map(Object::as_dict)), - Some(Object::Ref(r)) if seen.insert(*r) => { - if let Object::Array(list) = &*doc.get(*r) { - // More than the chain limit can't be read anyway. - held.extend(list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).cloned()); + Some(Object::Array(list)) => { + *read = read.saturating_add(list.len().min(MAX_ACTION_CHAIN)); + // A wide array is refused before any of it is queued. + if *read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); } + hold_referenced(doc, list, seen, held); + stack.extend(list.iter().filter_map(Object::as_dict)); } + Some(Object::Ref(r)) if seen.insert(*r) => held.push(doc.resolve(&Object::Ref(*r))), _ => {} } } @@ -1291,6 +1376,26 @@ fn strip_actions(doc: &Document, d: &mut Dict) -> bool { changed } +/// Take the annotations `gone` off every page that lists them, each page's list rewritten once, and +/// replace their indirect objects with `Null`. +fn remove_from_pages(doc: &mut Document, gone: &std::collections::HashSet) -> Result<(), pdfcraft_cos::CosError> { + if gone.is_empty() { + return Ok(()); + } + for p in pdfcraft_model::pages(doc) { + let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); + let listed = |a: &Object| a.as_ref().is_some_and(|r| gone.contains(&r)); + if annots.iter().any(listed) { + let kept: Vec = annots.into_iter().filter(|a| !listed(a)).collect(); + doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; + } + } + for r in gone { + doc.set(*r, Object::Null); + } + Ok(()) +} + /// Collects every indirect annotation reference listed in any page's `/Annots` array. fn page_annot_refs(doc: &Document) -> std::collections::HashSet { let mut refs = std::collections::HashSet::new(); @@ -1304,19 +1409,6 @@ fn page_annot_refs(doc: &Document) -> std::collections::HashSet { refs } -/// Removes annotation `r` from every page's `/Annots` list and replaces its indirect object with `Null`. -fn remove_annot_from_pages(doc: &mut Document, r: ObjRef) -> Result<(), pdfcraft_cos::CosError> { - for p in pdfcraft_model::pages(doc) { - let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); - if annots.iter().any(|a| a.as_ref() == Some(r)) { - let kept: Vec = annots.into_iter().filter(|a| a.as_ref() != Some(r)).collect(); - doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; - } - } - doc.set(r, Object::Null); - Ok(()) -} - fn fix_annot_or_field(doc: &Document, nd: &mut Dict, is_annot: bool, sub: Option<&[u8]>, actions: &mut usize, flags: &mut usize) -> bool { let mut changed = strip_actions(doc, nd); *actions += changed as usize; @@ -1413,8 +1505,10 @@ pub fn convert(doc: &mut Document, level: Level) -> Result Result::to_vec); let sub = d.name(b"Subtype").map(<[u8]>::to_vec); - let is_annot = matches!(&*o, Object::Dict(_)) && is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)); + let is_annot = matches!(&*o, Object::Dict(_)) + // A widget is an annotation whether or not it says `/Type /Annot` (the form reader + // finds it by its field), so its actions go the same way. + && (is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)) || sub.as_deref() == Some(b"Widget")); if is_annot && sub.as_deref().is_some_and(|s| FORBIDDEN_ANNOTS.contains(&s)) { - remove_annot_from_pages(doc, r)?; + // Taken off its page below, all of them in one pass over the pages. + forbidden_annots.insert(r); removed += 1; continue; } @@ -1489,6 +1588,7 @@ pub fn convert(doc: &mut Document, level: Level) -> Result>", pages, page, &link(&format!("<< /S /GoTo /D [3 0 R /Fit] /Next {js} >>"))], ""), // `/Next` refers to an array that holds the action directly. build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), &format!("[{js}]")], ""), + // `/Next` refers to the action itself, which carries `/P` and `/K` as a structure + // element would: those keys must not exempt it. + build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + pages, + page, + &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), + "<< /S /JavaScript /JS (app.alert(1)) /P 3 0 R /K 0 >>", + ], + "", + ), + // A `/Next` array refers to the action. + build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next [5 0 R] >>"), js], ""), ]; for (i, mut doc) in cases.into_iter().enumerate() { let found = messages(&verify(&doc, Level::A2b)).contains(&"A JavaScript action isn't allowed"); @@ -495,6 +509,116 @@ fn an_output_intent_profile_header_is_read_within_a_size_cap() { } } +/// A widget the form reader finds by its field is an annotation whether or not it writes +/// `/Type /Annot`: conversion takes its additional actions off all the same. +#[test] +fn a_widget_without_a_type_loses_its_additional_actions() { + let pages = "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>"; + let page = "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>"; + let widget = "<< /Subtype /Widget /Rect [0 0 10 10] /F 4 /T (f) /AA << /K << /S /JavaScript /JS (1) >> >> >>"; + let mut doc = build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, widget], ""); + convert(&mut doc, Level::A2b).unwrap(); + let w = doc.get(pdfcraft_cos::ObjRef::new(4, 0)); + assert!(w.as_dict().is_some_and(|d| d.get(b"AA").is_none()), "{w:?}"); +} + +/// Forbidden annotations on one page are taken off it in one pass, all of them. +#[test] +fn forbidden_annotations_leave_their_page_together() { + let n = 300; + let annots: Vec = (0..n).map(|i| format!("{} 0 R", i + 4)).collect(); + let page = format!("<< /Type /Page /Parent 2 0 R /Annots [{} 2 0 R] >>", annots.join(" ")); + let mut objs = + vec!["<< /Type /Catalog /Pages 2 0 R >>".to_string(), "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>".into(), page]; + objs.extend((0..n).map(|_| "<< /Type /Annot /Subtype /Sound /Rect [0 0 0 0] /P 3 0 R >>".to_string())); + let refs: Vec<&str> = objs.iter().map(String::as_str).collect(); + let mut doc = build(&refs, ""); + let r = convert(&mut doc, Level::A2b).unwrap(); + assert!(r.fixed.iter().any(|f| f.starts_with(&format!("{n}: removed multimedia"))), "{:?}", r.fixed); + let left = pdfcraft_model::pages(&doc)[0].dict.get(b"Annots").and_then(|a| doc.resolve(a).as_array().map(Vec::len)); + assert_eq!(left, Some(1)); +} + +/// Content that can't be read may draw in DeviceCMYK: convert then adds no sRGB output intent. +#[test] +fn convert_adds_no_srgb_intent_when_colour_could_not_all_be_read() { + let mut doc = build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>", + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << >> >>", + // A /Filter that isn't a name can't be decoded: the content isn't read. + "<< /Length 9 /Filter 5 >>\nstream\nnot flate\nendstream", + ], + "", + ); + convert(&mut doc, Level::A2b).unwrap(); + assert!(declared(&doc).output_intents.is_empty()); +} + +/// A `/Next` chain of arrays alone (each array naming the next) still reaches the chain limit, +/// so the walk's work is bounded however the arrays are linked. +#[test] +fn an_array_only_next_chain_is_too_deeply_chained() { + let mut doc = Document::new_empty(); + let mut next = Object::Array(Vec::new()); + for _ in 0..2 * MAX_ACTION_CHAIN { + next = Object::Array(vec![Object::Ref(doc.add(next))]); + } + let mut action = Dict::new(); + action.set(b"S".to_vec(), Object::name("URI")); + action.set(b"Next".to_vec(), Object::Ref(doc.add(next))); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // A short chain of arrays is read through. + let mut short = Object::Array(Vec::new()); + for _ in 0..10 { + short = Object::Array(vec![Object::Ref(doc.add(short))]); + } + action.set(b"Next".to_vec(), Object::Ref(doc.add(short))); + assert_eq!(chain_forbidden(&doc, &action), None); +} + +/// A direct `/Next` array wider than the chain limit is refused as soon as it is charged, and a +/// `/Next` that reaches its array through a reference alias is read through it. +#[test] +fn a_wide_direct_next_array_is_refused_and_an_alias_is_followed() { + let uri = || { + let mut d = Dict::new(); + d.set(b"S".to_vec(), Object::name("URI")); + d + }; + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Array((0..=MAX_ACTION_CHAIN).map(|_| Object::Dict(uri())).collect())); + let doc = Document::new_empty(); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // `/Next` -> 1 0 R -> 2 0 R -> [ << /S /JavaScript >> ] + let mut doc = Document::new_empty(); + let mut js = Dict::new(); + js.set(b"S".to_vec(), Object::name("JavaScript")); + let list = doc.add(Object::Array(vec![Object::Dict(js)])); + let alias = doc.add(Object::Ref(list)); + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Ref(alias)); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("JavaScript")); +} + +/// Every distinct output intent is declared, past eight; an identifier longer than any real one +/// keeps its start. +#[test] +fn every_output_intent_is_declared_with_its_identifier_cut_to_the_bound() { + let long = "x".repeat(4 * MAX_IDENTIFIER_BYTES); + let intents: Vec = (0..9) + .map(|i| { + format!("<< /Type /OutputIntent /S /GTS_PDFA1 /OutputConditionIdentifier ({}) >>", if i == 8 { long.clone() } else { format!("c{i}") }) + }) + .collect(); + let catalog = format!("<< /Type /Catalog /Pages 2 0 R /OutputIntents [{}] >>", intents.join(" ")); + let doc = build(&[catalog.as_str(), "<< /Type /Pages /Kids [] /Count 0 >>"], ""); + let d = declared(&doc); + assert_eq!(d.output_intents.len(), 9); + assert_eq!(d.output_intents[8].len(), MAX_IDENTIFIER_BYTES); +} + #[test] fn typeless_annotations_indexed_cmyk_and_indirect_filter_elements_are_handled() { // Annotations in /Annots that omit optional /Type /Annot and /P (while keeping required /Subtype diff --git a/crates/print/Cargo.toml b/crates/print/Cargo.toml index 9d349b6568..bec4e283a7 100644 --- a/crates/print/Cargo.toml +++ b/crates/print/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +craft-cms = { workspace = true } craft-print = { workspace = true } log = { workspace = true } pdfcraft-cos = { workspace = true } diff --git a/crates/print/src/colour.rs b/crates/print/src/colour.rs new file mode 100644 index 0000000000..5f13de3352 --- /dev/null +++ b/crates/print/src/colour.rs @@ -0,0 +1,338 @@ +//! App-managed colour for printing (Acrobat's Color Handling ▸ "app manages colours"): printed +//! pages, rendered in sRGB, are converted into the printer's ICC profile with a rendering intent +//! and optional black point compensation before the driver sees them. The default leaves colour +//! to the printer, as before. +//! +//! The pages go as converted pictures, so the driver's own colour management should be off for +//! the job; otherwise the printer converts them a second time. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub use craft_cms::Intent; +use craft_cms::{Builtin, ColorSpace, Profile, TransformOptions}; + +/// The largest profile read from disk. Printer profiles with large tables are a few megabytes. +pub const MAX_PROFILE_BYTES: u64 = 32 << 20; + +/// How many files the installed-profile listing looks at, at most. +const MAX_LISTED: usize = 2000; + +/// An ICC profile pages can be printed into: an RGB or grey profile that can be a destination. +#[derive(Clone, Debug, PartialEq)] +pub struct PrinterProfile { + profile: Arc, + name: String, +} + +impl PrinterProfile { + /// `bytes` as a printer profile. Refused when they are not an ICC profile, when it has no + /// path from the PCS to the device, or when its device space is neither RGB nor grey (the + /// pictures a print carries are RGB). + pub fn from_bytes(bytes: &[u8]) -> Result { + let profile = Profile::parse(bytes).map_err(|e| format!("not a usable ICC profile: {e}"))?; + if !matches!(profile.color_space, ColorSpace::Rgb | ColorSpace::Gray) { + return Err(format!("a {:?} profile can't be printed into: choose an RGB or grey printer profile", profile.color_space)); + } + if !profile.can_be_destination() { + return Err("the profile only describes a source: it has no table to convert colours into it".into()); + } + let name = profile.description.trim().to_string(); + Ok(PrinterProfile { profile: Arc::new(profile), name }) + } + + /// The profile in the file at `path`, at most [`MAX_PROFILE_BYTES`] long. Anything but a + /// regular file (links followed) is refused before it is opened: opening a FIFO or a device + /// can wait forever. + pub fn read(path: &Path) -> Result { + let at = |e: String| format!("{}: {e}", path.display()); + let meta = std::fs::metadata(path).map_err(|e| at(e.to_string()))?; + if !meta.is_file() { + return Err(at("not a regular file".into())); + } + if meta.len() > MAX_PROFILE_BYTES { + return Err(at(too_large())); + } + let file = std::fs::File::open(path).map_err(|e| at(e.to_string()))?; + let bytes = read_limited(file, MAX_PROFILE_BYTES).map_err(at)?; + PrinterProfile::from_bytes(&bytes).map_err(at) + } + + /// The profile's description (its `desc` tag), possibly empty. + pub fn name(&self) -> &str { + &self.name + } + + /// The ICC bytes, as read. + pub fn bytes(&self) -> Arc> { + self.profile.to_bytes() + } + + /// Whether the printer's device space is grey. + pub fn is_gray(&self) -> bool { + self.profile.color_space == ColorSpace::Gray + } +} + +/// At most `limit` bytes of `source`: a file that grew after it was measured is refused, not read +/// whole. +fn read_limited(source: impl std::io::Read, limit: u64) -> Result, String> { + use std::io::Read as _; + let mut bytes = Vec::new(); + source.take(limit.saturating_add(1)).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() as u64 > limit { + return Err(too_large()); + } + Ok(bytes) +} + +fn too_large() -> String { + format!("larger than an ICC profile can be here ({} MB)", MAX_PROFILE_BYTES >> 20) +} + +/// App-managed colour: the profile the pages are converted into, how, and whether black point +/// compensation maps the darkest sRGB black onto the printer's darkest black. +#[derive(Clone, Debug, PartialEq)] +pub struct ManagedColour { + pub profile: PrinterProfile, + pub intent: Intent, + pub black_point: bool, +} + +impl ManagedColour { + /// Converts opaque sRGB pixels (RGBA, alpha ignored and kept) into the printer profile in + /// place. A grey profile's value is written to all three channels. + pub fn convert(&self, rgba: &mut [u8]) -> Result<(), String> { + let opts = TransformOptions { intent: self.intent, bpc: self.black_point, ..TransformOptions::default() }; + let t = craft_cms::cached(Builtin::Srgb.profile(), &self.profile.profile, opts) + .map_err(|e| format!("can't convert into the printer profile: {e}"))?; + let source = rgba.to_vec(); + if self.profile.is_gray() { + let mut grey = vec![0u8; source.len() / 4]; + t.convert_u8(&source, 4, &mut grey, 1, false).map_err(|e| format!("can't convert into the printer profile: {e}"))?; + for (px, g) in rgba.as_chunks_mut::<4>().0.iter_mut().zip(grey) { + px[..3].fill(g); + } + Ok(()) + } else { + t.convert_u8(&source, 4, rgba, 4, true).map_err(|e| format!("can't convert into the printer profile: {e}")) + } + } +} + +/// A profile installed on this computer that pages can be printed into. +#[derive(Clone, Debug, PartialEq)] +pub struct InstalledProfile { + pub path: PathBuf, + /// Its description, else its file name. + pub name: String, + pub gray: bool, +} + +/// The folders the system keeps colour profiles in, those that exist. +pub fn profile_folders() -> Vec { + let mut dirs: Vec = Vec::new(); + if cfg!(windows) { + let root = std::env::var_os("SystemRoot").map_or_else(|| PathBuf::from(r"C:\Windows"), PathBuf::from); + dirs.push(root.join("System32").join("spool").join("drivers").join("color")); + } else if cfg!(target_os = "macos") { + dirs.push("/System/Library/ColorSync/Profiles".into()); + dirs.push("/Library/ColorSync/Profiles".into()); + if let Some(home) = std::env::var_os("HOME") { + dirs.push(PathBuf::from(home).join("Library/ColorSync/Profiles")); + } + } else { + dirs.push("/usr/share/color/icc".into()); + dirs.push("/usr/local/share/color/icc".into()); + if let Some(home) = std::env::var_os("HOME") { + dirs.push(PathBuf::from(&home).join(".local/share/color/icc")); + dirs.push(PathBuf::from(home).join(".color/icc")); + } + } + dirs.retain(|d| d.is_dir()); + dirs +} + +/// The RGB and grey profiles in `folders` and their immediate subfolders that pages can be +/// printed into, sorted by name. Files that aren't such a profile are skipped. +pub fn installed_profiles_in(folders: &[PathBuf]) -> Vec { + installed_profiles_where(folders, &|_| true) +} + +/// [`installed_profiles_in`], reading only the files `allow` passes (a caller confined to some +/// folders checks each file, links followed, before it is opened). +pub fn installed_profiles_where(folders: &[PathBuf], allow: &dyn Fn(&Path) -> bool) -> Vec { + let mut files: Vec = Vec::new(); + for dir in folders { + collect(dir, 1, &mut files); + } + let mut out: Vec = files + .into_iter() + .filter(|path| allow(path)) + .filter_map(|path| { + let p = PrinterProfile::read(&path).ok()?; + let name = if p.name().is_empty() { path.file_name()?.to_string_lossy().into_owned() } else { p.name().to_string() }; + Some(InstalledProfile { gray: p.is_gray(), name, path }) + }) + .collect(); + out.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase()).then_with(|| a.path.cmp(&b.path))); + out +} + +/// The profiles the system has installed (see [`profile_folders`]). +pub fn installed_profiles() -> Vec { + installed_profiles_in(&profile_folders()) +} + +/// The `.icc` and `.icm` regular files in `dir` (links followed), descending `depth` more levels, +/// up to [`MAX_LISTED`]. A FIFO or device with such a name is skipped: reading it can block. +fn collect(dir: &Path, depth: u32, files: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + if files.len() >= MAX_LISTED { + return; + } + let path = entry.path(); + let Ok(kind) = entry.file_type() else { continue }; + if kind.is_dir() { + if depth > 0 { + collect(&path, depth - 1, files); + } + } else if path.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("icc") || e.eq_ignore_ascii_case("icm")) + && std::fs::metadata(&path).is_ok_and(|m| m.is_file()) + { + files.push(path); + } + } +} + +/// The installed profile `name` names: its description or its file name, ignoring case. +pub fn find_installed<'a>(profiles: &'a [InstalledProfile], name: &str) -> Option<&'a InstalledProfile> { + let name = name.trim(); + profiles.iter().find(|p| p.name.eq_ignore_ascii_case(name)).or_else(|| { + profiles.iter().find(|p| { + let file = p.path.file_name().map(|f| f.to_string_lossy()).unwrap_or_default(); + let stem = p.path.file_stem().map(|f| f.to_string_lossy()).unwrap_or_default(); + file.eq_ignore_ascii_case(name) || stem.eq_ignore_ascii_case(name) + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_cms::Curve; + + /// A source longer than the limit is refused after reading one byte past it, not read whole. + #[test] + fn a_profile_read_stops_one_byte_past_the_limit() { + let source = [7u8; 100]; + assert_eq!(read_limited(&source[..], 100).map(|b| b.len()), Ok(100)); + let mut rest = &source[..]; + assert!(read_limited(&mut rest, 10).is_err()); + assert_eq!(rest.len(), 89, "read the limit and one byte more"); + } + + /// A FIFO named like a profile is neither listed nor opened: opening it for reading would wait + /// for a writer that never comes. + #[cfg(unix)] + #[test] + fn a_fifo_named_like_a_profile_is_skipped() { + let dir = std::env::temp_dir().join(format!("pdfcraft-print-fifo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("printer.icc"); + let _ = std::fs::remove_file(&fifo); + let made = std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()); + if !made { + eprintln!("mkfifo is unavailable; skipped"); + return; + } + // On a thread, so that a read which blocks fails the test instead of hanging it. + let (tx, rx) = std::sync::mpsc::channel(); + let (folders, path) = (vec![dir.clone()], fifo.clone()); + std::thread::spawn(move || { + let _ = tx.send((installed_profiles_in(&folders).len(), PrinterProfile::read(&path).err())); + }); + let outcome = rx.recv_timeout(std::time::Duration::from_secs(10)); + let _ = std::fs::remove_dir_all(&dir); + let (listed, refused) = outcome.expect("reading the FIFO blocked"); + assert_eq!(listed, 0, "the listing skipped the FIFO"); + assert!(refused.is_some_and(|e| e.contains("not a regular file")), "the FIFO was refused before it was opened"); + } + + /// A grey printer profile whose darkest black is a light charcoal (8% luminance, L* 34): + /// what a matte paper or a worn toner gives. Synthetic, from the built-in sGray. + fn lifted_black_grey() -> Vec { + let mut p = Builtin::SGray.profile().clone(); + p.gray_trc = Some(Curve::Table((0..=255).map(|i| 0.08 + 0.92 * (i as f32 / 255.0)).collect())); + p.black_point = None; + p.description = "Lifted black grey".into(); + p.with_encoded_bytes().to_bytes().to_vec() + } + + fn convert(profile: &[u8], bpc: bool, rgba: &[u8]) -> Vec { + let colour = ManagedColour { profile: PrinterProfile::from_bytes(profile).unwrap(), intent: Intent::RelativeColorimetric, black_point: bpc }; + let mut out = rgba.to_vec(); + colour.convert(&mut out).unwrap(); + out + } + + /// Dark sRGB greys below the printer's black: without compensation they all clip to its + /// black; with it they stay apart and in order (test the shadows, not only mid grey). + #[test] + fn black_point_compensation_keeps_dark_shadows_apart() { + let profile = lifted_black_grey(); + let darks = [20u8, 40, 60].iter().flat_map(|&v| [v, v, v, 255]).collect::>(); + let clipped = convert(&profile, false, &darks); + assert_eq!((clipped[0], clipped[4], clipped[8]), (0, 0, 0), "{clipped:?}"); + let kept = convert(&profile, true, &darks); + assert!(kept[0] < kept[4] && kept[4] < kept[8], "{kept:?}"); + assert!(kept.as_chunks::<4>().0.iter().all(|&[r, g, b, a]| r == g && g == b && a == 255), "grey written to R, G and B: {kept:?}"); + } + + /// An RGB printer profile changes the values (sRGB's green is not the profile's green) and + /// keeps alpha. + #[test] + fn rgb_profiles_convert_the_values_and_keep_alpha() { + let wide = Builtin::AdobeRgbCompat.profile().to_bytes().to_vec(); + let out = convert(&wide, true, &[0, 255, 0, 128, 255, 255, 255, 255]); + assert!(out[0] > 100 && out[1] > 240 && out[2] > 20, "sRGB green inside a wider gamut: {out:?}"); + assert_eq!(out[3], 128); + assert!(out[4..7].iter().all(|&v| v >= 254), "white stays white: {out:?}"); + let same = convert(&Builtin::Srgb.profile().to_bytes(), true, &[10, 120, 230, 255]); + assert!(same.iter().zip([10u8, 120, 230, 255]).all(|(a, b)| a.abs_diff(b) <= 1), "sRGB into sRGB: {same:?}"); + } + + #[test] + fn cmyk_and_other_bytes_are_refused() { + let cmyk = Builtin::CoatedCmyk.profile().to_bytes(); + assert!(PrinterProfile::from_bytes(&cmyk).is_err_and(|e| e.contains("RGB or grey"))); + assert!(PrinterProfile::from_bytes(b"not a profile").is_err()); + assert!(PrinterProfile::from_bytes(&[]).is_err()); + } + + /// The listing finds RGB and grey profiles one folder deep, by description, and skips CMYK + /// profiles, other files and broken ones; a name finds one by description or file name. + #[test] + fn installed_profiles_are_rgb_and_grey_ones() { + let dir = std::env::temp_dir().join(format!("pdfcraft-print-profiles-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("vendor").join("deeper")).unwrap(); + std::fs::write(dir.join("office.ICC"), Builtin::Srgb.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("vendor").join("matte.icm"), lifted_black_grey()).unwrap(); + std::fs::write(dir.join("vendor").join("deeper").join("hidden.icc"), Builtin::Srgb.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("press.icc"), Builtin::CoatedCmyk.profile().to_bytes().as_slice()).unwrap(); + std::fs::write(dir.join("broken.icc"), b"acsp?").unwrap(); + std::fs::write(dir.join("notes.txt"), b"hello").unwrap(); + let found = installed_profiles_in(std::slice::from_ref(&dir)); + let names: Vec<(&str, bool)> = found.iter().map(|p| (p.name.as_str(), p.gray)).collect(); + assert_eq!(names.len(), 2, "{found:?}"); + assert!(names.contains(&("Lifted black grey", true)), "{names:?}"); + assert!(found.iter().any(|p| !p.gray && p.path.ends_with("office.ICC")), "{found:?}"); + assert!(find_installed(&found, "lifted BLACK grey").is_some_and(|p| p.gray)); + assert!(find_installed(&found, "matte").is_some_and(|p| p.gray), "by file stem"); + assert!(find_installed(&found, "office.icc").is_some_and(|p| !p.gray), "by file name"); + assert!(find_installed(&found, "press").is_none()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/print/src/lib.rs b/crates/print/src/lib.rs index 5707a42012..815a8351d3 100644 --- a/crates/print/src/lib.rs +++ b/crates/print/src/lib.rs @@ -20,6 +20,7 @@ use pdfcraft_content::Matrix; use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; +pub mod colour; mod font_names; pub mod range; pub mod spool; diff --git a/crates/print/src/settings.rs b/crates/print/src/settings.rs index 50657b328a..49d90c0308 100644 --- a/crates/print/src/settings.rs +++ b/crates/print/src/settings.rs @@ -186,17 +186,20 @@ pub struct Settings { } /// How pages are rendered when printing as image. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct Raster { /// Dots per inch, 72–600 (Acrobat offers 150, 300 and 600). pub dpi: f64, /// Render the pictures in grey, so the print-ready PDF itself is grayscale. pub grayscale: bool, + /// App-managed colour: the pictures are converted into the printer's profile. `None` leaves + /// colour to the printer. + pub colour: Option, } impl Default for Raster { fn default() -> Self { - Raster { dpi: 300.0, grayscale: false } + Raster { dpi: 300.0, grayscale: false, colour: None } } } @@ -204,7 +207,7 @@ impl Default for Raster { /// Not when the pages go as grey pictures (`raster` with [`Raster::grayscale`]): they are grey /// already, so a driver that keeps no grayscale setting (Microsoft Print to PDF, many inkjets) /// still prints them instead of the job being refused (#5806). -pub fn driver_needs_grayscale(grayscale: bool, raster: Option) -> bool { +pub fn driver_needs_grayscale(grayscale: bool, raster: Option<&Raster>) -> bool { grayscale && !raster.is_some_and(|r| r.grayscale) } diff --git a/crates/redact/README.md b/crates/redact/README.md index e47df98f17..d8f4b46e94 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -130,7 +130,10 @@ Text is placed with the font and size set by `Tf` or by an ExtGState's `/Font` t form XObject starts from the text state of the content that draws it. Content that can't be decoded makes the operation fail rather than leave it unredacted, and so does an inline image the renderer's tokenizer reads no image for (`RedactError::Unreadable`): where the page's operators -resume after it would be a guess. A Redact annotation with neither a usable `/QuadPoints` array nor a usable +resume after it would be a guess. So do bytes the content parser can't read, in the page's content +or in a form under an area (a stray `)` or `}` in a `TJ` array, say): the interpreter reads past +them, so text they hide, which other readers may draw, would be neither found nor removed. Remove +hidden information refuses a page for both too, in whatever content it walks. A Redact annotation with neither a usable `/QuadPoints` array nor a usable `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..12636dc88f 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -93,6 +93,12 @@ struct Gs { scale: f64, leading: f64, rise: f64, + /// The stroke parameters that widen a painted path beyond its construction points. + line_width: f64, + miter_limit: f64, + line_join: i64, + /// `J` / ExtGState `/LC`: a projecting square cap (2) reaches past the endpoint's corner. + line_cap: i64, /// The tiling or shading pattern selected as the fill colour (`/Name scn`), and as the stroke /// colour (`/Name SCN`). Part of the graphics state, so `q`/`Q` save and restore them with /// everything else. A pattern's cell is a content stream of its own and this pass does not @@ -143,6 +149,11 @@ pub(crate) struct Output { /// ([`pdfcraft_content::Parsed::ambiguous_images`]): what this scope placed after it may not /// be what the page shows, so a redaction must not trust it. pub ambiguous: bool, + /// The content holds bytes the parser can't read ([`pdfcraft_content::Parsed::skipped`]), in + /// this scope or a nested form: the walk reads past them, so text they hide (a stray `)` in a + /// `TJ` array, say), which other readers may still draw, is neither found nor removed, and a + /// rewrite would drop the bytes but keep the strings. A redaction must not trust it. + pub unparsed: bool, pub tag_changes: Vec<(ObjRef, ObjRef, HashSet, HashSet)>, pub error: Option, } @@ -389,6 +400,10 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< scale: 1.0, leading: 0.0, rise: 0.0, + line_width: 1.0, + miter_limit: 10.0, + line_join: 0, + line_cap: 0, fill_pattern: None, stroke_pattern: None, type3_inherits: false, @@ -407,6 +422,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], let joined = Pieces::join(streams); let mut parsed = joined.parse(); walk.out.ambiguous = parsed.ambiguous_images > 0; + walk.out.unparsed = parsed.skipped > 0; // A rewritten piece loses its comments, so keep where they began and ended: a comment that // ran on into the next piece still hides that piece's first line. let comment_edges = joined.comment_edges(&parsed); @@ -432,6 +448,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], } if !walk.path.is_empty() { done.ops.append(&mut walk.path); + done.ops.append(&mut walk.pending); } processed.push((done.ops, done.changed)); } @@ -499,6 +516,10 @@ struct Walk { path: Vec, path_box: Option<[f64; 4]>, clip: bool, + /// Line-state operators (`w`, `M`, `j`, `J`, `d`, `gs`) met while a path is open: they apply to + /// the stroke that ends it, so the path stays whole for the hit test, and they are written + /// after it, kept even when the path goes. + pending: Vec, /// Painting operators written again under a clip that cuts the regions out: unchanged as /// operators, but what they paint is no longer all there. clipped: HashSet, @@ -520,6 +541,7 @@ impl Walk { path: Vec::new(), path_box: None, clip: false, + pending: Vec::new(), clipped: HashSet::new(), out: Output::default(), } @@ -536,14 +558,20 @@ impl Walk { self.paint_path(scope, op, piece); return; } + if !self.path.is_empty() && matches!(o, b"w" | b"M" | b"j" | b"J" | b"d" | b"gs") { + self.graphics_state(doc, scope, &op); + self.pending.push(op); + return; + } if !self.path.is_empty() { // A path not ended by a painting operator: pass it through. piece.ops.append(&mut self.path); + piece.ops.append(&mut self.pending); self.path_box = None; self.clip = false; } let kept = match o { - b"q" | b"Q" | b"cm" | b"Tf" | b"gs" => { + b"q" | b"Q" | b"cm" | b"Tf" | b"gs" | b"w" | b"M" | b"j" | b"J" => { self.graphics_state(doc, scope, &op); Some(op) } @@ -571,7 +599,12 @@ impl Walk { Some(op) } b"cs" | b"CS" => { - if op.name(0) != Some(b"Pattern") { + // A Pattern space is told by the name `/Pattern` alone: one named through the + // resources' `/ColorSpace` clears the pattern, which the app's renderer keeps and + // Poppler doesn't (`cs` resets the colour, ISO 32000-2 §8.6.8). Looking the name up + // at each `cs` wants a cache for the page: a `/ColorSpace` too large for the + // document to keep is parsed again at each read. + if op.last_name() != Some(b"Pattern") { if o == b"cs" { self.gs.fill_pattern = None; } else { @@ -598,8 +631,12 @@ impl Walk { let o = op.op.as_slice(); let pts: Vec<(f64, f64)> = match o { b"re" => op.nums::<4>().map(|[x, y, w, h]| vec![(x, y), (x + w, y), (x, y + h), (x + w, y + h)]).unwrap_or_default(), - b"h" | b"W" | b"W*" => Vec::new(), - _ => op.operands.iter().filter_map(Object::as_f64).collect::>().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).collect(), + // Each takes its points from the end, as the renderer reads them: a surplus before + // them is ignored, not read as a point. + b"m" | b"l" => op.nums::<2>().map(|[x, y]| vec![(x, y)]).unwrap_or_default(), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![(x1, y1), (x2, y2), (x3, y3)]).unwrap_or_default(), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![(x1, y1), (x2, y2)]).unwrap_or_default(), + _ => Vec::new(), }; for (x, y) in pts { let (u, v) = self.gs.ctm.apply(x, y); @@ -612,11 +649,35 @@ impl Walk { self.path.push(op); } + /// The box a stroke of the current path can paint, grown from its construction box `b` by + /// what the pen adds: half the line width (a width of 0 is one device pixel, taken as one + /// unit) scaled by the CTM's largest stretch, and never less than half a unit of page space + /// (a renderer paints a hairline, or a line a CTM shrinks, about a device pixel wide); times + /// the miter limit when the path has joins and they are mitred (a rectangle has four), or + /// √2 for a projecting square cap (its outer corner). Caps and joins never reach further, so + /// a hit test on it is conservative, and a horizontal or vertical line no longer has a + /// zero-area box. + fn stroke_box(&self, b: [f64; 4]) -> [f64; 4] { + let m = self.gs.ctm.0; + let scale = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2] + m[3] * m[3]).sqrt(); + let joins = self.path.iter().filter(|op| matches!(op.op.as_slice(), b"l" | b"c" | b"v" | b"y" | b"h" | b"re")).count() >= 2 + || self.path.iter().any(|op| op.op.as_slice() == b"re"); + let miter = if joins && self.gs.line_join == 0 { self.gs.miter_limit.max(1.0) } else { 1.0 }; + let reach = if self.gs.line_cap == 2 { miter.max(std::f64::consts::SQRT_2) } else { miter }; + let pad = (self.gs.line_width.max(1.0) / 2.0 * scale).max(0.5) * reach; + [b[0] - pad, b[1] - pad, b[2] + pad, b[3] + pad] + } + /// A painting operator ends the path; one under a region is removed (or kept as a clip). fn paint_path(&mut self, scope: &mut Scope<'_>, op: Op, piece: &mut Piece) { + let strokes = matches!(op.op.as_slice(), b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*"); + // A `W`/`W*` makes the path as constructed the clip; the stroke expansion only describes + // what the painting itself covers. + let path_box = self.path_box.take(); + let bbox = path_box.map(|b| if strokes { self.stroke_box(b) } else { b }); let body = std::mem::take(&mut self.path); - let bbox = self.path_box.take(); let clips = std::mem::replace(&mut self.clip, false); + let pending = std::mem::take(&mut self.pending); let hit = scope.mode == Mode::Apply && op.op.as_slice() != b"n" && bbox.is_some_and(|b| scope.hits(b)); if !hit { // The paint survives, so the pattern it draws with is still in use. `n` paints @@ -626,22 +687,30 @@ impl Walk { let fills = matches!(o, b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*"); uses_patterns(&mut self.out, &self.gs, fills, strokes); piece.ops.extend(body); + piece.ops.extend(pending); piece.ops.push(op); return; } let b = bbox.unwrap_or_default(); piece.changed = true; scope.report.paths_removed += 1; - if scope.covered(b) && clips { - // Keep the clipping, drop the painting. - piece.ops.extend(body); - piece.ops.push(Op::new("n", vec![])); + if scope.covered(path_box.unwrap_or(b)) && clips { + // Everything this clip lets through is under a mark, so an empty clip hides the same + // content. The original outline is confidential and must not stay in the stream. + piece.ops.extend(pending); + for o in ["re", "W", "n"] { + let operands = if o == "re" { vec![num(0.0); 4] } else { vec![] }; + piece.ops.push(Op::new(o, operands)); + } + } else { + piece.ops.extend(pending); } // Partial vector redaction must not retain underlying drawing commands // behind clipping (F-008). Refuse partial clip retention and remove the path. } fn graphics_state(&mut self, doc: &Document, scope: &mut Scope<'_>, op: &Op) { + let last = op.nums::<1>().map(|[v]| v); match op.op.as_slice() { b"q" => self.stack.push((self.gs.clone(), self.tm, self.tlm)), b"Q" => { @@ -655,11 +724,23 @@ impl Walk { } } b"Tf" => self.select_font(doc, scope, op), + // The operand before the operator, as the renderer reads it: `1 12 w` is a width of 12. + b"w" => self.gs.line_width = last.unwrap_or(self.gs.line_width), + b"M" => self.gs.miter_limit = last.unwrap_or(self.gs.miter_limit), + b"j" => self.gs.line_join = last.map_or(self.gs.line_join, |j| j as i64), + b"J" => self.gs.line_cap = last.map_or(self.gs.line_cap, |j| j as i64), _ => { - if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { + if let Some((font, vertical, size)) = op.last_name().and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); // As for `Tf`: a Type 3 font without /Resources paints through this scope's. - self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); + self.gs.type3_inherits = op.last_name().is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); + } + if let Some(state) = op.name(0).and_then(|n| self.ext_gstates.get(n)).and_then(|s| doc.resolve(s).as_dict().cloned()) { + let num = |key: &[u8]| state.get(key).and_then(|v| doc.resolve(v).as_f64()); + self.gs.line_width = num(b"LW").unwrap_or(self.gs.line_width); + self.gs.miter_limit = num(b"ML").unwrap_or(self.gs.miter_limit); + self.gs.line_join = num(b"LJ").map_or(self.gs.line_join, |j| j as i64); + self.gs.line_cap = num(b"LC").map_or(self.gs.line_cap, |j| j as i64); } } } @@ -864,7 +945,8 @@ fn clean_replacements( if !op.is("BDC") || !affected.contains(&op.span.start) { continue; } - let property = match op.operands.get(1) { + // The property list is the last operand, as the renderer reads it. + let property = match op.back(0) { Some(Object::Name(name)) => properties.get(name).map(|p| doc.resolve(p)), Some(p) => Some(doc.resolve(p)), None => None, @@ -874,7 +956,7 @@ fn clean_replacements( for key in [&b"ActualText"[..], b"Alt", b"E", b"T"] { stripped |= dict.remove(key).is_some(); } - if stripped && let Some(property) = op.operands.get_mut(1) { + if stripped && let Some(property) = op.operands.last_mut() { *property = Object::Dict(dict); *changed = true; report.replacement_text += 1; @@ -1071,7 +1153,32 @@ fn xobject( } } Some(b"Form") => { - let m = s.dict.get(b"Matrix").and_then(|m| Matrix::from_operands(doc.resolve(m).as_array()?)).unwrap_or_default(); + // Read as the proof reads it, entries by reference included: read as the identity, a + // form moved onto a mark would be tested against the wrong place and never walked. + let m = match crate::verify::matrix_of(doc, &s.dict) { + Ok(m) => m, + // Not six numbers given directly or through one reference. Removing hidden layers alone never asks where content lands, so + // the form is walked as it stands. + Err(()) if scope.everywhere() && scope.hidden_text.is_none() => Matrix::IDENTITY, + // Otherwise where the form draws, and so whether it reaches a mark or leaves the + // page, can't be told. Apply refuses the page, and the scans count it, as for a + // form past the page's budget below. + Err(()) => { + if scope.mode == Mode::Apply { + out.error = Some(CosError::Syntax { + offset: 0, + detail: + "a form XObject's /Matrix isn't six numbers given directly or through one reference, so where it draws can't be told" + .into(), + }); + } else if scope.everywhere() { + scope.layer_blocks += 1; + } else { + out.residue += 1; + } + return None; + } + }; let fctm = m.then(&gs.ctm); let bbox = s.dict.get(b"BBox").and_then(|b| { let v: Vec = doc.resolve(b).as_array()?.iter().filter_map(Object::as_f64).collect(); @@ -1150,6 +1257,7 @@ fn xobject( out.residue += inner.residue; out.vertical |= inner.vertical; out.ambiguous |= inner.ambiguous; + out.unparsed |= inner.unparsed; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); // A Type 3 glyph without resources shown inside the form may paint with the page's diff --git a/crates/redact/src/interp/layers.rs b/crates/redact/src/interp/layers.rs index 88d1c4e677..7b8ef82120 100644 --- a/crates/redact/src/interp/layers.rs +++ b/crates/redact/src/interp/layers.rs @@ -20,7 +20,7 @@ impl<'a> Scope<'a> { // groups. A membership dictionary's default policy is AnyOn. let d = doc.resolve(o); let Some(d) = d.as_dict() else { return false }; - if d.name(b"Type") != Some(b"OCMD") { + if !has_type(doc, d, b"OCMD") { return false; } let groups: Vec = match d.get(b"OCGs").map(|g| (*doc.resolve(g)).clone()) { @@ -102,7 +102,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let complete = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => { + Object::Dict(d) if has_type(doc, d, b"OCG") => { if let Some(r) = reference && hidden_layers.contains(&r) { @@ -144,6 +144,11 @@ impl<'a> Scope<'a> { } *budget -= 1; let reference = expression.as_ref(); + // A reference to a reference is no listed group: the renderer rejects the alias, so it + // is not taken as visible because the object it names isn't hidden. + if reference.is_some_and(|r| matches!(*doc.get(r), Object::Ref(_))) { + return None; + } if let Some(r) = reference && !path.insert(r) { @@ -151,7 +156,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let result = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), Object::Array(items) if (2..=4097).contains(&items.len()) => { let operator = match items.first() { Some(Object::Name(n)) => n.as_slice(), @@ -195,6 +200,9 @@ impl<'a> Scope<'a> { if self.hidden_layers.contains(&r) { return true; } + if matches!(*doc.get(r), Object::Ref(_)) { + return true; + } if !path.insert(r) { return true; } @@ -202,7 +210,7 @@ impl<'a> Scope<'a> { let resolved = doc.resolve(expression); let found = match &*resolved { Object::Array(items) => items.iter().any(|item| self.expression_mentions_hidden(doc, item, depth + 1, path, budget)), - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), Object::Dict(_) => true, _ => false, }; @@ -225,8 +233,10 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve match op.op.as_slice() { b"BDC" | b"BMC" => { let this = op.is("BDC") - && op.name(0) == Some(b"OC") - && match op.operands.get(1) { + // The tag and the property list are the last two operands, as the renderer + // reads them: `(x) /OC /MC0 BDC` is a layer's block too. + && op.name_back(1) == Some(b"OC") + && match op.back(0) { Some(Object::Name(n)) => props.get(n).is_some_and(|o| scope.layer_hidden(doc, o)), Some(o) => scope.layer_hidden(doc, o), None => false, @@ -254,3 +264,9 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve } (out, removed) } + +/// Is this dictionary's `/Type` the given name? An indirect `/Type` is followed, as a viewer +/// follows it: a group or membership dictionary must not escape classification through one. +fn has_type(doc: &Document, d: &Dict, name: &[u8]) -> bool { + d.get(b"Type").is_some_and(|t| crate::resolve_chain(doc, t).as_name() == Some(name)) +} diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 39089601ec..254d0c7a06 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -9,8 +9,9 @@ use super::{ /// by a form without /Resources of its own, which draws through `xo` too. Such forms are read /// within `budget`, what is left of the page's allowance ([`resource_ops`]). Fails, as the cleanup /// of the fonts and properties these forms draw with does, when that can't be told: such a form -/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget`, the walk -/// passes 2,000,000 operators, or the parser can't read all of such a form or of `content`. +/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget` (for the +/// fonts' cleanup, an allowance of its own when no Type 3 font is listed), the walk passes +/// 2,000,000 operators, or the parser can't read all of such a form or of `content`. /// Corrupt data in such a form is read, tolerantly, up to the damage, which may be nothing. pub(crate) fn prune(doc: &Document, budget: &mut DecodeBudget, xo: &mut Dict, replaced: &[Vec], content: &[Vec]) -> Result<(), CosError> { if replaced.is_empty() { @@ -357,7 +358,7 @@ fn prune_retired_properties(doc: &Document, res: &mut Dict, names: &[Vec], c // Each is one page's content or one form's, with the allowance of a page. for op in resource_ops(doc, &mut DecodeBudget::page(), res, content)? { if matches!(op.op.as_slice(), b"BDC" | b"DP") - && let Some(name) = op.name(1) + && let Some(name) = op.last_name() { used.insert(name.to_vec()); } @@ -473,7 +474,8 @@ pub(super) fn clip_out(rects: &[[f64; 4]], ctm: &Matrix, around: [f64; 4]) -> Op /// The operators of `content`, and of every form without /Resources of its own that it draws /// through `res`, nested ones too: what draws with these resources. Each such form is read once, /// with its `/DecodeParms` given by reference, its decoded bytes charged to `budget`, so the -/// cleanups of one page that walk the same forms spend one allowance between them. An error +/// cleanups of one page that walk the same forms spend one allowance between them (all but the +/// fonts' cleanup when no Type 3 font is listed, which has one of its own). An error /// when that can't all be read: the parser can't read all of `content` or of such a form, the walk /// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. Operators are /// read one at a time, so the walk stops at one past the limit rather than holding them all. @@ -520,7 +522,7 @@ pub(crate) fn prune_properties(doc: &Document, budget: &mut DecodeBudget, res: & let ops = resource_ops(doc, budget, res, content)?; let names: Vec> = properties.iter().map(|(n, _)| n.to_vec()).collect(); for name in names { - if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.name(1) == Some(&name)) + if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.last_name() == Some(&name)) && let Some(value) = properties.remove(&name) { retired.push(value); diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 7c1a65f55a..819cae99cd 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -58,6 +58,12 @@ pub enum RedactError { NothingToApply, #[error("page {0} has content that can't be read, so it can't be redacted safely")] Unreadable(usize), + /// The page tree and the renderer count a different number of pages (a page repeated in + /// `/Kids`, say), so a mark's page can't be matched to the page the renderer drew. + #[error( + "the document's page tree and its rendered pages disagree on how many pages there are, so the redaction could not be verified; nothing was changed" + )] + AmbiguousPages, /// Redact annotations with no readable area (neither usable `/QuadPoints` nor a usable `/Rect`). /// Skipping them would leave the text under them in the file while the others report success. #[error( @@ -247,6 +253,19 @@ pub(crate) fn annots_of(doc: &Document, page: &Dict) -> Vec { page.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default() } +/// An annotation's `/Subtype`, resolved: it may be an indirect reference (ISO 32000-2 §7.3.10), +/// and the audit reads it that way, so a mark it reports must not be one [`apply`] passes over. +pub(crate) fn subtype(doc: &Document, d: &Dict) -> Option> { + d.get(b"Subtype").and_then(|s| doc.resolve(s).as_name().map(<[u8]>::to_vec)) +} + +/// The pop-up `d` names (`/Popup`), when it is one: it goes with `d`. Anything else it might name +/// (a mark on another page, say) is left alone. +fn named_popup(doc: &Document, d: &Dict) -> Option { + let r = d.reference(b"Popup")?; + doc.get(r).as_dict().is_some_and(|p| subtype(doc, p).as_deref() == Some(b"Popup")).then_some(r) +} + /// Every annotation the pages list. Compared before and after a removal, what is no longer listed /// leaves the structure tree too ([`tags::drop_object_refs`]). pub(crate) fn listed_annotations(doc: &Document) -> HashSet { @@ -267,14 +286,14 @@ pub(crate) fn marks_checked(doc: &Document) -> (Vec, Vec) { let Some(r) = a.as_ref() else { // A mark written in place has no object to find, apply or remove: refuse it // rather than apply the others as if it were not there. - if doc.resolve(&a).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Redact")) { + if doc.resolve(&a).as_dict().is_some_and(|d| subtype(doc, d).as_deref() == Some(b"Redact")) { unreadable.push(pi); } continue; }; let obj = doc.get(r); let Some(d) = obj.as_dict() else { continue }; - if d.name(b"Subtype") != Some(b"Redact") { + if subtype(doc, d).as_deref() != Some(b"Redact") { continue; } let mut rects: Vec<[f64; 4]> = quad_points(doc, d) @@ -580,18 +599,41 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co let annots = annots_of(doc, &p.dict); let mut kept = Vec::with_capacity(annots.len()); for a in &annots { - let Some(r) = a.as_ref() else { - kept.push(a.clone()); - continue; - }; - let obj = doc.get(r); + // An entry written in place is read as one listed by reference: its `/IRT` or + // `/Parent` keeps a removed comment reachable all the same, and it has its own text. + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { kept.push(a.clone()); continue; }; - let is_reply = - d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); - let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); + let is_reply = d.reference(b"IRT").is_some_and(|x| removed_everywhere.contains(&x)) && subtype(doc, d).as_deref() != Some(b"Widget"); + let is_popup = d.reference(b"Parent").is_some_and(|x| removed_everywhere.contains(&x)); + let Some(r) = a.as_ref() else { + if is_reply || is_popup { + grew = true; + if is_reply { + report.annotations += 1; + } + // Its appearance can be shared with an annotation that stays elsewhere. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } + // It has no reference to remove elsewhere; the pop-up it names has. + if let Some(p) = named_popup(doc, d) { + removed_everywhere.insert(p); + } + } else { + kept.push(a.clone()); + } + continue; + }; if removed_everywhere.contains(&r) || is_reply || is_popup { // A reply or pop-up removed here can share its appearance with an annotation // that stays on an unselected page, like one removed under a mark: the @@ -638,6 +680,30 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co report.fields += 1; } } + // A widget written in place on another page names its deleted field through `/Parent`, which + // would keep the field and its value in the file: it goes with the field. + if !doomed_fields.is_empty() { + for p in pdfcraft_model::pages(doc) { + let annots = annots_of(doc, &p.dict); + let kept: Vec = annots + .iter() + .filter(|a| { + a.as_ref().is_some() || !matches!(&**a, Object::Dict(d) if d.reference(b"Parent").is_some_and(|f| doomed_fields.contains(&f))) + }) + .cloned() + .collect(); + if kept.len() != annots.len() { + report.annotations += annots.len() - kept.len(); + doc.update_dict(p.obj, |d| { + if kept.is_empty() { + d.remove(b"Annots"); + } else { + d.set(b"Annots".to_vec(), Object::Array(kept)); + } + })?; + } + } + } // 3a. A tagged PDF's structure tree still points at the removed comments and widgets // (`/OBJR`, #2874): drop those references, or a full rewrite keeps them reachable, the @@ -791,8 +857,10 @@ fn redact_page_content( } // An inline image the renderer reads no image for (no `EI` it accepts, or parameters it // can't read): where the page's operators resume after it is a guess, so neither removal - // nor the verify pass can be trusted on this page. - if out.ambiguous { + // nor the verify pass can be trusted on this page. Likewise bytes the parser can't read in + // what the walk read, the page's content or a form under a mark: text they hide is neither + // found nor removed, and other readers may draw it. + if out.ambiguous || out.unparsed { return Err(RedactError::Unreadable(pi + 1)); } doomed_xobjects.extend_from_slice(&out.doomed); @@ -883,30 +951,66 @@ fn remove_page_annotations( page_marks: &[&Mark], ) -> Result<(), RedactError> { let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; - // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). + // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). An entry + // written in place (a dictionary in `/Annots`, not a reference) is judged like any other: its + // `/Contents` and `/IRT` stay in the file for as long as the page lists it. let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); + /// What decides an entry's fate, read once. + struct Entry { + r: Option, + widget: bool, + irt: Option, + parent: Option, + popup: Option, + } + let entries: Vec> = annots + .iter() + .map(|a| { + let obj = doc.resolve(a); + let d = obj.as_dict()?; + Some(Entry { + r: a.as_ref(), + widget: subtype(doc, d).as_deref() == Some(b"Widget"), + irt: d.reference(b"IRT"), + parent: d.reference(b"Parent"), + popup: named_popup(doc, d), + }) + }) + .collect(); let mut removed: Vec = Vec::new(); - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - let obj = doc.get(r); + // Entries written in place that go, by position in `/Annots`. + let mut inline_gone = vec![false; annots.len()]; + for (i, a) in annots.iter().enumerate() { + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { continue }; - let subtype = d.name(b"Subtype").unwrap_or(b""); - if subtype == b"Redact" { - if page_marks.iter().any(|m| m.obj == r) { + let r = a.as_ref(); + let kind = subtype(doc, d).unwrap_or_default(); + if kind == b"Redact" { + // A mark written in place was refused before anything changed. + if let Some(r) = r + && page_marks.iter().any(|m| m.obj == r) + { removed.push(r); } continue; } - if subtype == b"Popup" { + if kind == b"Popup" { continue; } if rect_of(doc, d.get(b"Rect")).is_some_and(|b| rects.iter().any(|x| pdfcraft_content::overlaps(*x, b, 0.0))) { - removed.push(r); - if subtype == b"Widget" { - if let Some((_, field)) = widget_owner.iter().find(|(w, _)| *w == r) - && !doomed_fields.contains(field) + match r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + if kind == b"Widget" { + // A widget written in place has no reference of its own; its `/Parent` is the field, + // even one no referenced widget belongs to (it then has no entry in `widget_owner`). + // Step 3 deletes only what the form lists as a field, so any `/Parent` is safe here. + let owner = r.and_then(|r| widget_owner.iter().find(|(w, _)| *w == r).map(|(_, f)| *f)).or_else(|| d.reference(b"Parent")); + if let Some(field) = owner + && !doomed_fields.contains(&field) { - doomed_fields.push(*field); + doomed_fields.push(field); } } else { report.annotations += 1; @@ -917,29 +1021,25 @@ fn remove_page_annotations( // those: a reply placed elsewhere would keep its parent — the comment and the QuadPoints // of the redacted words — in the file through its `/IRT` (#2659). Each pass removes at // least one more annotation of this page or stops, so it ends. - let replies: Vec<(ObjRef, ObjRef)> = annots - .iter() - .filter_map(Object::as_ref) - .filter_map(|r| { - let obj = doc.get(r); - let d = obj.as_dict()?; - (d.name(b"Subtype") != Some(b"Widget")).then_some(())?; - Some((r, d.get(b"IRT").and_then(Object::as_ref)?)) - }) - .collect(); loop { - let before = removed.len(); - for (r, parent) in &replies { - if !removed.contains(r) && removed.contains(parent) { - removed.push(*r); - report.annotations += 1; + let mut grew = false; + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone || e.widget || !e.irt.is_some_and(|p| removed.contains(&p)) { + continue; } + match e.r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + report.annotations += 1; + grew = true; } - if removed.len() == before { + if !grew { break; } } - removed_everywhere.extend(removed.iter().copied()); // A removed annotation's appearance streams can be shared with a kept one on another page, // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. for reference in &removed { @@ -959,24 +1059,53 @@ fn remove_page_annotations( } } } - // Pop-ups dropped with their parent below count too: a reply on another page may answer one. - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - if doc.get(r).as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) { - removed_everywhere.insert(r); + // An entry written in place that goes keeps no reference, but its appearances count as well. + for (i, a) in annots.iter().enumerate() { + if *appearance_scan_limit { + break; + } + if let (true, Some(annotation)) = (inline_gone[i], doc.resolve(a).as_dict()) { + match appearance_refs(doc, annotation) { + Some(references) => removed_appearances.extend(references), + None => *appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } } } + // Pop-ups go with their annotation: those that name it (`/Parent`), and those it names + // (`/Popup`), the only link an annotation written in place can have to its pop-up. + let mut going: HashSet = removed.iter().copied().collect(); + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone { + going.extend(e.popup); + } + } + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + if e.parent.is_some_and(|p| removed.contains(&p)) { + match e.r { + Some(r) => { + going.insert(r); + } + None => inline_gone[i] = true, + } + } + } + // Pop-ups dropped with their parent count too: a reply on another page may answer one. + removed_everywhere.extend(going.iter().copied()); let kept: Vec = annots .into_iter() - .filter(|a| { - let Some(r) = a.as_ref() else { return true }; - if removed.contains(&r) { - return false; - } - // Pop-ups of removed annotations go too. - let obj = doc.get(r); - !obj.as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) + .enumerate() + .filter(|(i, a)| match a.as_ref() { + Some(r) => !going.contains(&r), + None => !inline_gone[*i], }) + .map(|(_, a)| a) .collect(); doc.update_dict(page.obj, |d| { if kept.is_empty() { @@ -1014,6 +1143,19 @@ pub fn clear_marks(doc: &mut Document, pages: Option<&[usize]>) -> Result std::sync::Arc { + let mut o = doc.resolve(o); + for _ in 0..16 { + if !matches!(*o, pdfcraft_cos::Object::Ref(_)) { + break; + } + o = doc.resolve(&o); + } + o +} + /// The name to save a document under after [`apply`]: `_Redacted` before the extension, as /// Acrobat appends it ("contract.pdf" gives "contract_Redacted.pdf"). The `.pdf` extension is /// matched in any letter case. A name that already ends in `_Redacted` keeps it once. diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index 0d4ee2e18a..61c7cedda4 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -479,12 +479,13 @@ fn hidden_layers(doc: &Document) -> Vec { o.and_then(|g| doc.resolve(g).as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() }; let all = refs(oc.get(b"OCGs")); - let (on, off) = (refs(d.get(b"ON")), refs(d.get(b"OFF"))); - let mut hidden: Vec = if d.name(b"BaseState") == Some(b"OFF") { - all.iter().copied().filter(|g| !on.contains(g)).collect() - } else { - off.into_iter().filter(|g| all.contains(g)).collect() - }; + // sets: the lists are the document's, and a filter over every group would scan them each time + let (on, off): (std::collections::HashSet, std::collections::HashSet) = + (refs(d.get(b"ON")).into_iter().collect(), refs(d.get(b"OFF")).into_iter().collect()); + // The default state is /BaseState, then /ON turns groups on and /OFF turns them off, so a + // group both lists name ends up off (§8.11.4.3, Table 101) — the viewer reads it the same way. + let base_off = d.get(b"BaseState").map(|b| crate::resolve_chain(doc, b)).is_some_and(|b| b.as_name() == Some(b"OFF".as_slice())); + let mut hidden: Vec = all.iter().copied().filter(|g| off.contains(g) || (base_off && !on.contains(g))).collect(); // Each rule, layer list and layer is read once, however often the rules name them: many rules // can share one long list. let known: std::collections::HashSet = all.iter().copied().collect(); @@ -659,6 +660,17 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re if let Some(error) = out.error { return Err(error.into()); } + // Bytes the parser can't read, or an inline image whose end is a guess: the walk reads past + // them, so what it reads around them may not be what other readers draw (hidden text in a + // broken `TJ` is neither found nor removed). As for content that doesn't decode, count the + // page so the removal runs and refuses it. + if out.unparsed || out.ambiguous { + if write { + return Err(RedactError::Unreadable(pi + 1)); + } + total += 1; + continue; + } let blocks = scope.layer_blocks; let glyphs = scope.report.glyphs; let retained_for_page = std::mem::take(&mut scope.retained_off_layers); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 648c0c29e5..03573a9743 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -35,6 +35,14 @@ impl Marked { /// left are taken as changed (their descriptions go) rather than compared. const MAX_COMPARED: usize = 1 << 28; +/// `f` of `o`, or of the object it refers to, without copying a direct one. +fn with_held(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> R) -> R { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + o => f(o), + } +} + fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { let mut out = Marked::default(); // Open sequences: the MCID with where its content starts in `ops` and the drawing operators @@ -49,14 +57,22 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { e.1 |= drawn > drawn_before; } }; + // The named property lists, read once, and each name's MCID, read once: a stream can name one + // at every `BDC` of thousands, and reading an object again can parse it again, all of it (the + // document keeps an object larger than its cache only until it reads another). A direct one + // is read where it is. + let held = resources.get(b"Properties").filter(|p| matches!(p, Object::Ref(_))).map(|p| doc.resolve(p)); + let named = held.as_deref().or_else(|| resources.get(b"Properties")).and_then(Object::as_dict); + let mut mcids: HashMap, Option> = HashMap::new(); for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { - let id = match op.operands.get(1) { - Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { - let props = doc.resolve(p); - let prop = doc.resolve(props.as_dict()?.get(n)?); - prop.as_dict()?.int(b"MCID") + // The property list is the last operand, as the renderer reads it. + let id = match op.back(0) { + Some(Object::Name(n)) => *mcids.entry(n.to_vec()).or_insert_with(|| { + let prop = named?.get(n)?; + // A named property list may give its MCID by reference. + with_held(doc, prop, |prop| prop.as_dict()?.get(b"MCID").and_then(|id| with_held(doc, id, Object::as_int))) }), Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, @@ -86,9 +102,7 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { } let draws = match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().and_then(Object::as_string).is_some_and(|s| !s.bytes.is_empty()), - b"TJ" => { - op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))) - } + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))), b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s" | b"sh" | b"Do" | b"BI" => true, _ => false, }; @@ -139,7 +153,7 @@ fn mcid_of(doc: &Document, o: &Object, page: Option, target: ObjRef, str Object::Dict(m) if m.name(b"Type") == Some(b"MCR") => { let pg = m.get(b"Pg").and_then(Object::as_ref).or(page); let stm = m.get(b"Stm").and_then(Object::as_ref); - (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.int(b"MCID")).flatten() + (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.get(b"MCID").and_then(|id| doc.resolve(id).as_int())).flatten() } _ => None, } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..43b6caaadc 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -837,6 +837,22 @@ fn vectors_are_removed_or_clipped_and_inline_images_go() { assert!(c.contains("150 150 m"), "the line elsewhere stays"); } +/// A horizontal stroke's construction box has no area, so it overlapped nothing and stayed under +/// the mark. The painted stroke is as wide as the line: a 12 pt line centred 5 pt above the mark +/// reaches into it and goes; a 1 pt line there does not and stays. +#[test] +fn a_stroke_is_removed_by_its_painted_width() { + let src = b"12 w 10 60 m 100 60 l S 1 w 10 60 m 100 60 l S 4 w 10 50 m 100 50 l S"; + let mut doc = one_page(src, "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 2, "{r:?}"); + let c = content(&doc, 0); + assert!(!c.contains("12 w\n10 60 m"), "the wide line reaches the mark: {c}"); + assert!(c.contains("1 w\n10 60 m\n100 60 l\nS"), "the thin line beside it stays: {c}"); + assert!(!c.contains("10 50 m"), "the line through the mark goes: {c}"); +} + /// A tiling pattern's cell is a content stream of its own and can draw text. Redacting a shape /// filled with such a pattern removed the fill from the page but left the pattern in the page's /// resources, so its cell -- with the text that was under the mark -- stayed in the saved file @@ -1057,6 +1073,20 @@ fn partially_covered_vector_paths_do_not_leak_drawing_commands() { assert!(c.contains("300 300 m\n350 350 l\nS"), "unaffected vector path must be kept: {c}"); } +#[test] +fn a_covered_clip_and_paint_path_leaves_an_empty_clip_not_its_outline() { + // The outline of a fully covered path that clips and paints is confidential geometry: only + // an empty clip may stay, and what follows is hidden as the covered clip would have hidden it. + let src = b"q 100 100 m 150 180 l 200 100 l h W f 300 300 50 50 re f Q"; + let mut doc = one_page(src, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 210.0, 190.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("150 180 l") && !c.contains("100 100 m"), "outline must not stay: {c}"); + assert!(c.contains("0 0 0 0 re\nW\nn"), "an empty clip stays: {c}"); +} + /// A chain of nested forms deeper than the pass can walk was removed from the page, taking /// whatever else it drew with it even when nothing in it was marked. The limit now reaches as /// deep as the renderer and Edit text read (50), so anything a reader can see can be walked; @@ -3297,31 +3327,42 @@ fn image_and_a_tail(catalog: &str, image: &str, image_dict: &str, tail: &[u8]) - ]) } -/// The refusal of the check of what content still draws through its resources (`resource_ops`). -fn unread_content_refusal(result: &Result) -> bool { - matches!(result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) - if detail == "redaction could not fully read the content that uses these resources") -} - -/// What the page draws can't be told when a content stream the redaction leaves as it was holds -/// bytes the parser can't read: the redaction of an image is refused, as it was already on a page -/// with fonts or properties to clean up. +/// What the page draws can't be told when content the redaction leaves as it was holds bytes the +/// parser can't read: the redaction of an image is refused. In a content stream of the page, the +/// walk reads past them and refuses the page itself (`Unreadable`). In a form without resources of +/// its own that lies away from the mark, which the walk doesn't enter, the check of what the page +/// still draws through its resources refuses it. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_the_redaction_of_an_image() { for (tail, refused) in [(&b"q Q )"[..], true), (b"q Q", false)] { let mut doc = image_and_a_tail("", "Im1", "", tail); mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - let result = apply(&mut doc, None); - if refused { - assert!(unread_content_refusal(&result), "{result:?}"); - } else { - assert_eq!(result.map(|r| r.images_removed).ok(), Some(1)); - } + let result = apply(&mut doc, None).map(|r| r.images_removed); + assert_eq!(result, if refused { Err(RedactError::Unreadable(1)) } else { Ok(1) }); } + let mut doc = image_and_a_tail("", "Im1", "", b"/Fm Do"); + let bbox = Object::Array(vec![0.into(), 0.into(), 300.into(), 100.into()]); + let form = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject")), (b"Subtype".to_vec(), Object::name("Form")), (b"BBox".to_vec(), bbox)]); + let form = doc.add(Object::Stream(Stream::from_raw(form, b"q Q )".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |d| { + if let Some(Object::Dict(res)) = d.get_mut(b"Resources") + && let Some(Object::Dict(xo)) = res.get_mut(b"XObject") + { + xo.set(b"Fm".to_vec(), Object::Ref(form)); + } + }) + .unwrap(); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail == "redaction could not fully inspect inherited form resources"), + "{result:?}" + ); } -/// Remove Hidden Information makes the same check: removing an off layer's image is refused on a -/// page whose drawing can't be told. +/// Remove Hidden Information refuses the same page: removing an off layer's image is refused on a +/// page whose drawing can't be told. Its walk reads every content stream, and refuses the page +/// itself. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3329,7 +3370,7 @@ fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() let mut doc = image_and_a_tail("/OCProperties << /OCGs [7 0 R] /D << /OFF [7 0 R] >> >>", "ImOC", "/OC 7 0 R", tail); let result = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); if refused { - assert!(unread_content_refusal(&result), "{result:?}"); + assert_eq!(result, Err(RedactError::Unreadable(1))); } else { assert!(result.unwrap().iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)); } @@ -3571,14 +3612,43 @@ fn what_content_draws_through_its_resources_is_read_within_its_allowance() { } } +/// An unused Type 3 font, listed as /T3 beside /F1 in the `/Font` of `holder`'s resources (a page's +/// or a form's), so that the font cleanup, which prunes only Type 3 fonts, charges its read of what +/// they draw to the page's allowance (without one it has an allowance of its own). +fn list_type3(doc: &mut Document, holder: ObjRef) { + let font = Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Font")), + (b"Subtype".to_vec(), Object::name("Type3")), + (b"FontBBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1000.into(), 1000.into()])), + (b"FontMatrix".to_vec(), Object::Array(vec![Object::Real(0.001), 0.into(), 0.into(), Object::Real(0.001), 0.into(), 0.into()])), + (b"CharProcs".to_vec(), Object::Dict(Dict::new())), + ])); + let font = Object::Ref(doc.add(font)); + let list = |d: &mut Dict| { + let Some(Object::Dict(res)) = d.get_mut(b"Resources") else { panic!("{holder:?} has resources") }; + let Some(Object::Dict(fonts)) = res.get_mut(b"Font") else { panic!("{holder:?} lists fonts") }; + fonts.set(b"T3".to_vec(), font); + }; + match &*doc.get(holder) { + Object::Stream(s) => { + let mut s = s.clone(); + list(&mut s.dict); + doc.set(holder, Object::Stream(s)); + } + _ => doc.update_dict(holder, list).unwrap(), + } +} + /// The cleanups after the content walk spend what the walk left of the page's allowance, one /// allowance between them. A form without resources of its own that crosses the mark is read by -/// the walk, then again by the check of what the page still draws and by the font cleanup: at -/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at -/// 50 MiB it goes on. +/// the walk, then again by the check of what the page still draws and by the font cleanup, which +/// charges the page's allowance when the page lists a Type 3 font: at 100 MiB, three reads are more +/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. With no Type 3 font +/// the font cleanup's read has an allowance of its own, so the page's pays for two reads of +/// 100 MiB, and the redaction goes on. #[test] fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", @@ -3588,19 +3658,70 @@ fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), ], ); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } +/// Text under the mark in a content stream that holds bytes the parser can't read, inside a `TJ` +/// array, is read past by the walk and would stay. The font cleanup's read of what the content +/// draws refuses such bytes in the streams and forms the walk leaves as they were, and makes that +/// read with any font listed, a Type 3 font or not (with none, on an allowance of its own). Here +/// the walk rewrites the text in the page's first stream and leaves the broken `TJ` as it was: +/// Apply Redactions refuses the page, whether that `TJ` is in the page's second stream or in a +/// form without resources of its own, and so does Remove Hidden Information. The cleanups run only +/// when the walk rewrites some of the page, and read what it rewrote as rewritten, so such bytes in +/// a stream it rewrites still go by, and so do those on a page it rewrites nothing of, unless the +/// document still lists a Type 3 font (`unlink_type3` then reads every page). +#[test] +fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { + use crate::sanitize::{Hidden, remove_hidden}; + let page_of = |second: &[u8]| { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET"), + stream("", second), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]) + }; + for tail in [ + &b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET"[..], + b"BT /F1 10 Tf 20 260 Td [(TAIL) } (SECRET)] TJ ET", + b"BT /F1 10 Tf 20 260 Td [(TAILSECRET) )] TJ ET", + b"BT /F1 10 Tf 20 260 Td [) (TAILSECRET)] TJ ET", + ] { + let mut doc = page_of(tail); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "{}", String::from_utf8_lossy(tail)); + } + let mut doc = one_page( + b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /FmBare Do", + "/XObject << /FmBare 7 0 R >>", + vec![stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET")], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "a form's text"); + let mut doc = page_of(b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET"); + doc.set(ObjRef::new(4, 0), Object::Stream(Stream::from_raw(Dict::new(), b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET".to_vec()))); + assert!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), "hidden text"); +} + /// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten /// form has resources of its own, so the page's cleanups don't follow it, and draws through them a /// form without any that crosses the mark: the walk reads that form, then the check of what the -/// rewritten form still draws and its font cleanup read it again. At 100 MiB, three reads are more -/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. +/// rewritten form still draws and its font cleanup (when it lists a Type 3 font) read it again. At +/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at 50 MiB +/// it goes on. With no Type 3 font the font cleanup's read has an allowance of its own, so the +/// page's pays for two reads of 100 MiB, and the redaction goes on. #[test] fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmText Do", @@ -3613,20 +3734,25 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); + if type3 { + list_type3(&mut doc, ObjRef::new(7, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } /// Removing hidden text spends what its content walk left of the page's allowance too. The page /// draws a form holding hidden text, which is rewritten, and a form without resources of its own: /// the walk reads the second, then the check of what the page still draws and the font cleanup -/// read it again. At 100 MiB, three reads are more than the page's 256 MiB, and the removal is -/// refused; at 50 MiB it goes on. +/// (when the page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the +/// page's 256 MiB, and the removal is refused; at 50 MiB it goes on. With no Type 3 font the font +/// cleanup's read has an allowance of its own, so the page's pays for two reads of 100 MiB, and the +/// removal goes on. #[test] fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { use crate::sanitize::{Hidden, remove_hidden}; - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmHidden Do /FmBare Do", @@ -3636,7 +3762,10 @@ fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); - assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } @@ -4038,6 +4167,8 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { (Some("AllOn"), false, Some("not_off"), false), (None, false, Some("malformed_not"), true), (None, false, Some("malformed_without_ocgs"), true), + // `/VE` names the hidden group through an alias object (10 0 R is a reference to 8 0 R). + (None, false, Some("alias"), true), ]; for (policy, malformed_policy, visibility, should_remove) in cases { let mut doc = one_page( @@ -4075,9 +4206,11 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { Some("malformed_without_ocgs") => { membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("UnknownOperator"), Object::Ref(ObjRef::new(8, 0))])) } + Some("alias") => membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("Or"), Object::Ref(ObjRef::new(10, 0))])), _ => {} } doc.set(ObjRef::new(9, 0), Object::Dict(membership)); + doc.set(ObjRef::new(10, 0), Object::Ref(ObjRef::new(8, 0))); let root = doc.root().unwrap(); doc.update_dict(root, |d| { d.set( @@ -6143,15 +6276,18 @@ fn unused_descriptions_in_shared_or_inherited_resources_are_retired() { mod descriptions; mod elsewhere; +mod form_matrix; mod hidden_scope; mod identification; mod inline_images; mod limits; mod metadata_fail_closed; mod metadata_matching; +mod operands_from_the_end; mod property_retirement; mod surviving_references; mod text_state; +mod unparsed; /// `[(FIRSTARR)] [(LASTARR)] TJ` is malformed (a surplus array operand), but the operator still /// draws only the one immediately before it (LASTARR, 7 glyphs, 35 pt wide from x = 10) -- not @@ -6554,6 +6690,131 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// A hairline under a shrinking CTM still paints about a device pixel: its pad must not shrink +/// with the CTM, or a mark on that ink misses it. +#[test] +fn a_hairline_under_a_shrinking_ctm_is_padded_by_half_a_unit() { + // Device line from (10, 10) to (50, 10); the mark lies on its painted pixel, above its axis. + let mut doc = one_page(b"0.01 0 0 0.01 0 0 cm 0 w 1000 1000 m 5000 1000 l S", "", vec![]); + mark(&mut doc, 0, &[[20.0, 10.2, 30.0, 10.4]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// One `re` has four mitred corners: it is padded by the miter limit like any joined path. +#[test] +fn a_rectangle_stroke_is_padded_for_its_mitred_corners() { + let mut doc = one_page(b"2 w 100 100 50 50 re S", "", vec![]); + // Past half the line width from the rectangle, inside the miter reach. + mark(&mut doc, 0, &[[155.0, 120.0, 160.0, 130.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// A projecting square cap's outer corner reaches lw/2 x sqrt(2) from the endpoint. Under a shear +/// it maps past a pad of lw/2 times the CTM's stretch (the review's example: corner at x = 12). +#[test] +fn a_square_cap_corner_under_a_shear_is_inside_the_pad() { + let mut doc = one_page(b"1 0 1 1 0 0 cm 2 J 2 w 0 0 m 10 0 l S", "", vec![]); + mark(&mut doc, 0, &[[11.8, 0.8, 11.9, 0.95]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1); +} + +/// A line width set between the path and its `S` still applies to that stroke, so the stroke is +/// tested at the width it is painted with; the width itself stays, the path goes. +#[test] +fn a_width_set_between_the_path_and_its_stroke_does_not_hide_it_from_the_test() { + let mut doc = one_page(b"10 60 m 100 60 l 12 w S 5 5 m 9 9 l S", "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1, "{r:?}"); + let c = content(&doc, 0); + assert!(!c.contains("10 60 m") && !c.contains("100 60 l"), "the wide line reaches the mark: {c}"); + assert!(c.contains("12 w"), "the width stays: {c}"); + assert!(c.contains("5 5 m"), "the line elsewhere stays: {c}"); +} + +/// The renderer takes a line-state operator's operand from the end: `1 12 w` strokes 12 points wide, +/// so the stroke is tested at that width. +#[test] +fn a_surplus_operand_before_the_line_width_does_not_hide_the_stroke() { + let mut doc = one_page(b"1 12 w 10 60 m 100 60 l S", "", vec![]); + mark(&mut doc, 0, &[[40.0, 45.0, 60.0, 55.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1, "{r:?}"); + assert!(!content(&doc, 0).contains("100 60 l")); +} + +/// One page drawing `SECRET` in the `/OC /L` block, with layer 7 0 R in the default +/// configuration `config` (objects 8 and up are `extra`). +fn layer_state_page(config: Vec<(&str, Object)>, resource: &str, extra: Vec>) -> Document { + let mut objects = vec![b"<< /Type /OCG /Name (Secret) >>".to_vec()]; + objects.extend(extra); + let mut doc = one_page(b"/OC /L BDC BT /F1 10 Tf 10 100 Td (SECRET) Tj ET EMC", &format!("/Properties << /L {resource} >>"), objects); + let config = Object::Dict(Dict::from_iter(config.into_iter().map(|(k, v)| (k.as_bytes().to_vec(), v)))); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([(b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), (b"D".to_vec(), config)])), + ); + }) + .unwrap(); + doc +} + +fn assert_layer_secret_removed(mut doc: Document) { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let counts = scan(&doc); + assert!(counts.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0), "{counts:?}"); + remove_hidden(&mut doc, &[Hidden::HiddenLayers]).unwrap(); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!pdfcraft_edit::text_lines(&saved, 0).unwrap().iter().any(|line| line.text.contains("SECRET"))); +} + +#[test] +fn an_indirect_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page(vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], "7 0 R", vec![b"/OFF".to_vec()])); +} + +#[test] +fn a_chained_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], + "7 0 R", + vec![b"9 0 R".to_vec(), b"/OFF".to_vec()], + )); +} + +#[test] +fn a_chained_membership_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"10 0 R".to_vec(), b"/OCMD".to_vec()], + )); +} + +#[test] +fn off_wins_over_on_when_the_base_state_is_off() { + assert_layer_secret_removed(layer_state_page( + vec![ + ("BaseState", Object::name("OFF")), + ("ON", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ("OFF", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ], + "7 0 R", + Vec::new(), + )); +} + +#[test] +fn a_membership_dictionary_with_an_indirect_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"/OCMD".to_vec()], + )); +} + /// An invisible OCR-style text layer drawn in a Type3 font (what the OCR output writes for /// characters beyond WinAnsi) is redacted like any other text: its glyphs are hit-tested by /// their real advances, removed, and the verifier passes. @@ -6589,6 +6850,78 @@ fn a_type3_text_layer_is_redacted_like_any_other() { assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 0, "nothing is left under the mark"); } +#[test] +fn a_mark_whose_subtype_is_an_indirect_name_is_applied() { + // `/Subtype` may be an indirect reference (ISO 32000-2 §7.3.10); the audit reads it that way. + // Passing over the mark would leave the text under it while the others report success. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [7 0 R] >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 20 Td (SECRET) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Annot /Subtype 8 0 R /Rect [10 10 90 40] >>".to_vec(), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + assert_eq!(marks(&doc).len(), 1, "the mark is found"); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.marks, 1); + assert!(r.glyphs >= 6, "the text under the mark went: {r:?}"); + assert!(marks(&doc).is_empty(), "the mark itself went"); +} + +#[test] +fn a_mark_written_in_place_with_an_indirect_subtype_is_refused() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype 5 0 R /Rect [10 10 70 40] >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[100.0, 100.0, 120.0, 120.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::UnreadableMark(1)))); +} + +#[test] +fn annotations_written_in_place_go_when_covered_or_hanging_off_a_removed_one() { + // `/Annots` may hold dictionaries rather than references. One under the mark, a reply to a + // removed highlight on the same page and on another, and a reply to that inline comment's + // pop-up must all go; an inline comment elsewhere stays. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [6 0 R \ + << /Type /Annot /Subtype /Text /Rect [30 15 40 25] /Contents (INLINE-UNDER) /Popup 7 0 R >> \ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [250 100 270 120] /Contents (INLINE-KEPT) >> 7 0 R] >>" + .to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [\ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (OTHER-PAGE-INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [200 200 220 220] /IRT 7 0 R /Contents (POPUP-INLINE-REPLY) >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Annot /Subtype /Highlight /Rect [20 20 60 30] /QuadPoints [20 30 60 30 20 20 60 20] /Contents (HIGHLIGHT-NOTE) >>".to_vec(), + b"<< /Type /Annot /Subtype /Popup /Rect [200 150 290 190] /Contents (INLINE-POPUP) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.annotations, 5, "the highlight, the inline comment and the three replies: {r:?}"); + let pages = pdfcraft_model::pages(&doc); + assert_eq!(annots_of(&doc, &pages[0].dict).len(), 1, "only the inline comment elsewhere stays"); + assert!(annots_of(&doc, &pages[1].dict).is_empty(), "the replies leave page 2"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + for gone in [&b"HIGHLIGHT-NOTE"[..], b"INLINE-UNDER", b"INLINE-REPLY", b"OTHER-PAGE-INLINE-REPLY", b"POPUP-INLINE-REPLY", b"INLINE-POPUP"] { + assert!(!bytes.windows(gone.len()).any(|w| w == gone), "{} survived the save", String::from_utf8_lossy(gone)); + } + assert!(bytes.windows(11).any(|w| w == b"INLINE-KEPT")); +} + #[test] fn a_rewritten_form_xobject_drops_the_source_text_of_added_content() { // A form drawing invisible text whose dictionary repeats it in /PCAdded. @@ -6615,6 +6948,49 @@ fn a_redacted_form_xobject_drops_the_source_text_of_added_content() { assert!(!bytes.windows(4).any(|w| w == b"1234"), "redacted text left in the saved file"); } +#[test] +fn an_inline_widget_takes_its_field_even_with_no_referenced_widget() { + // The field's only widget is written in place in `/Annots`, so no referenced widget names it: + // the field (and its value) must still go with the widget under the mark. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 6 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (FIELD-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.fields, 1, "{r:?}"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(12).any(|w| w == b"FIELD-SECRET"), "the field value survived the save"); +} + +#[test] +fn an_inline_widget_of_the_field_on_another_page_goes_with_it() { + // Two pages: the widget under the mark, and a sibling widget in place on the second page with the + // same `/Parent`. Deleting the field must not leave the sibling naming it. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [7 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 7 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [200 200 260 230] /Parent 7 0 R >>] >>" + .to_vec(), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (SIBLING-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + apply(&mut doc, None).unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(14).any(|w| w == b"SIBLING-SECRET"), "the field value survived through the sibling widget"); +} + /// Private data takes an image's prepress data too: its OPI link to the original file and its /// alternate images, which can be the unedited original. #[test] diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs new file mode 100644 index 0000000000..e4bb33608e --- /dev/null +++ b/crates/redact/src/tests/form_matrix.rs @@ -0,0 +1,131 @@ +//! A form's `/Matrix` places what it draws. The redaction walk reads it as the proof does, the +//! array and each entry direct or by reference: read as the identity, a form moved onto a mark +//! was tested against the wrong place, never walked, and its text stayed. + +use super::*; + +/// One page drawing a form with resources of its own, whose content shows SECRET at (20, 60) in a +/// `/BBox` of [0 0 300 100], with `matrix` in its dictionary; object 8 is `extra`. +fn moved_form(matrix: &str, extra: &[u8]) -> Document { + one_page( + b"/Fm Do", + "/XObject << /Fm 7 0 R >>", + vec![ + stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ), + extra.to_vec(), + ], + ) +} + +/// Over the top third of the 300 × 300 page, where a move of 200 puts the text. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +#[test] +fn a_form_moved_onto_a_mark_is_redacted_however_its_matrix_is_given() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 200]", &b"null"[..]), + ("/Matrix 8 0 R", b"[1 0 0 1 0 200]"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"200"), + ("/Matrix [8 0 R 0 0 1 0 200]", b"1"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix} with 8 0 obj {}", String::from_utf8_lossy(extra)); + } +} + +/// A `/Matrix` of null, given directly or by a reference to null, is no `/Matrix` (ISO 32000-2 +/// §7.3.9): the identity, under which the text lies below the mark and stays. +#[test] +fn a_form_whose_matrix_is_null_draws_where_it_stands() { + for (matrix, extra) in [("/Matrix null", &b"null"[..]), ("/Matrix 8 0 R", b"null")] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(0), "{matrix}"); + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[[0.0, 0.0, 300.0, 100.0]], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix}, marked where it stands"); + } +} + +/// A `/Matrix` that isn't six numbers: where the form draws can't be told, so neither can whether +/// it reaches the mark, and the page is refused. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_is_refused() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 (up)]", &b"null"[..]), + ("/Matrix [1 0 0 1 0]", b"null"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"(up)"), + ("/Matrix 8 0 R", b"(up)"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix} with 8 0 obj {}: {result:?}", + String::from_utf8_lossy(extra) + ); + } +} + +/// A reference to a reference, for the matrix or one of its entries: Poppler doesn't follow it (it +/// reads such a matrix as the identity, such an entry as 0), the app's renderer follows it and +/// reads the move, so where the form draws can't be told, and the page is refused, as for a matrix that isn't six numbers. Read as either, +/// a mark where the other reader shows the text would leave it in the file. +#[test] +fn a_form_whose_matrix_goes_through_a_reference_to_a_reference_is_refused() { + for (matrix, target) in [("/Matrix [1 0 0 1 0 8 0 R]", &b"200"[..]), ("/Matrix 8 0 R", b"[1 0 0 1 0 200]")] { + for mark_at in [MARK, [0.0, 0.0, 300.0, 100.0]] { + let form = stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form, b"9 0 R".to_vec(), target.to_vec()]); + mark(&mut doc, 0, &[mark_at], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix}, 8 0 obj 9 0 R, 9 0 obj {}, mark at {mark_at:?}: {result:?}", + String::from_utf8_lossy(target) + ); + } + } +} + +/// The scans count such a form as content that may remain: the verify pass under a mark, and +/// Remove Hidden Information's scan, whose removal then refuses the page. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_counts_in_the_scans() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + assert_eq!(under(&mut doc, 0, &[MARK]), 1); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenText], 1); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(matches!(&removed, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), "{removed:?}"); +} + +/// Removing hidden layers alone never asks where content lands, so such a form is walked as it +/// stands: nothing is counted or refused over it in a document without layers. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_leaves_removing_hidden_layers_alone() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenLayers], 0); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::NothingToApply)); +} + +/// Remove Hidden Information reads it the same way: text a matrix entry by reference moves off the +/// page is hidden text, and goes. +#[test] +fn text_a_matrix_entry_by_reference_moves_off_the_page_is_hidden_text() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 8 0 R]", b"1000"); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenText && *n > 0)), "{removed:?}"); +} diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs new file mode 100644 index 0000000000..0c802763e9 --- /dev/null +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -0,0 +1,340 @@ +//! An operator takes its operands from the end, as the renderer reads them, and a surplus before +//! them is ignored: `(x) /OC /MC0 BDC` marks a layer's block, `(x) /GS1 gs` selects `/GS1`. Read +//! from the front, the surplus stood in for the operands, and what they named went unseen. + +use super::*; + +/// Hidden-layer text in a block whose tag and property list follow a string: Remove Hidden +/// Information found nothing to remove, and the text stayed. +#[test] +fn a_hidden_layers_block_after_a_surplus_operand_is_removed() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = one_page( + b"(x) /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET", + "/Properties << /MC0 7 0 R >>", + vec![b"<< /Type /OCG /Name (Hidden) >>".to_vec()], + ); + doc.update_dict(ObjRef::new(1, 0), |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([ + (b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + (b"D".to_vec(), Object::Dict(Dict::from_iter([(b"OFF".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))]))]))), + ])), + ); + }) + .unwrap(); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + let c = content(&doc, 0); + assert!(!c.contains("LAYERSECRET") && c.contains("PUBLIC"), "{c}"); +} + +/// Paint under a mark named by `/ActualText` in a block after a surplus operand: the name, which +/// would still read the word, kept it. +#[test] +fn actual_text_after_a_surplus_operand_goes_with_the_paint_it_names() { + let mut doc = one_page(b"(x) /Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h f EMC", "", vec![]); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("SECRET"), "{c}"); +} + +/// A tagged paragraph whose `MCID` follows a surplus operand: its structure element kept the +/// removed text in `/ActualText` and `/Alt`. +#[test] +fn a_tag_after_a_surplus_operand_loses_what_redaction_removed() { + let content = b"(x) /P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the secret) /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); +} + +/// A property list still drawn with, named after a surplus operand, stays in the page's resources +/// when the redaction cleans them; read from the front, it was taken for unused and retired. +#[test] +fn a_property_list_named_after_a_surplus_operand_is_kept() { + let mut doc = one_page( + b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET (x) /Span /Pr1 BDC BT /F1 10 Tf 20 20 Td (KEPT) Tj ET EMC", + "/Properties << /Pr1 << /Lang (en-GB) >> >>", + vec![], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let props = res.get(b"Properties").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(props.contains(b"Pr1"), "{res:?}"); +} + +/// A font set only by an ExtGState selected after a surplus operand: read from the front, no font +/// was set, every glyph stood at the start of the line with no size, and a mark over the end of +/// the word removed nothing. +#[test] +fn a_font_from_an_extgstate_selected_after_a_surplus_operand_places_the_text() { + let mut doc = one_page(b"BT (x) /GS1 gs 20 260 Td (SECRET) Tj ET", "/ExtGState << /GS1 << /Font [5 0 R 10] >> >>", vec![]); + // Over the last two glyphs, E and T at x = 40 to 50. + mark(&mut doc, 0, &[[42.0, 255.0, 55.0, 275.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 2); +} + +/// `/Pattern cs` after a surplus operand, once the pattern's paint under the mark went: it selects +/// the Pattern space and keeps the pattern, so the fill after it still counts as painting with it +/// and the pattern stays. Read from the front, the surplus cleared the pattern, which was pruned. +/// A grey space clears it. +#[test] +fn a_pattern_space_selected_after_a_surplus_operand_keeps_the_pattern() { + for (space, pruned) in [("(x) /Pattern", 0), ("/Pattern", 0), ("(x) /DeviceGray", 1)] { + let mut doc = one_page( + format!("/Pattern cs /P0 scn 10 260 20 20 re f {space} cs 200 20 50 50 re f").as_bytes(), + "/Pattern << /P0 7 0 R >>", + vec![stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f")], + ); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, pruned), "{space}"); + } +} + +/// A Type 3 font selected after a surplus operand, outside the mark, whose glyph selects an +/// ExtGState, a colour space, a property list and a font after one too: the cleanups after the +/// walk took the font for undrawn and those for unused. All stay, with the glyph. +#[test] +fn a_type3_font_and_its_glyphs_resources_named_after_a_surplus_operand_are_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /T3 12 Tf 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] /Resources << /ExtGState << /GS0 << /CA 0.5 >> >> \ + /ColorSpace << /CS0 /DeviceGray >> /Properties << /P0 << >> >> /Font << /F0 5 0 R >> >> >>" + .to_vec(), + stream("", b"10 0 d0 (x) /GS0 gs (x) /CS0 cs (x) /Span /P0 BDC 0 0 10 10 re f EMC BT (x) /F0 1 Tf ET"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let fonts = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); + let font = fonts.get(b"T3").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()); + let font = font.unwrap_or_else(|| panic!("the drawn Type 3 font stays: {fonts:?}")); + let procs = font.get(b"CharProcs").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(procs.contains(b"a"), "its glyph stays: {font:?}"); + let font_res = font.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let states = font_res.get(b"ExtGState").map(|s| doc.resolve(s)).and_then(|s| s.as_dict().cloned()).unwrap_or_default(); + assert!(states.contains(b"GS0"), "the ExtGState its glyph selects stays: {font_res:?}"); + for (kind, name) in [(&b"ColorSpace"[..], &b"CS0"[..]), (b"Properties", b"P0"), (b"Font", b"F0")] { + let held = font_res.get(kind).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default(); + assert!(held.contains(name), "the {} its glyph uses stays: {font_res:?}", String::from_utf8_lossy(name)); + } +} + +/// A tagged paragraph whose `TJ` after a surplus operand still draws once the mark has removed the +/// word beside it: read from the front, that `TJ` drew nothing, the paragraph was taken for empty, +/// and its structure element lost the content the page still shows. +#[test] +fn a_tagged_tj_after_a_surplus_operand_still_draws() { + let content = b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj 10 -100 Td (x) [(KEPT)] TJ ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let paragraph = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert_eq!(paragraph.get(b"K").and_then(Object::as_int), Some(0), "it still tags what the page shows: {paragraph:?}"); +} + +/// A path's points follow a surplus number: read from the front, each point was shifted by one, +/// so a square under the mark was placed beside it and kept, and one beside the mark was placed +/// under it and removed. +#[test] +fn a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them() { + let under = b"0 0 1 rg 9 100 100 m 9 140 100 l 9 140 140 l 9 100 140 l h f 9 100 100 m 9 140 140 l S".as_slice(); + let mut doc = one_page(under, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 2, "both paths under the mark go"); + let beside = + b"0 0 1 rg 0 200 200 m 250 200 l 250 250 l 200 250 l h f 0 200 200 m 210 210 220 220 230 230 c 240 240 250 250 v 200 250 200 260 y h f" + .as_slice(); + let mut doc = one_page(beside, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "the paths beside the mark stay"); + // Curves whose points, after a surplus, reach under the mark. + for curve in [&b"200 200 m 9 10 10 12 12 15 15 c f"[..], b"200 200 m 9 10 10 15 15 v f", b"200 200 m 9 10 10 15 15 y f"] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1, "{}", String::from_utf8_lossy(curve)); + } + // A line, and curves, from a point beside the mark whose points, after a surplus, stay beside + // it: read from the front, the surplus was a point under it. + for curve in [ + &b"100 100 500 m 510 510 l S"[..], + b"500 500 m 100 100 510 l S", + b"100 500 m 100 100 500 110 500 120 500 c f", + b"100 500 m 100 110 500 120 500 v f", + b"100 500 m 100 110 500 120 500 y f", + ] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "{}", String::from_utf8_lossy(curve)); + } +} + +/// An `MCID` given by reference: in a named property list, and in a structure element's `/MCR`. +/// Read directly, it was missed, and the element kept the removed text. +#[test] +fn an_mcid_given_by_reference_loses_what_redaction_removed() { + for (content, properties, kid) in [ + (b"/P /MC0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC".as_slice(), "/Properties << /MC0 << /MCID 10 0 R >> >>", "0"), + (b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", "", "<< /Type /MCR /Pg 3 0 R /MCID 10 0 R >>"), + ] { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> {properties} >> >>").into_bytes(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + format!("<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the SECRET) /K {kid} >>").into_bytes(), + b"0".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1, "{properties}{kid}"); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); + } + // An `/MCR` naming the same MCID in a form's stream (`/Stm`) is other content: it stays. + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (FORMTEXT) /K << /Type /MCR /Pg 3 0 R /Stm 11 0 R /MCID 10 0 R >> >>".to_vec(), + b"0".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"/P <> BDC EMC"), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, None).unwrap(); + let other = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(other.get(b"ActualText").is_some(), "{other:?}"); +} + +/// The resources of page `page`, resolved. +fn resources_of(doc: &Document, page: usize) -> Dict { + let page = pdfcraft_model::pages(doc).swap_remove(page); + page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default() +} + +/// `dict`'s dictionary `key`, resolved. +fn dict_in(doc: &Document, dict: &Dict, key: &[u8]) -> Dict { + dict.get(key).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default() +} + +/// A property list a second page sharing the resources names after a surplus operand, once the +/// first page's block that named it was redacted: read from the front, the second page's name went +/// unseen and the list was retired from under it. +#[test] +fn a_property_list_another_page_names_after_a_surplus_operand_is_kept() { + use pdfcraft_cos::Stream; + let mut doc = one_page( + b"/Span /P0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", + "/Properties << /P0 7 0 R >>", + vec![b"<< /ActualText (SECRET) >>".to_vec()], + ); + let shared = doc.add(Object::Dict(resources_of(&doc, 0))); + doc.update_dict(ObjRef::new(3, 0), |d| d.set(b"Resources".to_vec(), Object::Ref(shared))).unwrap(); + let mut second = doc.get(ObjRef::new(3, 0)).as_dict().cloned().unwrap(); + let contents = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"(x) /Span /P0 BDC BT /F1 10 Tf 10 100 Td (Other) Tj ET EMC".to_vec()))); + second.set(b"Contents".to_vec(), Object::Ref(contents)); + let second = doc.add(Object::Dict(second)); + doc.update_dict(ObjRef::new(2, 0), |d| { + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(3, 0)), Object::Ref(second)])); + d.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, Some(&[0])).unwrap(); + let properties = dict_in(&doc, &resources_of(&doc, 1), b"Properties"); + assert!(properties.contains(b"P0"), "the second page's property list stays: {properties:?}"); +} + +/// A Type 3 font set only by an ExtGState's `/Font`, selected after a surplus operand: read from the +/// front, the cleanups took it for undrawn, dropped it from the page and unlinked its glyph. +#[test] +fn a_type3_font_an_extgstate_selects_after_a_surplus_operand_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /GS1 gs 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R /b 9 0 R >> \ + /Encoding << /Differences [97 /a /b] >> /FirstChar 97 /LastChar 98 /Widths [10 10] >>" + .to_vec(), + stream("", b"10 0 d0 0 0 10 10 re f"), + stream("", b"10 0 d0 0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let fonts = dict_in(&doc, &resources_of(&doc, 0), b"Font"); + assert!(fonts.contains(b"T3"), "the page still lists the font: {fonts:?}"); + let font = doc.get(ObjRef::new(7, 0)).as_dict().cloned().unwrap_or_default(); + let procs = dict_in(&doc, &font, b"CharProcs"); + assert!(procs.contains(b"a"), "the glyph the page draws stays: {procs:?}"); +} + +/// A Type 3 font without resources of its own, set by an ExtGState selected after a surplus operand: +/// its glyph paints through the page's resources, so it may paint with the page's pattern, which +/// stays once the page's own fill with it was redacted (the text itself fills with grey). Read from the front, the font was taken +/// for one with resources, and the pattern was pruned. +#[test] +fn a_type3_font_without_resources_an_extgstate_selects_after_a_surplus_operand_keeps_the_pattern() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Pattern << /P0 9 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 10 260 20 20 re f 0 g BT (x) /GS1 gs 150 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 /Pattern cs /P0 scn 0 0 10 10 re f"), + stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, 0)); +} diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index 0d171b4008..bec850e302 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -390,6 +390,35 @@ fn type3_font_used_in_a_form_xobject_has_its_redacted_glyph_removed() { assert!(!full_save_holds(&doc, b"SECRET_FORM_GLYPH_PAYLOAD"), "redacted glyph program survived"); } +/// Type 3 cleanup decoded a glyph program again for every font listing it: 2 000 fonts sharing +/// one large program read it 2 000 times. It is read once now, and the cleanup still keeps the +/// glyph every font shows. +#[test] +fn a_glyph_program_many_type3_fonts_share_is_read_once() { + const FONTS: usize = 2_000; + let names: String = (0..FONTS).map(|i| format!("/T{i} {} 0 R ", 6 + i)).collect(); + let shows: String = (0..FONTS).map(|i| format!("/T{i} 10 Tf (A) Tj ")).collect(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << {names}>> >> >>").into_bytes(), + stream("", format!("BT 10 100 Td {shows}ET").as_bytes()), + stream("", [b"SHARED_GLYPH ".as_slice(), &b"0 0 m 1 1 l S ".repeat(50_000)].concat()), + ]; + objs.extend((0..FONTS).map(|_| { + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 65 /Widths [0] /Encoding << /Differences [65 /A] >> /CharProcs << /A 5 0 R >> >>".to_vec() + })); + let mut doc = pdf(objs); + // The glyphs are drawn at x 10 with no width, so a mark away from them leaves them all. + mark(&mut doc, 0, &[[200.0, 200.0, 250.0, 250.0]], ""); + let started = std::time::Instant::now(); + apply(&mut doc, None).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(60), "{:?}", started.elapsed()); + let font = doc.get(ObjRef::new(6, 0)); + let procs = doc.resolve(font.as_dict().unwrap().get(b"CharProcs").unwrap()); + assert!(procs.as_dict().unwrap().contains(b"A"), "the glyph every font shows was unlinked"); +} + /// A Type 3 font with glyphs A (its program object 7) and B (object 8), at object 6. const TYPE3: &str = "<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> >>"; @@ -783,6 +812,37 @@ fn a_type3_font_only_a_soft_mask_without_resources_selects_stays_on_the_page() { assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); } +/// The same soft mask, its ExtGState selected after a surplus operand (`(x) /GS1 gs`), which the +/// renderer ignores: read from the front, the cleanups never read the group, and dropped the font +/// it selects and its glyph. +#[test] +fn a_type3_font_only_a_soft_mask_selected_after_a_surplus_operand_selects_stays_on_the_page() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AC) Tj ET (x) /GS1 gs 150 150 100 100 re f"), + b"<< /Font << /T3 6 0 R /T4 7 0 R >> /ExtGState << /GS1 << /Type /ExtGState /SMask << /Type /Mask /S /Luminosity /G 8 0 R >> >> >> >>" + .to_vec(), + type3_abc(9), + type3_abc(12), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency /CS /DeviceGray >>", + b"1 g BT /T4 40 Tf 160 180 Td (B) Tj ET", + ), + stream("", b"% REDACTED_GLYPH_A"), + stream("", b"% UNUSED_GLYPH_B"), + stream("", b"% PAGE_GLYPH_C"), + stream("", b"% UNUSED_T4_GLYPH_A"), + stream("", b"% MASK_GLYPH_B"), + stream("", b"% UNUSED_T4_GLYPH_C"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(full_save_holds(&doc, b"MASK_GLYPH_B"), "the glyph the soft mask shows was unlinked"); + assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); +} + /// The same group, its content behind a predictor whose `/DecodeParms` are given by reference: /// the page's cleanup reads it through them, and keeps the font it selects. #[test] diff --git a/crates/redact/src/tests/unparsed.rs b/crates/redact/src/tests/unparsed.rs new file mode 100644 index 0000000000..90dcb1ae47 --- /dev/null +++ b/crates/redact/src/tests/unparsed.rs @@ -0,0 +1,170 @@ +//! Bytes the content parser can't read are read past by the redaction walk, so text they hide (a +//! stray `)` or `}` in a `TJ` array) is neither found nor removed, while other readers still draw +//! it. A page whose walk meets such bytes is refused, by Apply Redactions and by Remove Hidden +//! Information. + +use super::*; +use crate::sanitize::{Hidden, remove_hidden, sanitize, scan}; + +/// `TJ` arrays each holding a byte the parser can't read beside the text. +const BROKEN: [&[u8]; 4] = [b"[(TAIL) ) (SECRET)] TJ", b"[(TAIL) } (SECRET)] TJ", b"[(TAILSECRET) )] TJ", b"[) (TAILSECRET)] TJ"]; + +/// Over the top third of the 300 × 300 page. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +/// `tj` shown in /F1 at (20, 260), under [`MARK`]. +fn shown(tj: &[u8]) -> Vec { + [b"BT /F1 10 Tf 20 260 Td ".as_slice(), tj, b" ET"].concat() +} + +/// The only text under the mark sits in such a `TJ`: the walk finds nothing there and rewrites +/// nothing, so it went on, and the text stayed in the file. +#[test] +fn text_under_a_mark_beside_bytes_the_parser_cannot_read_is_refused() { + for tj in BROKEN { + let mut doc = one_page(&shown(tj), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } + // With other text on the page, outside the mark. + let mut doc = one_page(&[b"BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET ".as_slice(), &shown(BROKEN[0])].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "beside other text"); +} + +/// In a stream the walk rewrites, it removed the text it could read and wrote the broken `TJ` out +/// as `(TAIL) (SECRET) TJ`: the stray byte and the brackets went, the strings stayed. +#[test] +fn bytes_the_parser_cannot_read_in_a_stream_the_walk_rewrites_are_refused() { + for tj in BROKEN { + let mut doc = one_page(&[b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET ".as_slice(), &shown(tj)].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } +} + +/// A form under the mark with resources of its own, which the walk enters but doesn't rewrite: +/// no later read of the page follows it. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_under_a_mark_are_refused() { + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", shown(BROKEN[0])); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, and +/// the walk doesn't refuse the page over them. This one has resources of its own, so the check after +/// the walk doesn't read it either (it would refuse one without), and the page is redacted. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_away_from_the_marks_are_left() { + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 100] /Resources << /Font << /F1 5 0 R >> >>", + [b"BT /F1 10 Tf 20 20 Td ".as_slice(), BROKEN[0], b" ET"].concat(), + ); + let mut doc = one_page(b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("(SECRET)")); +} + +/// The page sets its font only through an ExtGState `/Font` and lists no `/Font` resources, so the +/// cleanups after the walk, which look at the listed fonts, read nothing. +#[test] +fn bytes_the_parser_cannot_read_on_a_page_without_font_resources_are_refused() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /ExtGState << /GS1 << /Font [6 0 R 10] >> >> >> >>" + .to_vec(), + stream("", b"BT /GS1 gs 20 230 Td (SECRET) Tj ET"), + stream("", b"BT /GS1 gs 20 260 Td [(TAIL) ) (SECRET)] TJ ET"), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// Hidden text inside a broken `TJ` is never counted: alone, Remove Hidden Information found +/// nothing to remove and Sanitize Document went on; beside other hidden text, the removal went on +/// and left it. The scan now counts the page, and removing hidden text refuses it. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_is_refused() { + let alone = b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET".as_slice(); + let beside = [b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET ".as_slice(), alone].concat(); + for content in [alone, &beside] { + let label = String::from_utf8_lossy(content); + let doc = one_page(content, "", vec![]); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert!(counts[&Hidden::HiddenText] > 0, "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(sanitize(&mut doc), Err(RedactError::Unreadable(1)), "{label}"); + } + // In a form with resources of its own, which the walk enters on every page. + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", alone); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "in a form"); +} + +/// Removing hidden layers alone refuses such bytes too, even in a document without +/// `/OCProperties`: a membership dictionary (`/AllOff` of a layer that is on) still hides what it +/// marks, for the walk as for the app's renderer. With nothing in front of `/OC`, the marked text +/// is removed. Here a string beside a stray `}` is left in front of it, which keeps the block from +/// the walk (it reads a `BDC`'s first operands; a string alone does so too, a matter of its own), +/// and the stray byte has the page refused rather than the text kept. +#[test] +fn bytes_the_parser_cannot_read_refuse_removing_hidden_layers_alone() { + let marked = |before: &[u8]| { + let content = [before, b" /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET"].concat(); + one_page( + &content, + "/Properties << /MC0 8 0 R >>", + vec![b"<< /Type /OCG /Name (Layer) >>".to_vec(), b"<< /Type /OCMD /OCGs [7 0 R] /P /AllOff >>".to_vec()], + ) + }; + let mut doc = marked(b""); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + assert!(!content(&doc, 0).contains("LAYERSECRET")); + let mut doc = marked(b"(x) }"); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::Unreadable(1))); +} + +/// With form fields removed too, a field's appearance is flattened into page content and read the +/// same way, though the scan, which runs before flattening, doesn't count it: hidden text in a +/// broken `TJ` there refuses removing hidden text, and Sanitize Document, which went on and left it. +/// Removing form fields alone doesn't read the content, and goes on. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_in_a_field_appearance_is_refused() { + let fresh = || { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>" + .to_vec(), + stream("", b"BT /F1 10 Tf 10 230 Td (Name:) Tj ET"), + FONT.replace("95 0 R", "8 0 R").into_bytes(), + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /P 3 0 R /Rect [50 225 200 240] /AP << /N 7 0 R >> >>".to_vec(), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 150 15] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 3 Tr 2 3 Td [(TAIL) ) (HIDDEN)] TJ ET", + ), + widths(), + ]) + }; + assert!(remove_hidden(&mut fresh(), &[Hidden::FormFields]).is_ok()); + assert_eq!(remove_hidden(&mut fresh(), &[Hidden::FormFields, Hidden::HiddenText]), Err(RedactError::Unreadable(1))); + assert_eq!(sanitize(&mut fresh()), Err(RedactError::Unreadable(1))); +} + +/// An inline image whose end the parser guesses, as Apply Redactions already refuses it: what +/// follows it may not be what the page draws, so Remove Hidden Information refuses it too. +#[test] +fn hidden_text_beside_an_inline_image_the_renderer_cannot_end_is_refused() { + let page = b"BT /F1 10 Tf 3 Tr 10 100 Td (HIDDEN) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F".as_slice(); + let mut doc = one_page(page, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1))); +} diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index 136cb2b207..6e34fba352 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -897,6 +897,119 @@ fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { assert!(proof.entries[0].detail.contains("glyph(s) still show")); } +/// The proof reads an operator's operands from the end, as the renderer does: a surplus before +/// them doesn't hide the text a `Tf`, `TJ` or `Do` after it draws, or the image a `Do` draws. +#[test] +fn the_proof_reads_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", + ); + let image = stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"); + let image_form = + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 8 0 R >> >>", b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q"); + for (content, spec) in [ + (&b"BT (x) /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"[..], Spec::default()), + // The size: text starting outside the region reaches into it only at its size. + (b"BT (x) /F1 10 Tf -60 100 Td (AAAAAAAAAAAAAAAAAAAAAAAAAAAA) Tj ET", Spec::default()), + (b"BT /F1 10 Tf 10 100 Td (x) [(SECRETVALUE)] TJ ET", Spec::default()), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form.clone()], ..Spec::default() }), + // An image under the mark, and one in a form under it. + (b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q", Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image.clone()], ..Spec::default() }), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![image_form.clone(), image.clone()], ..Spec::default() }), + ] { + let mut doc = marked_content(content, spec); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// The proof reads the text state's operands from the end too. In each case the text starts outside +/// the region and only the operand after the surplus brings a glyph into it: char spacing, word +/// spacing, horizontal scaling, leading with `T*`, rise, and the spacing `"` sets. +#[test] +fn the_proof_reads_the_text_states_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"BT /F1 10 Tf -60 100 Td (x) 70 Tc (AB) Tj ET"[..], + b"BT /F1 10 Tf -60 100 Td (x) 70 Tw (A B) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 1500 Tz (AB) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) 100 TL T* (A) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) -100 Ts (A) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 0 70 (AB) \" ET", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// The proof reads a path's points from the end too: a square under the region, its points after +/// a surplus number, still paints there. +#[test] +fn the_proof_reads_a_paths_points_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"0 0 1 rg 9 20 98 m 9 60 98 l 9 60 108 l 9 20 108 l h f"[..], + b"0 0 1 rg 9 20 98 40 10 re f", + // Curves from a point outside the region whose own points reach into it. + b"0 0 1 rg 9 200 200 m 9 20 98 40 100 60 108 c h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 v h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 y h f", + // A segment after a surplus, from a point outside the region. + b"0 0 1 rg 200 200 m 200 20 100 l h f", + b"0 0 1 rg 200 200 m 200 20 100 30 100 40 100 c h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 v h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 y h f", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// Which font a `Tf` selects, and which strings a `TJ` shows, are read from the end too: text an +/// undecodable font shows after a surplus operand, away from the mark, leaves the region +/// unverifiable. Read from the front, no font was set, or no string was shown, and it passed. +#[test] +fn the_proof_reads_the_font_and_the_strings_from_the_end() { + for content in [&b"BT (x) /F2 10 Tf 10 200 Td <00410042> Tj ET"[..], b"BT /F2 10 Tf 10 200 Td (x) [<00410042>] TJ ET"] { + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws +/// after a surplus operand can't be placed, as one it draws without one can't: the region is +/// unverifiable. +#[test] +fn an_image_a_type3_glyph_draws_after_a_surplus_operand_is_found() { + let spec = Spec { + resources: "/Font << /F1 5 0 R /T3 7 0 R >> /XObject << /Im0 9 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 (x) /Im0 Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), + ], + ..Spec::default() + }; + let mut doc = open(build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET BT /T3 12 Tf 150 20 Td (a) Tj ET", spec)); + mark(&mut doc, &[REGION]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); +} + #[test] fn an_ambiguous_needle_is_still_swept_outside_page_content() { // The word also stays on the page legitimately (so page text, the raw file and decoded diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 4c3f2211e8..1817fb3c39 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::Pieces; -use pdfcraft_cos::{CosError, Dict, Document, ObjRef, Object}; +use pdfcraft_cos::{CosError, DecodeBudget, Dict, Document, ObjRef, Object}; use crate::{RedactError, page_streams}; @@ -130,6 +130,25 @@ struct UsedInGlyphs { properties: HashSet>, } +impl UsedInGlyphs { + fn merge(&mut self, other: &UsedInGlyphs) { + self.xobjects.extend(other.xobjects.iter().cloned()); + self.colorspaces.extend(other.colorspaces.iter().cloned()); + self.patterns.extend(other.patterns.iter().cloned()); + self.shadings.extend(other.shadings.iter().cloned()); + self.extgstates.extend(other.extgstates.iter().cloned()); + self.fonts.extend(other.fonts.iter().cloned()); + self.properties.extend(other.properties.iter().cloned()); + } +} + +/// What one Type 3 cleanup reads of glyph programs: the resource names each indirect program +/// uses, read once however many fonts share it, and one decoding allowance for all of them. +struct GlyphReads { + used: HashMap, + budget: DecodeBudget, +} + fn collect_used_resources_in_stream(data: &[u8], used: &mut UsedInGlyphs) -> Result<(), CosError> { // One operator at a time, so a glyph program of millions of them is never held whole. A byte // that can't be read anywhere in it still refuses, once the loop has read it all. @@ -151,7 +170,7 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"cs" | b"CS" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.colorspaces.insert(n.to_vec()); } } @@ -166,17 +185,17 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"gs" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.extgstates.insert(n.to_vec()); } } b"Tf" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.name_back(1) { used.fonts.insert(n.to_vec()); } } b"BDC" | b"DP" => { - if let Some(n) = op.name(1) { + if let Some(n) = op.last_name() { used.properties.insert(n.to_vec()); } } @@ -315,7 +334,7 @@ fn scan_content_stream( } b"gs" => { apply_text_op(ctx, res, &op, &mut current_font); - if let Some(name) = op.name(0) { + if let Some(name) = op.last_name() { scan_soft_mask(ctx, res, holder_ref, name, depth)?; } } @@ -338,12 +357,12 @@ fn scan_content_stream( fn apply_text_op(ctx: &mut ScanContext<'_>, res: &Dict, op: &pdfcraft_content::Op, current_font: &mut Option) { match op.op.as_slice() { b"Tf" => { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { *current_font = resolve_font_in_res(ctx.doc, res, name).map(|(k, _)| k); } } b"gs" => { - if let Some(name) = op.name(0) + if let Some(name) = op.last_name() && let Some((k, _)) = resolve_extgstate_font(ctx.doc, res, name) { *current_font = Some(k); @@ -477,7 +496,7 @@ fn add_soft_mask_ops(doc: &Document, res: &Dict, ops: &mut Vec op - .name(0) + .last_name() .and_then(|n| states.get(n)) .and_then(|s| doc.resolve(s).as_dict().and_then(|s| s.get(b"SMask").cloned())) .and_then(|m| doc.resolve(&m).as_dict().and_then(|m| m.get(b"G").and_then(Object::as_ref))), @@ -539,10 +558,19 @@ fn gs_font_object(doc: &Document, state_obj: &Object) -> Option { font_entry.into_iter().next() } +/// Whether font `f` (a reference, or a font dictionary) is a Type 3 font, the only kind the font +/// cleanup prunes: its `/Subtype` is read directly, as the cleanup's other readers read it, so one +/// given by reference isn't recognised by any of them. Which allowance the cleanup's read is charged +/// to, and what it prunes, both go by this. +fn is_type3(doc: &Document, f: &Object) -> bool { + doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3")) +} + /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when /// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's -/// allowance. +/// allowance, when the resources list a Type 3 font, and otherwise to an allowance of a page of its +/// own (below). pub(crate) fn prune_type3_from_resources( doc: &Document, budget: &mut pdfcraft_cos::DecodeBudget, @@ -556,12 +584,20 @@ pub(crate) fn prune_type3_from_resources( } // The operators of forms that draw through these resources count too: dropping a font a child - // form still shows would leave its text without one. - let mut ops = crate::interp::resource_ops(doc, budget, res, content)?; + // form still shows would leave its text without one. Without a Type 3 font nothing is pruned, + // but the read is still made: it refuses bytes the parser can't read in what the content draws, + // and on a page with nothing replaced and no properties listed no other check does, unless the + // document still lists a Type 3 font (`unlink_type3` then reads every page). The walk reads + // past them, so text they hide would stay under the mark. The read is then charged to an + // allowance of a page of its own, not to what the walk left of the page's, since it decides + // nothing about the fonts. + let type3 = fonts.iter().any(|(_, f)| is_type3(doc, f)); + let mut own = pdfcraft_cos::DecodeBudget::page(); + let mut ops = crate::interp::resource_ops(doc, if type3 { &mut *budget } else { &mut own }, res, content)?; // So do those of a page's annotation appearances, and of soft mask groups, without resources of // their own. They are read only when there is a Type 3 font to keep, so other files are not // refused over them. - if fonts.iter().any(|(_, f)| doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3"))) { + if type3 { if let Some(page) = page { // One total for all of them, as for the content: each appearance under the cap could // still add up to far more than it. Their decoded bytes, and their forms', are charged to @@ -584,11 +620,11 @@ pub(crate) fn prune_type3_from_resources( let mut drawn_font_names = HashSet::new(); for op in &ops { if op.is("Tf") { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { drawn_font_names.insert(name.to_vec()); } } else if op.is("gs") - && let Some(name) = op.name(0) + && let Some(name) = op.last_name() { let extg = res_dict(doc, res, b"ExtGState"); if let Some(font_obj) = extg.get(name).and_then(|state_obj| gs_font_object(doc, state_obj)) { @@ -605,15 +641,12 @@ pub(crate) fn prune_type3_from_resources( let mut changed = false; let font_names: Vec> = fonts.iter().map(|(n, _)| n.to_vec()).collect(); for name in font_names { - if let Some(font_obj) = fonts.get(&name) { - let resolved = doc.resolve(font_obj); - if let Some(d) = resolved.as_dict() - && d.name(b"Subtype") == Some(b"Type3") - && !drawn_font_names.contains(&name) - { - fonts.remove(&name); - changed = true; - } + if let Some(font_obj) = fonts.get(&name) + && is_type3(doc, font_obj) + && !drawn_font_names.contains(&name) + { + fonts.remove(&name); + changed = true; } } @@ -717,9 +750,10 @@ pub(crate) fn unlink_type3(doc: &mut Document) -> Result<(), RedactError> { } // For each Type 3 font: unlink unused glyphs from CharProcs and prune exclusive resources. + let mut reads = GlyphReads { used: HashMap::new(), budget: DecodeBudget::page() }; for (font_key, font_dict) in &type3_fonts { let surviving = used_glyphs.get(font_key).cloned().unwrap_or_default(); - clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving)?; + clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving, &mut reads)?; } Ok(()) @@ -794,6 +828,7 @@ fn clean_type3_font( font_key: &FontKey, font_dict: &Dict, surviving: &HashSet>, + reads: &mut GlyphReads, ) -> Result<(), RedactError> { let Some(char_procs_val) = font_dict.get(b"CharProcs") else { return Err(CosError::Syntax { offset: 0, detail: "redaction could not establish safe Type 3 font cleanup: missing CharProcs".into() }.into()); @@ -837,6 +872,10 @@ fn clean_type3_font( let mut used_in_glyphs = UsedInGlyphs::default(); for (_, proc_val) in char_procs.iter() { + if let Some(used) = proc_val.as_ref().and_then(|r| reads.used.get(&r)) { + used_in_glyphs.merge(used); + continue; + } let resolved = doc.resolve(proc_val); let Object::Stream(s) = &*resolved else { return Err(CosError::Syntax { @@ -845,8 +884,14 @@ fn clean_type3_font( } .into()); }; - let data = doc.with_direct_parms(s).decoded()?; - collect_used_resources_in_stream(&data, &mut used_in_glyphs)?; + // Past the allowance the cleanup can't be shown complete, and the redaction is refused. + let data = reads.budget.decode(&doc.with_direct_parms(s))?; + let mut used = UsedInGlyphs::default(); + collect_used_resources_in_stream(&data, &mut used)?; + used_in_glyphs.merge(&used); + if let Some(r) = proc_val.as_ref() { + reads.used.insert(r, used); + } } if prune_type3_font_resources(doc, &mut f_res, &used_in_glyphs) { diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index fe92e4e78d..4c76cc6131 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -16,7 +16,7 @@ pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), - b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), _ => false, } } @@ -396,17 +396,38 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); } -/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error -/// (guessing the identity would put its content in the wrong place). -pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { - match dict.get(b"Matrix") { - None => Ok(Matrix::IDENTITY), - Some(m) => { - let m = doc.resolve(m); - let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); - Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) +/// A form's or an annotation appearance's `/Matrix`, the array and each entry direct or by reference: absent or null +/// is the identity, present but malformed is an error (guessing the identity would put its content in the +/// wrong place). A reference to a reference is malformed here: readers disagree on following it +/// (Poppler doesn't, the app's renderer does), so where the content draws can't be told. The +/// redaction walk reads a form's with it too. +pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + let Some(m) = dict.get(b"Matrix") else { return Ok(Matrix::IDENTITY) }; + // Each reference is followed one hop. The array is read in place when written directly, and + // its length checked before any entry is read: the walk reads it at every `Do`, and an array of + // any length would be copied each time. + let held; + let m = match m { + Object::Ref(r) => { + held = doc.get(*r); + &*held } + direct => direct, + }; + // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. + if matches!(m, Object::Null) { + return Ok(Matrix::IDENTITY); } + let items = m.as_array().filter(|a| a.len() == 6).ok_or(())?; + let mut values = [0.0; 6]; + for (value, item) in values.iter_mut().zip(items) { + let n = match item { + Object::Ref(r) => doc.get(*r).as_f64(), + direct => direct.as_f64(), + }; + *value = n.filter(|n| n.is_finite()).ok_or(())?; + } + Ok(Matrix(values)) } /// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. @@ -552,7 +573,7 @@ impl Extractor<'_> { let o = op.op.as_slice(); // Strings and TJ adjustments, in order. let items: Vec<&Object> = match o { - b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + b"TJ" => op.back(0).and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), _ => op.operands.last().into_iter().collect(), }; let mut any_text = false; @@ -637,14 +658,15 @@ impl Extractor<'_> { tm = Matrix::IDENTITY; tlm = Matrix::IDENTITY; } - b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), - b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), - b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, - b"TL" => gs.tl = op.num(0).unwrap_or(0.0), - b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + // Operands are read from the end, as the renderer reads them. + b"Tc" => gs.tc = op.num_back(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num_back(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num_back(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num_back(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num_back(0).unwrap_or(0.0), b"Tf" => { - gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); - gs.size = op.num(1).unwrap_or(0.0); + gs.font = op.name_back(1).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num_back(0).unwrap_or(0.0); } b"Td" | b"TD" => { if let Some([x, y]) = op.nums::<2>() { @@ -671,7 +693,7 @@ impl Extractor<'_> { tm = tlm; } if o == b"\"" - && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + && let (Some(aw), Some(ac)) = (op.num_back(2), op.num_back(1)) { (gs.tw, gs.tc) = (aw, ac); } @@ -686,7 +708,7 @@ impl Extractor<'_> { self.show_text(font, &op, &gs, &mut tm, placed); } b"Do" => { - let Some(name) = op.name(0) else { continue }; + let Some(name) = op.last_name() else { continue }; let Some(entry) = xobjects.get(name) else { continue }; if let Object::Stream(s) = &*doc.resolve(entry) && s.dict.name(b"Subtype") == Some(b"Form") diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index 7ea0de5699..7b6833edef 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -188,7 +188,7 @@ impl Geo<'_> { } } b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let obj = self.doc.resolve(entry); let Object::Stream(s) = &*obj else { continue }; match s.dict.name(b"Subtype") { @@ -344,7 +344,7 @@ impl Geo<'_> { match op.op.as_slice() { b"BI" => return Ok(true), b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; match x.dict.name(b"Subtype") { Some(b"Image") => return Ok(true), @@ -372,21 +372,28 @@ impl Geo<'_> { /// carries) by one construction operator. fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { let o = op.op.as_slice(); - let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); let at = |x: f64, y: f64| ctm.apply(x, y); + // Each takes its numbers from the end, as the renderer reads them: a surplus before them is + // ignored, not read as a point. + let points: Option> = match o { + b"l" => op.nums::<2>().map(|[x, y]| vec![at(x, y)]), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![at(x1, y1), at(x2, y2), at(x3, y3)]), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![at(x1, y1), at(x2, y2)]), + _ => None, + }; match o { b"m" => { - if let [x, y] = nums[..] { + if let Some([x, y]) = op.nums::<2>() { sub.push(vec![at(x, y)]); } } b"l" | b"c" | b"v" | b"y" => { - if let Some(cur) = sub.last_mut() { - cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + if let (Some(cur), Some(points)) = (sub.last_mut(), points) { + cur.extend(points); } } b"re" => { - if let [x, y, w, h] = nums[..] { + if let Some([x, y, w, h]) = op.nums::<4>() { sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); } } diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs index 3a36c9e08e..d58d4c307c 100644 --- a/crates/redact/src/verify/mod.rs +++ b/crates/redact/src/verify/mod.rs @@ -70,6 +70,7 @@ mod snapshot; mod sweep; mod util; +pub(crate) use extract::matrix_of; pub use snapshot::Snapshot; /// What the proof established for one region. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/redact/tests/matrix_work.rs b/crates/redact/tests/matrix_work.rs new file mode 100644 index 0000000000..d96447ff71 --- /dev/null +++ b/crates/redact/tests/matrix_work.rs @@ -0,0 +1,59 @@ +//! Work regression for a form's `/Matrix`, which the redaction walk reads at every `Do` of the +//! form: an array of any length costs no more than one of seven entries, written directly or by +//! reference. A test binary of its own, so that none of the crate's other tests shares its process. +//! The test compares two timings of its own, the least of three runs each, not a fixed time. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page drawing a form `draws` times under a mark, the form's `/Matrix` an array of `entries` +/// zeros, written in its dictionary or, `by_reference`, as object 6. +fn document(draws: usize, entries: usize, by_reference: bool) -> Document { + let zeros = format!("[{}]", vec!["0"; entries].join(" ")); + let matrix = if by_reference { "6 0 R".to_string() } else { zeros.clone() }; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Fm 5 0 R >> >> >>".to_vec(), + stream_obj("", "/Fm Do ".repeat(draws).as_bytes()), + stream_obj(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix {matrix}"), b"0 0 10 10 re f"), + zeros.into_bytes(), + ]; + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 0.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// How long redacting `document(draws, entries, by_reference)` takes, the least of three runs. A +/// matrix that isn't six numbers is refused. +fn redaction(draws: usize, entries: usize, by_reference: bool) -> Duration { + (0..3) + .map(|_| { + let mut doc = document(draws, entries, by_reference); + let start = Instant::now(); + let refused = apply(&mut doc, None).is_err(); + let took = start.elapsed(); + assert!(refused, "a /Matrix of {entries} entries is refused"); + took + }) + .min() + .unwrap_or_default() +} + +/// Written when the walk resolved and copied every entry before checking there were six: 4,000 +/// draws of a form whose `/Matrix` is 4,000 entries long read sixteen million entries. +#[test] +fn a_long_form_matrix_costs_no_more_than_a_short_one() { + for by_reference in [false, true] { + let (short, long) = (redaction(4_000, 7, by_reference), redaction(4_000, 4_000, by_reference)); + assert!(long < short * 3 + Duration::from_millis(50), "by reference: {by_reference}: long {long:?}, short {short:?}"); + } +} diff --git a/crates/redact/tests/operator_work.rs b/crates/redact/tests/operator_work.rs new file mode 100644 index 0000000000..e4340fc021 --- /dev/null +++ b/crates/redact/tests/operator_work.rs @@ -0,0 +1,98 @@ +//! Work regressions for what a redaction looks up at operators a stream can hold thousands of: the +//! tags cleanup's named property lists at each `BDC` are not read whole each time. A test binary of +//! its own, so that none of the crate's other tests shares its process, and its tests run one at a +//! time. Each test compares two timings of its own, the least of three runs each, with many +//! entries or operators and with one, not a fixed time. The bound leaves room for slower machines: +//! each regression took seconds against milliseconds. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page with `resources` and `extra` objects (from object 5 on), drawing a word under a mark +/// (so the redaction rewrites the page) and then `ops`. +fn page(resources: &str, ops: &[u8], extra: Vec>) -> Document { + let content = [b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET ".as_slice(), ops].concat(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> {resources} >> >>" + ) + .into_bytes(), + stream_obj("", content), + ]; + objs.extend(extra); + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 200.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// One page whose `/Properties` holds `entries` entries, drawing `ops`. +fn properties(entries: usize, ops: &[u8]) -> Document { + let lists: String = (0..entries).map(|i| format!("/M{i} << /Lang (en) >> ")).collect(); + page(&format!("/Properties << {lists}>>"), ops, Vec::new()) +} + +/// One page whose property list `/M0` is object 5, which gives its `MCID` as object 6 and holds +/// 400,000 more entries: larger than the document keeps, so that reading it again parses it again, +/// all of it, once another object has been read since. The page draws `ops`. +fn oversized(ops: &[u8]) -> Document { + let more: String = (0..400_000).map(|i| format!("/J{i} /DeviceGray ")).collect(); + page("/Properties << /M0 5 0 R >>", ops, vec![format!("<< /MCID 6 0 R {more}>>").into_bytes(), b"0".to_vec()]) +} + +/// The tests' lock: they run one at a time, so that none is timed while another works. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn one_at_a_time() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// How long redacting a document `make` builds takes, the least of three runs. +fn redaction(make: impl Fn() -> Document) -> Duration { + (0..3) + .map(|_| { + let mut doc = make(); + let start = Instant::now(); + let report = apply(&mut doc, None).unwrap(); + let took = start.elapsed(); + assert_eq!(report.glyphs, 6, "the word under the mark goes"); + took + }) + .min() + .unwrap_or_default() +} + +/// Whether `many` took about as long as `one`. +fn about_as_long(many: Duration, one: Duration) -> bool { + many < one * 10 + Duration::from_secs(1) +} + +/// Written when the tags cleanup resolved the page's `/Properties` at every named `BDC`: 20,000 of +/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). Each `BDC` here +/// names another entry, so that it is the lists, not each name's answer, that are read once. +#[test] +fn named_property_lists_are_read_once_for_a_page() { + let _serial = one_at_a_time(); + let ops: Vec = (0..5_000).map(|i| format!("/P /M{i} BDC EMC ")).collect::().into_bytes(); + let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(5_000, &ops))); + assert!(about_as_long(many, one), "5,000 entries {many:?}, one {one:?}"); +} + +/// Written when the tags cleanup read a named property list's `MCID` by reference at each `BDC`: a +/// list larger than the document keeps was parsed again at each, 50 of them taking 12.3 s against +/// 0.44 s with the `MCID` given directly (optimized). +#[test] +fn an_oversized_property_list_is_read_once_for_a_stream() { + let _serial = one_at_a_time(); + let (one, many) = (redaction(|| oversized(b"/P /M0 BDC EMC ")), redaction(|| oversized(&b"/P /M0 BDC EMC ".repeat(100)))); + assert!(about_as_long(many, one), "100 operators {many:?}, one {one:?}"); +} diff --git a/crates/render/src/inspect/annotations.rs b/crates/render/src/inspect/annotations.rs index 7313716a2e..673d930743 100644 --- a/crates/render/src/inspect/annotations.rs +++ b/crates/render/src/inspect/annotations.rs @@ -144,12 +144,7 @@ impl<'a> Inspector<'a> { info.attachments.push(self.attachment(fs, AttachmentSource::Annotation { page, index }, text_budget, fallback)); } if subtype == "Link" { - if let Some(link) = self.link(page, index, d, info.uri_base.as_deref(), text_budget, &mut unfit_quads) { - if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { - info.has_javascript = true; - } - info.links.push(link); - } + self.collect_link(info, (page, index), d, text_budget, &mut unfit_quads); continue; } // Not comments: form widgets, pop-ups and non-markup annotations such as the @@ -161,6 +156,20 @@ impl<'a> Inspector<'a> { if comments_full { continue; } + // Text that trims to nothing is not kept, so `retained` never sees it: charge what is + // read, so many comments sharing one huge blank string stop being decoded over and over. + let raw: usize = [&b"T"[..], b"Contents", b"M", b"NM", b"State", b"DA"] + .iter() + .filter_map(|k| match d.get(k).map(|o| self.resolve(o)) { + Ok(Object::String(b, _) | Object::Name(b)) => Some(b.len()), + _ => None, + }) + .fold(0, usize::saturating_add); + // (charged once: below, by the larger of these raw bytes and what the annotation keeps) + if kept.saturating_add(raw) > MAX_ANNOTATION_BYTES { + comments_full = true; + continue; + } let rect = rect4(self.resolve(d.get(b"Rect").unwrap_or(&Object::Null))); let c = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => Some(c), @@ -262,7 +271,7 @@ impl<'a> Inspector<'a> { text_box, popup, }; - kept = kept.saturating_add(retained(&annotation)); + kept = kept.saturating_add(retained(&annotation).max(raw)); if kept > MAX_ANNOTATION_BYTES { comments_full = true; continue; @@ -356,6 +365,23 @@ impl<'a> Inspector<'a> { Some(Link { page, index, rect, quads, target }) } + /// Lists the link annotation `d`, and notes the JavaScript it runs. JavaScript is read from the + /// action, not the listed link: once the text budget is spent the action's name is not kept. + fn collect_link(&self, info: &mut DocInfo, at: (usize, usize), d: &Dictionary, text_budget: &mut usize, unfit_quads: &mut HashSet) { + info.has_javascript |= self.runs_javascript(d); + if let Some(link) = self.link(at.0, at.1, d, info.uri_base.as_deref(), text_budget, unfit_quads) { + if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { + info.has_javascript = true; + } + info.links.push(link); + } + } + + /// Whether the link's own action is a JavaScript action. + fn runs_javascript(&self, d: &Dictionary) -> bool { + d.get(b"A").ok().and_then(|a| self.dict(a)).is_some_and(|a| self.name(a, b"S").as_deref() == Some("JavaScript")) + } + /// What an action does when a link or a bookmark runs it. Only a go-to's `/D` is a place in /// this document; a remote go-to's names a page or destination of another file (ISO 32000-2 /// §12.6.4.3). `None` for an action without a type, or a go-to whose destination is not here. @@ -376,7 +402,7 @@ impl<'a> Inspector<'a> { "SetOCGState" => self.layer_state(a), "GoToR" => self.remote_go_to(a).unwrap_or_else(|| LinkTarget::Other("GoToR".into())), "Launch" => self.launch_pdf(a).unwrap_or_else(|| LinkTarget::Other("Launch".into())), - other => LinkTarget::Other(other.to_string()), + other => LinkTarget::Other(spend(other.as_bytes(), text_budget, |b| String::from_utf8_lossy(b).into_owned()).unwrap_or_default()), }) } @@ -492,6 +518,53 @@ impl<'a> Inspector<'a> { // ── form fields ───────────────────────────────────────────────────────────────────────── + /// Whether `o` or one of its descendants under `/Kids` carries an action (`/A` or `/AA`). It + /// walks the same tree `field` does, bounded the same way: a field left out of the listing for + /// want of text allowance can still have its action on a widget several levels down. + pub(super) fn has_action(&self, o: &Object, depth: u32) -> bool { + self.has_action_seen(o, depth, &mut HashMap::new()) + } + + /// The walk behind [`Self::has_action`]. Each indirect descendant is read again only from a + /// shallower depth than before (the depth limit may have cut its subtree): a field tree whose + /// `/Kids` repeat the same objects would otherwise be walked exponentially, and this reads each + /// object at most once per depth. + fn has_action_seen(&self, o: &Object, depth: u32, seen: &mut HashMap) -> bool { + if depth > 32 { + return false; + } + if let Object::Reference(id) = o { + if seen.get(id).is_some_and(|&first| first <= depth) { + return false; + } + seen.insert(*id, depth); + } + let Some(d) = self.dict(o) else { return false }; + if self.carries_action(d) { + return true; + } + let kids = match d.get(b"Kids").map(|k| self.resolve(k)) { + Ok(Object::Array(k)) => k, + _ => return false, + }; + kids.iter().any(|k| self.has_action_seen(k, depth + 1, seen)) + } + + /// Whether `/A` or `/AA` here holds an action. An editor can leave an empty dictionary (or + /// array) behind, and that is no action: a document with none must not report JavaScript. An + /// `/A` is an action when it carries `/S`; an `/AA` is one when an event maps to such an action. + fn carries_action(&self, d: &Dictionary) -> bool { + let action = |o: &Object| matches!(self.resolve(o), Object::Dictionary(a) if a.get(b"S").is_ok()); + [&b"A"[..], &b"AA"[..]].iter().any(|k| match d.get(k) { + Ok(o) => match self.resolve(o) { + Object::Dictionary(x) => x.get(b"S").is_ok() || x.iter().any(|(_, v)| action(v)), + Object::Array(a) => a.iter().any(action), + _ => false, + }, + Err(_) => false, + }) + } + pub(super) fn field( &self, o: &'a Object, @@ -510,19 +583,35 @@ impl<'a> Inspector<'a> { } let Some(d) = self.dict(o) else { return }; let (parent_name, parent_ft, parent_ff) = parent.unwrap_or(("", None, None)); - let partial = self.text(d, b"T"); - let full = match (&partial, parent_name.is_empty()) { - (Some(t), true) => t.clone(), - (Some(t), false) => format!("{parent_name}.{t}"), - (None, _) => parent_name.to_string(), - }; - let ft = self.name(d, b"FT").or(parent_ft); - let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); // Non-terminal: has kids that are fields (they carry /T). Widgets-only kids are terminal. let mut kids: Vec = match d.get(b"Kids").map(|k| self.resolve(k)) { Ok(Object::Array(k)) => k.clone(), _ => Vec::new(), }; + // A field left unlisted for want of text allowance still has its actions counted, through + // its whole subtree: an action can sit on a descendant widget, not only on the field. + let unlisted = || { + if self.has_action(o, 0) { + self.field_actions.set(true); + } + }; + // A /T that does not fit the allowance lists no field (not one under its parent's name). + let Ok(partial) = self.field_text_within(d, b"T") else { + unlisted(); + return; + }; + let full = match (&partial, parent_name.is_empty()) { + (Some(t), true) => t.clone(), + (Some(t), false) => format!("{parent_name}.{t}"), + (None, _) => parent_name.to_string(), + }; + // Every field copies its parent's full name: charge it, and list no more once it is spent. + if !self.field_charge(full.len()) { + unlisted(); + return; + } + let ft = self.name(d, b"FT").or(parent_ft); + let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); let (field_kids, widget_kids): (Vec, Vec) = kids.iter().cloned().partition(|k| self.dict(k).is_some_and(|kd| kd.get(b"T").is_ok())); let field_kids: Vec<_> = field_kids.iter().filter(|k| !self.not_a_child(walk, o, k)).collect(); @@ -570,17 +659,37 @@ impl<'a> Inspector<'a> { _ => FieldKind::Unknown, }; let value = match d.get(b"V").map(|v| self.resolve(v)) { - Ok(Object::Name(n)) => Some(String::from_utf8_lossy(n).into_owned()), - Ok(Object::String(bytes, _)) => Some(text_string(bytes)), - Ok(Object::Array(a)) => Some( - a.iter() - .filter_map(|x| match self.resolve(x) { - Object::String(bytes, _) => Some(text_string(bytes)), - _ => None, - }) - .collect::>() - .join(", "), - ), + Ok(Object::Name(n)) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + Ok(Object::String(bytes, _)) => self.field_spend(bytes, text_string), + Ok(Object::Array(a)) => { + let mut shown = Vec::new(); + let mut dropped = false; + for x in a { + let text = match self.resolve(x) { + Object::String(bytes, _) => self.field_spend(bytes, text_string), + // A name entry (`/V [/Opt1 (b)]`) is a selection too, as the forms reader + // takes it; leaving it out reported a partial selection as complete. + Object::Name(n) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + _ => continue, + }; + // The ", " between selections is charged to the allowance like the text. + match text { + Some(text) if shown.is_empty() || self.field_charge(2) => shown.push(text), + _ => dropped = true, + } + } + // A selected value the allowance could not hold shows as an omission rather than + // as part of a complete selection (its few bytes are charged, fit or not). + let mut value = shown.join(", "); + if dropped { + self.field_charge(5); + if !value.is_empty() { + value.push_str(", "); + } + value.push('…'); + } + Some(value) + } Ok(Object::Dictionary(_)) if kind == FieldKind::Signature => Some("signed".into()), _ => None, }; @@ -602,9 +711,11 @@ impl<'a> Inspector<'a> { .and_then(|p| self.page_of(p)) .or_else(|| self.find_widget_page(o, &kids)) .or_else(|| named.and_then(|id| self.annotation_pages.get_or_init(|| self.annotation_pages()).get(&id).copied())); - let has_actions = d.get(b"AA").is_ok() || widget.is_some_and(|w| w.get(b"AA").is_ok() || w.get(b"A").is_ok()); + let has_actions = self.carries_action(d) + || widget.is_some_and(|w| self.carries_action(w)) + || kids.iter().any(|k| self.dict(k).is_some_and(|w| self.carries_action(w))); let rect = widget.and_then(|w| w.get(b"Rect").ok()).map(|r| rect4(self.resolve(r))); - out.push(Field { name: full, kind, value, page, tooltip: self.text(d, b"TU"), has_actions, rect }); + out.push(Field { name: full, kind, value, page, tooltip: self.field_text(d, b"TU"), has_actions, rect }); } /// Whether `k`, listed in the `/Kids` of field `o`, is not a child of `o` after all: `o` itself diff --git a/crates/render/src/inspect/core.rs b/crates/render/src/inspect/core.rs index 88dc13c34a..adf3edffbd 100644 --- a/crates/render/src/inspect/core.rs +++ b/crates/render/src/inspect/core.rs @@ -155,16 +155,34 @@ pub(super) struct Inspector<'a> { /// has no `/P`. Searching every page's `/Annots` for each such field was O(fields × /// annotations): a second on a form of 8,000 fields. pub(super) annotation_pages: OnceCell>, + /// What is left of [`MAX_FIELD_TEXT_BYTES`]: names, values and tooltips of form fields and + /// layer names, which many entries can share one string of. + pub(super) field_text: std::cell::Cell, + /// Set when a field with an action went unlisted because the text allowance was spent, so the + /// document still reports JavaScript. + pub(super) field_actions: std::cell::Cell, /// The first page widget naming each field as its `/Parent`, built when a field has no widget /// among its `/Kids`. pub(super) parent_widgets: OnceCell>, } +/// Bytes of field and layer text one inspection reads and keeps together. +pub(super) const MAX_FIELD_TEXT_BYTES: usize = 64 << 20; + impl<'a> Inspector<'a> { pub(super) fn new(doc: &'a dyn Structure) -> Self { let page_ids = doc.page_ids(); let page_index = page_ids.iter().copied().enumerate().map(|(index, id)| (id, index)).collect(); - let mut me = Self { doc, page_ids, page_index, named: HashMap::new(), annotation_pages: OnceCell::new(), parent_widgets: OnceCell::new() }; + let mut me = Self { + doc, + page_ids, + page_index, + named: HashMap::new(), + annotation_pages: OnceCell::new(), + field_text: MAX_FIELD_TEXT_BYTES.into(), + field_actions: false.into(), + parent_widgets: OnceCell::new(), + }; let mut named = HashMap::new(); if let Some(tree) = doc.catalog().ok().and_then(|c| c.get(b"Names").ok()).and_then(|o| me.dict(o)).and_then(|n| n.get(b"Dests").ok()).and_then(|o| me.dict(o)) @@ -260,7 +278,7 @@ impl<'a> Inspector<'a> { } } } - info.has_javascript |= info.fields.iter().any(|f| f.has_actions); + info.has_javascript |= info.fields.iter().any(|f| f.has_actions) || self.field_actions.get(); let needs_rendering = catalog.get(b"NeedsRendering").ok().and_then(|o| self.resolve(o).as_bool().ok()).unwrap_or(false); if let Some(xfa) = catalog.get(b"AcroForm").ok().and_then(|o| self.dict(o)).and_then(|form| form.get(b"XFA").ok()) { // A form with no fields is taken to be laid out from its template, unless its @@ -351,11 +369,49 @@ impl<'a> Inspector<'a> { /// Whitespace-trimmed metadata, spent from `budget` as [`spend`] does, so text that trims /// short or to nothing still spends its raw length. + /// [`Self::budgeted_text`] against the shared field and layer text allowance. + pub(super) fn field_text(&self, d: &Dictionary, key: &[u8]) -> Option { + let mut left = self.field_text.get(); + let text = self.budgeted_text(d, key, &mut left); + self.field_text.set(left); + text + } + + /// Converts `bytes` against the field and layer text allowance, as [`spend`] does. + pub(super) fn field_spend(&self, bytes: &[u8], convert: impl FnOnce(&[u8]) -> String) -> Option { + let mut left = self.field_text.get(); + let text = spend(bytes, &mut left, convert); + self.field_text.set(left); + text + } + + /// Spends `n` bytes of the field and layer text allowance; `false` when too few are left. + pub(super) fn field_charge(&self, n: usize) -> bool { + let left = self.field_text.get(); + self.field_text.set(left.saturating_sub(n)); + n <= left + } + pub(super) fn budgeted_text(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Option { - let o = self.resolve(d.get(key).ok()?); - let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return None }; - let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default())?; - (!text.is_empty()).then_some(text) + self.budgeted_text_within(d, key, budget).ok().flatten() + } + + /// [`Self::budgeted_text`], but `Err` when the text is there and does not fit `budget` + /// (against `Ok(None)`: no text, or none that trims to anything). + pub(super) fn budgeted_text_within(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Result, ()> { + let Ok(o) = d.get(key) else { return Ok(None) }; + let o = self.resolve(o); + let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return Ok(None) }; + let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default()).ok_or(())?; + Ok((!text.is_empty()).then_some(text)) + } + + /// [`Self::field_text`], but `Err` when the text does not fit the allowance. + pub(super) fn field_text_within(&self, d: &Dictionary, key: &[u8]) -> Result, ()> { + let mut left = self.field_text.get(); + let text = self.budgeted_text_within(d, key, &mut left); + self.field_text.set(left); + text } pub(super) fn name(&self, d: &Dictionary, key: &[u8]) -> Option { diff --git a/crates/render/src/inspect/navigation.rs b/crates/render/src/inspect/navigation.rs index c5f2a1be86..f680b46246 100644 --- a/crates/render/src/inspect/navigation.rs +++ b/crates/render/src/inspect/navigation.rs @@ -250,7 +250,7 @@ impl<'a> Inspector<'a> { None => {} } let open = d.get(b"Count").ok().and_then(|c| c.as_i64().ok()).is_some_and(|c| c > 0); - let title = self.text(d, b"Title").unwrap_or_default(); + let title = self.field_text(d, b"Title").unwrap_or_default(); let color = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => outline_color(c.iter().map(|v| self.resolve(v).as_float().ok())), _ => None, diff --git a/crates/render/src/inspect/resources.rs b/crates/render/src/inspect/resources.rs index 3dbc1c665f..6d90f715e1 100644 --- a/crates/render/src/inspect/resources.rs +++ b/crates/render/src/inspect/resources.rs @@ -121,9 +121,14 @@ impl<'a> Inspector<'a> { .unwrap_or_default(); let auto = config.map(|c| self.view_auto_state(c)).unwrap_or_default(); let Ok(Object::Array(ocgs)) = props.get(b"OCGs").map(|o| self.resolve(o)) else { return }; + let mut seen = HashSet::new(); for g in ocgs { let Some(d) = self.dict(g) else { continue }; let id = g.as_reference().ok(); + // A group listed twice is one layer (and its name is read once). + if id.is_some_and(|id| !seen.insert(id)) { + continue; + } // The View auto state, when it manages the group, then the configuration. let visible = match id { Some(id) if let Some(&state) = auto.get(&id) => state, @@ -132,7 +137,7 @@ impl<'a> Inspector<'a> { _ => !base_off, }; let Some(id) = id else { continue }; - out.push(Layer { id, name: self.text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); + out.push(Layer { id, name: self.field_text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); } } diff --git a/crates/render/src/inspect/tests.rs b/crates/render/src/inspect/tests.rs index 68486d8097..c908b241cf 100644 --- a/crates/render/src/inspect/tests.rs +++ b/crates/render/src/inspect/tests.rs @@ -1161,6 +1161,50 @@ fn repeated_annotations_are_listed_once_within_a_byte_budget() { assert!(notes.iter().map(retained).sum::() <= MAX_ANNOTATION_BYTES); } +/// Blank text is not kept, so only reading it is charged: many comments sharing one large blank +/// string stop being built at the budget instead of decoding it once each. +#[test] +fn comments_sharing_a_blank_string_stop_at_the_read_budget() { + let small = "<< /Subtype /Text /Rect [0 0 1 1] /Contents 7 0 R >> ".repeat(1200); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Annots [{small}] >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + " ".repeat(64 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(!info.annotations.is_empty() && info.annotations.len() < 1200, "{}", info.annotations.len()); +} + +/// Many fields sharing one large tooltip, and one layer listed many times, are read within a byte +/// allowance rather than once each. +#[test] +fn fields_sharing_a_tooltip_and_repeated_layers_stop_at_the_text_budget() { + let fields = (0..1300).map(|i| format!("<< /FT /Tx /T (f{i}) /TU 7 0 R /V 7 0 R >> ")).collect::(); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> \ + /OCProperties << /OCGs [{}] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 6 0 obj << /Type /OCG /Name (Ink) >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + "6 0 R ".repeat(1000), + "x".repeat(32 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + let kept: usize = info.fields.iter().map(|f| f.tooltip.as_ref().map_or(0, String::len) + f.value.as_ref().map_or(0, String::len)).sum(); + assert!(kept > 0 && kept <= super::core::MAX_FIELD_TEXT_BYTES, "{kept}"); + assert!(info.fields.iter().any(|f| f.tooltip.is_none()), "text past the allowance is left out"); + assert_eq!(info.layers.len(), 1); +} + /// A link reached through an indirect object that holds only a reference to it is the same /// annotation as the link itself, and so is a page array reached the same way: each is listed once. #[test] @@ -2686,6 +2730,42 @@ fn inspection_keeps_no_private_page_data() { assert_eq!(report(with), report(without), "what inspection reports does not change"); } +/// A field with several widgets reports an action on any of them, not only the first, and so does +/// the document's JavaScript flag. +#[test] +fn a_field_reports_an_action_on_a_later_widget() { + let pdf = "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 4 0 obj << /FT /Tx /T (f) /Kids [5 0 R 6 0 R] >> endobj\n\ + 5 0 obj << /Subtype /Widget /Rect [0 0 1 1] >> endobj\n\ + 6 0 obj << /Subtype /Widget /Rect [0 0 1 1] /A << /S /JavaScript /JS (app.alert(1)) >> >> endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF"; + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(info.fields.iter().any(|f| f.name == "f" && f.has_actions), "{:?}", info.fields); + assert!(info.has_javascript); +} + +/// A shared descendant first reached at the depth limit (its action-bearing child cut) is read +/// again from a shallower path, so the action is still found. +#[test] +fn an_action_below_a_shared_field_first_met_deep_is_found_from_a_shallower_path() { + // 1 is the root, 2..=32 a chain of 31 fields, 33 the shared field, 34 holds the action, + // 35 a short path to 33. + let mut bodies = vec!["<< /Kids [2 0 R 35 0 R] >>".to_string()]; + for i in 2..=32 { + bodies.push(format!("<< /Kids [{} 0 R] >>", i + 1)); + } + bodies.push("<< /Kids [34 0 R] >>".into()); + bodies.push("<< /A << /S /JavaScript /JS (x) >> >>".into()); + bodies.push("<< /Kids [33 0 R] >>".into()); + let doc = Document::load_mem(&pdf_of(&bodies)).expect("loads"); + assert!(Inspector::new(&doc).has_action(&Object::Reference((1, 0)), 0)); +} + /// A `/Kids` entry that loops back, or names a root field that doesn't name this one as its /// `/Parent`, is not a child (ISO 32000-2 §12.7.3, Table 224; §12.7.4.1, Table 226): the field /// keeps its other kids and is listed with them. diff --git a/crates/render/src/raster/tests.rs b/crates/render/src/raster/tests.rs index 029a0608be..f1145e7d90 100644 --- a/crates/render/src/raster/tests.rs +++ b/crates/render/src/raster/tests.rs @@ -2741,7 +2741,7 @@ trailer << /Root 1 0 R >> assert_eq!(px(&hidden, 20, 80), vec![255, 0, 0, 255], "fields stay"); } -/// The viewer's overrides (patches 104 and 105) show a hidden widget, hide a shown one and draw +/// The viewer's overrides (patches 105 and 106) show a hidden widget, hide a shown one and draw /// one with another form, by object; an entry of `/Annots` that isn't a dictionary is skipped. #[test] fn annotation_and_appearance_overrides_change_only_the_view() { diff --git a/crates/render/src/raster/tests/isolated_cmyk_groups.rs b/crates/render/src/raster/tests/isolated_cmyk_groups.rs index aa15019277..65b44ec93e 100644 --- a/crates/render/src/raster/tests/isolated_cmyk_groups.rs +++ b/crates/render/src/raster/tests/isolated_cmyk_groups.rs @@ -88,11 +88,12 @@ endstream endobj assert!(distance(&group, &square) <= 3, "{mode}: group {group:?}, square {square:?}"); } - // A group whose contents have no ink tints (here the second element is RGB) is - // composited in RGB exactly as before: the same as a group blending in DeviceRGB. - let rgb = |_: [f32; 4]| "1 0 0 rg".to_string(); - let cmyk_group = render("/CS /DeviceCMYK /I true", &rgb); - let rgb_group = render("/CS /DeviceRGB /I true", &rgb); + // A group whose contents have no ink tints (here the second element draws an RGB image + // over the page) is composited in RGB exactly as before: the same as a group blending + // in DeviceRGB. + let image = |_: [f32; 4]| "q 370 0 0 60 0 0 cm BI /W 1 /H 1 /CS /DeviceRGB /BPC 8 /F /AHx ID ff0000> EI Q".to_string(); + let cmyk_group = render("/CS /DeviceCMYK /I true", &image); + let rgb_group = render("/CS /DeviceRGB /I true", &image); assert_eq!(cmyk_group.rgba, rgb_group.rgba, "contents without tints are drawn as before"); } @@ -247,6 +248,32 @@ trailer << /Root 1 0 R >> p.rgba[((y * p.width + x) * 4) as usize..][..3].to_vec() } +/// An RGB colour painted in an isolated `DeviceCMYK` group is blended as the process ink that +/// draws back to it through the CMYK profile (hayro-interpret patch (104)), instead of +/// sending the whole group back to RGB blending. +/// +/// Under `/BM /Difference` with black ink, a colour's cyan, magenta and yellow are +/// complemented and its black kept (ISO 32000-2 §11.3.4), so 50% grey, mostly cyan, magenta +/// and yellow ink, comes out a warm grey: neither the neutral grey of RGB blending nor the +/// near-black of a grey made from black ink alone. Acrobat draws the same blend on a CMYK page +/// as (137, 112, 106). Painted without a blend, a colour inside the profile's gamut draws as +/// itself, and pure red is clipped to the profile's red, as in Acrobat. +#[test] +fn an_rgb_colour_in_an_isolated_cmyk_group_blends_as_its_ink() { + let resources = "/ExtGState << /D << /BM /Difference >> >>"; + let pixel = |group: &str| cmyk_group_pixel([100, 100], "/G Do", [20, 20, 80, 80], group, resources, (50, 50)); + let near = |a: &[u8], b: [u8; 3], within: u8| a.iter().zip(b).all(|(a, b)| a.abs_diff(b) <= within); + + let blended = pixel("0.5 0.5 0.5 rg 20 20 60 60 re f /D gs 0 0 0 1 k 20 20 60 60 re f"); + assert!(near(&blended, [137, 112, 106], 10), "grey under Difference with black: {blended:?}"); + + // (colour, expected pixel, how far from it) + for (rgb, expected, within) in [("0.8 0.4 0.2", [204, 102, 51], 3), ("0.3 0.7 0.4", [77, 179, 102], 3), ("1 0 0", [238, 32, 37], 8)] { + let drawn = pixel(&format!("{rgb} rg 20 20 60 60 re f")); + assert!(near(&drawn, expected, within), "{rgb} drew as {drawn:?}, expected {expected:?}"); + } +} + /// From the review of #5228: a shading pattern painted before the group left its pattern /// transform on the context, and the finished group was drawn through it, so the cyan half /// of the group showed magenta. The group must look the same whether or not a pattern was @@ -325,3 +352,15 @@ trailer << /Root 1 0 R >> assert!(rgb[1] > 120 && rgb[0] < 60 && rgb[2] < 120, "blended in CMYK, green: {rgb:?}"); } } + +/// An RGB colour with a non-finite component has no tints: converting it to RGB first would +/// give some finite colour that is not its ink. +#[test] +fn an_rgb_colour_with_a_non_finite_component_has_no_tints() { + use hayro::hayro_interpret::color::AlphaColor; + use hayro::hayro_interpret::color::Color; + let rgb = |c: [f32; 3]| Color::from_rgba(AlphaColor::new([c[0], c[1], c[2], 1.0])).ink_tints(); + assert!(rgb([0.5, 0.5, 0.5]).is_some()); + assert_eq!(rgb([f32::NAN, 0.5, 0.5]), None); + assert_eq!(rgb([0.5, f32::INFINITY, 0.5]), None); +} diff --git a/crates/sign/src/pdf/listing.rs b/crates/sign/src/pdf/listing.rs index c2a7183db1..2d41477127 100644 --- a/crates/sign/src/pdf/listing.rs +++ b/crates/sign/src/pdf/listing.rs @@ -431,8 +431,12 @@ pub fn list(doc: &Document, bytes: &[u8], trust: &TrustStore) -> Vec Vec { let pages = annot_pages(doc); - let mut out = Vec::new(); + let mut out: Vec = Vec::new(); let mut field_values: HashSet = HashSet::new(); + // Fields that name one signature value are judged once: validation does not depend on the + // field, and repeating it per field would repeat the digest, chain and evidence work. + // The row of `out` that holds each value's verdict: a later field copies it, not a map of copies. + let mut judged: HashMap = HashMap::new(); let (fields, whole) = walk_sig_fields(doc); for f in fields { if let Some(v_ref) = f.dict.get(b"V").and_then(Object::as_ref) { @@ -445,6 +449,13 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig .filter(|r| r.len() == 4) .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]); let visible = widget.is_some_and(|(_, w)| widget_shown(doc, w)); + // A repeat of a judged value reads nothing of it: the signature dictionary is not copied. + let shared = f.dict.get(b"V").and_then(Object::as_ref).filter(|_| f.dict.get(b"V").is_some_and(|v| doc.resolve(v).as_dict().is_some())); + if let Some(first) = shared.and_then(|r| judged.get(&r)).and_then(|&i| out.get(i)) { + let own = SignatureInfo { field: f.name.clone(), signed: true, page, rect, visible, ..first.clone() }; + out.push(own); + continue; + } let v = f.signature(doc); let mut info = SignatureInfo { field: f.name.clone(), @@ -477,6 +488,9 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig if let Some(v) = v { validate_into(doc, bytes, trust, &v, &mut info, cache); } + if let Some(r) = shared { + judged.insert(r, out.len()); + } out.push(info); } // Standalone document timestamps (ISO 32000-2 §12.8.2.2) live outside AcroForm fields. diff --git a/crates/sign/src/pdf/validate.rs b/crates/sign/src/pdf/validate.rs index 52bf79351f..d3ad411a62 100644 --- a/crates/sign/src/pdf/validate.rs +++ b/crates/sign/src/pdf/validate.rs @@ -371,7 +371,11 @@ pub(crate) const MAX_DSS_BYTES: usize = 64 << 20; pub(crate) struct DssEvidence<'a> { doc: &'a Document, items: std::vec::IntoIter, - seen: HashSet, + /// Object numbers read: `Document::get` resolves a reference by number, so `5 0 R` and `5 1 R` + /// are one stream. + seen: HashSet, + /// Full references already tried, resolved or not. + tried: HashSet, taken: usize, left: usize, /// A cap stopped the array being read in full. @@ -381,7 +385,7 @@ pub(crate) struct DssEvidence<'a> { impl<'a> DssEvidence<'a> { pub(crate) fn new(doc: &'a Document, dss: &Dict, key: &[u8]) -> Self { let items = dss.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()).unwrap_or_default(); - DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } + DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), tried: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } } } @@ -395,11 +399,21 @@ impl Iterator for DssEvidence<'_> { // Repeats are skipped before the caps count anything, so repeating one stream cannot use // them up; the scan of the array itself is bounded by the parser's cap on array length. for item in self.items.by_ref() { - if item.as_ref().is_some_and(|r| !self.seen.insert(r)) { + // Resolved first: a reference to a generation the slot doesn't hold reads as null and + // must not make a later, valid reference to the same object number a repeat. + // A repeat is skipped before anything is resolved: a long chain of repeats costs no + // lookups. The number is recorded only once the reference gave a stream. + // Each full reference is tried once, so one that never reaches a stream (a cycle) costs + // its lookups once, however often it is listed; `seen` keeps object numbers apart so a + // dead generation can't hide a live one. + if item.as_ref().is_some_and(|r| self.seen.contains(&r.num) || !self.tried.insert(r)) { continue; } let o = self.doc.resolve(&item); let Object::Stream(s) = o.as_ref() else { continue }; + if let Some(r) = item.as_ref() { + self.seen.insert(r.num); + } if self.taken >= MAX_DSS_ENTRIES { self.cut = true; return None; diff --git a/crates/sign/src/pdf_discovery_tests.rs b/crates/sign/src/pdf_discovery_tests.rs index 29f115cb07..d3250d62ed 100644 --- a/crates/sign/src/pdf_discovery_tests.rs +++ b/crates/sign/src/pdf_discovery_tests.rs @@ -373,6 +373,19 @@ fn only_grew_compares_huge_arrays_without_quadratic_work() { assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); } +/// A reference to a generation the object doesn't have reads as null: it is no repeat of the valid +/// reference to the same object number that follows it. +#[test] +fn dss_evidence_skips_a_dead_reference_without_hiding_the_live_one() { + let mut doc = fixture(&[]); + let live = doc.add(Object::Stream(Stream::flate(Dict::new(), b"evidence"))); + let dead = ObjRef::new(live.num, live.generation + 1); + let mut dss = Dict::new(); + dss.set(b"CRLs".to_vec(), Object::Array(vec![Object::Ref(dead), Object::Ref(live)])); + let read: Vec> = DssEvidence::new(&doc, &dss, b"CRLs").collect(); + assert_eq!(read, vec![b"evidence".to_vec()]); +} + #[test] fn dss_evidence_decodes_each_stream_once_within_its_caps() { // One small stream listed 1000 times, then streams that inflate past the evidence cap. diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index e765941c9a..5d24d05dce 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -325,6 +325,21 @@ fn an_indirect_annots_array_is_judged_by_the_annotations_it_gains() { assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); } +/// A signature widget taken off the page's inline /Annots array is covered by no permission (the +/// indirect-array case is the test above), and stays a change whatever level the signature has. +#[test] +fn removing_a_widget_from_an_inline_annots_array_is_not_a_comment_change() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let page = pdfcraft_cos::ObjRef { num: 3, generation: 0 }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &opts()).unwrap(); + let inline = open(&signed).get(page).as_dict().and_then(|d| d.get(b"Annots").cloned()).unwrap(); + assert!(matches!(inline, Object::Array(ref a) if !a.is_empty()), "the page keeps its widgets inline"); + let stripped = edit_after(&signed, |doc| doc.update_dict(page, |p| p.set(b"Annots".to_vec(), Object::Array(vec![]))).unwrap()); + let s = signatures(&open(&stripped), &stripped, &trust).into_iter().find(|s| s.signed).unwrap(); + assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); +} + /// Issue #1328: a signature field whose /V is null, directly or through a reference, is unsigned: /// it is listed as unsigned and can be signed by name. #[test] @@ -3354,6 +3369,38 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val ); } +/// Fields that name one signature value are listed each, under their own name, with the verdict +/// the value earned once. +#[test] +fn fields_sharing_a_signature_value_are_listed_with_one_verdict() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + let v = open(&signed).get(pdfcraft_cos::ObjRef { num: 6, generation: 0 }).as_dict().and_then(|d| d.get(b"V").cloned()).unwrap(); + let shared = edit_after(&signed, |doc| { + let mut w = pdfcraft_cos::Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"T".to_vec(), PdfString::text("Twin")); + w.set(b"V".to_vec(), v.clone()); + w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4])); + let w = doc.add(Object::Dict(w)); + let root = doc.root().unwrap(); + let af = doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm").cloned()).unwrap(); + let af = doc.resolve(&af).as_dict().cloned().unwrap(); + let mut fields = af.get(b"Fields").map(|f| doc.resolve(f)).and_then(|f| f.as_array().cloned()).unwrap(); + fields.push(Object::Ref(w)); + let mut af = af; + af.set(b"Fields".to_vec(), Object::Array(fields)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(af))).unwrap(); + }); + let list = signatures(&open(&shared), &shared, &trust); + let (a, b) = (list.iter().find(|s| s.field == "Approval").unwrap(), list.iter().find(|s| s.field == "Twin").unwrap()); + assert!(a.signed && b.signed); + assert_eq!((a.status, &a.modification, &a.signer, a.signed_len), (b.status, &b.modification, &b.signer, b.signed_len)); +} + #[test] fn the_catalogs_perms_do_not_change_under_no_changes() { // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later diff --git a/crates/ui-egui/src/workflows/print_ui/draft.rs b/crates/ui-egui/src/workflows/print_ui/draft.rs index 2b661426a6..02dbb97856 100644 --- a/crates/ui-egui/src/workflows/print_ui/draft.rs +++ b/crates/ui-egui/src/workflows/print_ui/draft.rs @@ -441,7 +441,7 @@ impl PrintDraft { m => m, }); let paper = self.paper_size(); - let raster = self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale }); + let raster = self.raster(); // A selected area prints alone, its one page sized like a page of its own: Size is its // only handling (the dialog offers no other while it is chosen). if self.which == Which::Area { @@ -509,7 +509,7 @@ impl PrintDraft { /// Print as image: pictures at the chosen resolution, grey when grayscale is asked for. fn raster(&self) -> Option { - self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale }) + self.as_image.then_some(print::Raster { dpi: self.dpi, grayscale: self.grayscale, colour: None }) } /// Whether multiple pages per sheet should be cut and stacked into sorted piles. @@ -556,7 +556,7 @@ impl PrintDraft { self.chosen_duplex() }, scaling: spool::Scaling::Exact, - grayscale: print::driver_needs_grayscale(self.grayscale, self.raster()), + grayscale: print::driver_needs_grayscale(self.grayscale, self.raster().as_ref()), // Set when the copy sent is made grey (`spooler_copy`, #5806). grey_pages: false, title: title.to_string(), diff --git a/crates/ui-egui/tests/flatpak_updates.rs b/crates/ui-egui/tests/flatpak_updates.rs new file mode 100644 index 0000000000..536613f08a --- /dev/null +++ b/crates/ui-egui/tests/flatpak_updates.rs @@ -0,0 +1,27 @@ +//! Release and source Flatpaks must allow release checks and host update helpers. +fn finish_args(manifest: &str) -> Vec<&str> { + manifest + .split("finish-args:") + .nth(1) + .unwrap_or_default() + .lines() + .take_while(|line| line.is_empty() || line.starts_with(' ') || line.starts_with('#')) + .filter_map(|line| line.trim().strip_prefix("- ")) + .map(|arg| arg.trim_matches(['\'', '"'])) + .collect() +} + +#[test] +fn flatpak_update_permissions() { + let manifests = [ + include_str!("../../../packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml"), + include_str!("../../../packaging/linux/flatpak/ai.storyteller.pdfcraft.yml"), + ]; + let args = manifests.map(finish_args); + assert_eq!(args[0], args[1], "source and release permissions must agree"); + for permissions in args { + for required in ["--share=network", "--talk-name=org.freedesktop.Flatpak"] { + assert!(permissions.contains(&required), "missing update permission: {required}"); + } + } +} diff --git a/crates/xfa/src/data.rs b/crates/xfa/src/data.rs index 34be028d89..4e7de4af74 100644 --- a/crates/xfa/src/data.rs +++ b/crates/xfa/src/data.rs @@ -148,13 +148,21 @@ pub enum DataOp { impl DataNode { /// The node at `path`, made (with empty siblings before it) where missing; `None` past /// the index cap. - fn ensure(&mut self, path: &[(String, usize)]) -> Option<&mut DataNode> { + fn ensure(&mut self, path: &[(String, usize)], budget: &mut usize) -> Option<&mut DataNode> { let mut node = self; for (name, idx) in path { if *idx >= MAX_INDEX { return None; } let have = node.children.iter().filter(|c| &c.name == name).count(); + // Every skipped sibling owns a copy of the name: bounded as `Build::at` bounds a write. + // The budget is the whole path's, not each level's: a deep path that is affordable at + // every level taken alone would otherwise allocate its way past the bound. + let need = (*idx + 1).saturating_sub(have).saturating_mul(name.len()); + if need > *budget { + return None; + } + *budget -= need; for _ in have..=*idx { node.children.push(DataNode { name: name.clone(), ..Default::default() }); } @@ -166,9 +174,11 @@ impl DataNode { /// Apply `op` to this tree (the root is the `xfa:data` element), as [`write_data_ops`] /// will to the packet, so scripts that run before it is written see the change. pub fn apply(&mut self, op: &DataOp) { + // One budget for the op's whole path, as `Build::at` bounds a write. + let mut budget = build::MAX_NEW_NAME_BYTES; match op { DataOp::Value { path, text } => { - if let Some(n) = self.ensure(path) { + if let Some(n) = self.ensure(path, &mut budget) { // A multi-select list's picks, one `value` each (see `value_at`), told apart // as the packet writer does: `value` leaves only, or a group holding none. let picks = if n.children.is_empty() { n.group } else { n.children.iter().all(|c| c.name == "value" && c.children.is_empty()) }; @@ -189,7 +199,7 @@ impl DataNode { if let Some(&last) = count.checked_sub(1).as_ref() { let mut path = parent.clone(); path.push((name.clone(), last.min(MAX_INDEX.saturating_sub(1)))); - let _ = self.ensure(&path); + let _ = self.ensure(&path, &mut budget); } } DataOp::Remove(path) => { diff --git a/crates/xfa/src/data/build.rs b/crates/xfa/src/data/build.rs index 98e34f0f49..e90904d60c 100644 --- a/crates/xfa/src/data/build.rs +++ b/crates/xfa/src/data/build.rs @@ -38,7 +38,7 @@ pub(super) const MAX_INDEX: usize = 10_000; /// Most bytes of names the nodes one write creates hold together: every sibling an index /// skips over gets its own copy of the name. -const MAX_NEW_NAME_BYTES: usize = 4 << 20; +pub(super) const MAX_NEW_NAME_BYTES: usize = 4 << 20; /// What one write may still create: nodes and bytes of their names. pub(super) struct NewNodes { diff --git a/crates/xfa/src/data/passwords.rs b/crates/xfa/src/data/passwords.rs index 0f98a3f2b4..896d513f2c 100644 --- a/crates/xfa/src/data/passwords.rs +++ b/crates/xfa/src/data/passwords.rs @@ -1,26 +1,38 @@ use super::import::datasets_packet; use super::merge::Op; -use super::path::{data_path, parse_datasets}; +use super::path::{data_path, parse_datasets_packet}; use super::values::{Want, path_of, text_of, with_template}; use super::write::{datasets_stream_ref, rewrite}; use super::{DataNode, DataPath, Document, MAX_DEPTH, Node, ObjRef, Object, SomIndices, Template, Ui, XfaError}; +/// How deep the AcroForm field tree is walked for password fields: as deep as the form's own +/// field walk (`pdfcraft_forms::fields`), which is what editing and dataset synchronization see. +const MAX_ACRO_DEPTH: usize = 64; + /// The SOM path and default value of every AcroForm text field with the Password flag (bit 14, /// `1 << 13`) in `doc`, for static XFA forms whose password fields set the flag in the AcroForm -/// dictionary. -fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { +/// dictionary. `None` when the walk was cut short at [`MAX_ACRO_DEPTH`]: which branch is cut +/// depends on the order the tree is visited in, so a field the form's own walk reaches may have +/// been missed, and the caller can't tell that no password field is left. +fn acro_password_fields(doc: &Document) -> Option> { const PASSWORD_FLAG: i64 = 1 << 13; - let Some(root) = doc.root() else { return Vec::new() }; + let Some(root) = doc.root() else { return Some(Vec::new()) }; let catalog = doc.get(root); - let Some(acro) = catalog.as_dict().and_then(|c| c.get(b"AcroForm")).map(|a| doc.resolve(a)) else { return Vec::new() }; + let Some(acro) = catalog.as_dict().and_then(|c| c.get(b"AcroForm")).map(|a| doc.resolve(a)) else { return Some(Vec::new()) }; let Some(top) = acro.as_dict().and_then(|a| a.get(b"Fields")).map(|f| doc.resolve(f)).and_then(|f| f.as_array().cloned()) else { - return Vec::new(); + return Some(Vec::new()); }; struct AcroWalk<'a> { doc: &'a Document, - seen: std::collections::HashSet, + /// Each field reached, with what it was reached with: its parent's name, type, flags and + /// default, which decide what it classifies as. + seen: std::collections::HashMap, + /// Fields past the depth limit, with what they were reached with. One reached again from + /// another root in the same way was classified there; any other is not fully read. + cut: Vec<(ObjRef, Reach)>, out: Vec<(String, String)>, } + type Reach = (String, Option>, i64, Option); impl AcroWalk<'_> { fn walk( &mut self, @@ -31,12 +43,19 @@ fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { inherited_dv: Option<&str>, depth: usize, ) { - if depth > 32 { + if depth > MAX_ACRO_DEPTH { + let reach: Reach = (parent_name.to_string(), inherited_ft.map(<[u8]>::to_vec), inherited_ff, inherited_dv.map(str::to_string)); + self.cut.extend(items.iter().filter_map(Object::as_ref).map(|r| (r, reach.clone()))); return; } for item in items { let Some(r) = item.as_ref() else { continue }; - if !self.seen.insert(r) { + let reach: Reach = (parent_name.to_string(), inherited_ft.map(<[u8]>::to_vec), inherited_ff, inherited_dv.map(str::to_string)); + // The reach a field was classified with stays: a later visit that skips it classified + // nothing, so it must not stand in for the cut-off one. + if let std::collections::hash_map::Entry::Vacant(slot) = self.seen.entry(r) { + slot.insert(reach); + } else { continue; } let obj = self.doc.get(r); @@ -65,20 +84,35 @@ fn acro_password_fields(doc: &Document) -> Vec<(String, String)> { } } } - let mut w = AcroWalk { doc, seen: std::collections::HashSet::new(), out: Vec::new() }; + let mut w = AcroWalk { doc, seen: std::collections::HashMap::new(), cut: Vec::new(), out: Vec::new() }; w.walk(&top, "", None, 0, None, 0); - w.out + // Decided once every root has been visited: a branch cut deep down may be reached shallower + // from a later root, and is then read in full. + let incomplete = w.cut.iter().any(|(r, reach)| w.seen.get(r) != Some(reach)); + (!incomplete).then_some(w.out) } /// The `Want` entries needed to restore password fields in `doc`'s datasets packet to their /// defaults (or empty when they have no default). -fn password_dataset_wants(doc: &Document) -> Vec { +/// +/// The packet is read as the datasets writer reads it ([`parse_datasets_packet`]): a packet the +/// writer can't read holds no value it wrote, so there is nothing typed in it to clear. An error +/// when the password fields can't all be found (a field tree deeper than the field walk reaches): +/// the save is refused rather than written with a typed password left in the data. +fn password_dataset_wants(doc: &Document) -> Result, XfaError> { if datasets_stream_ref(doc).is_none() { - return Vec::new(); + return Ok(Vec::new()); } - let Ok(Some(packet)) = datasets_packet(doc) else { return Vec::new() }; - let Some(data) = parse_datasets(&packet) else { return Vec::new() }; - with_template(doc, None, |tpl| { + // Checked before the packet is read: a packet that can't be parsed leaves nothing to clear, but + // a form whose fields can't all be found is refused whatever the packet holds. + let Some(acro_fields) = acro_password_fields(doc) else { + return Err(XfaError::Malformed( + "the form's fields nest too deeply to find every password field, so typed passwords could not be removed from its XFA data".into(), + )); + }; + let Ok(Some(packet)) = datasets_packet(doc) else { return Ok(Vec::new()) }; + let Some(data) = parse_datasets_packet(&packet) else { return Ok(Vec::new()) }; + Ok(with_template(doc, None, |tpl| { let mut candidates: Vec<(String, DataPath, String)> = Vec::new(); let mut seen = std::collections::HashSet::new(); if let Some(t) = tpl { @@ -153,7 +187,7 @@ fn password_dataset_wants(doc: &Document) -> Vec { } } } - for (som, default) in acro_password_fields(doc) { + for (som, default) in acro_fields { let path = path_of(tpl, &som); if !path.is_empty() && seen.insert(path.clone()) { candidates.push((som, path, default)); @@ -166,27 +200,29 @@ fn password_dataset_wants(doc: &Document) -> Vec { (!cur.is_empty() && cur != default).then_some(Want { som, path, text: default, ensure: false }) }) .collect() - }) + })) } /// Whether any XFA password field (`` or an AcroForm text field with the Password -/// flag) holds a value in `doc`'s datasets packet beyond its default (#3944). +/// flag) holds a value in `doc`'s datasets packet beyond its default (#3944). A form whose +/// password fields can't all be found counts as holding some, so that clearing them +/// ([`clear_password_values`]) reports why the document can't be written without them. pub fn has_password_values(doc: &Document) -> bool { - !password_dataset_wants(doc).is_empty() + !password_dataset_wants(doc).is_ok_and(|w| w.is_empty()) } /// Clear typed password values from `doc`'s XFA datasets packet (restoring a template default /// where the field has one, else emptying the element), replacing the datasets stream in place /// (#3944). Returns how many password fields were cleared in the datasets. pub fn clear_password_values(doc: &mut Document) -> Result { - let wants = password_dataset_wants(doc); + let wants = password_dataset_wants(doc)?; if wants.is_empty() { return Ok(0); } let count = wants.len(); let reuse = datasets_stream_ref(doc); let report = rewrite(doc, &[Op::Values(&wants)], reuse)?; - if !report.written || !password_dataset_wants(doc).is_empty() { + if !report.written || !password_dataset_wants(doc)?.is_empty() { let reason = report.warnings.first().map_or("the XFA datasets password values could not be cleared", String::as_str); return Err(XfaError::Malformed(reason.to_string())); } diff --git a/crates/xfa/src/data/path.rs b/crates/xfa/src/data/path.rs index 23c5e5ba5d..7fdba77071 100644 --- a/crates/xfa/src/data/path.rs +++ b/crates/xfa/src/data/path.rs @@ -24,6 +24,33 @@ pub fn parse_datasets(xdp: &str) -> Option { Some(to_node(data, 0)) } +/// The `xfa:data` element of a datasets packet as the datasets writer reads it before changing it: +/// a packet of an array may use the `xfa` and `xdp` prefixes its XDP envelope declares, so it is +/// read inside an element declaring them when it doesn't parse alone, and the datasets element in +/// the XFA data namespace is preferred over another with the same local name. Reading the packet +/// any other way could miss the values the writer put in it. +pub(super) fn parse_datasets_packet(text: &str) -> Option { + if crate::xml_nesting_depth(text) > crate::MAX_XML_DEPTH { + return None; + } + let opts = || roxmltree::ParsingOptions { allow_dtd: false, nodes_limit: MAX_NODES, ..Default::default() }; + let wrapped; + let doc = match roxmltree::Document::parse_with_options(text, opts()) { + Ok(d) => d, + Err(_) => { + wrapped = format!("{}{text}{}", super::merge::WRAP_OPEN, super::merge::WRAP_CLOSE); + roxmltree::Document::parse_with_options(&wrapped, opts()).ok()? + } + }; + let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name; + let datasets = doc + .descendants() + .find(|n| is(n, "datasets") && n.tag_name().namespace() == Some(super::XFA_DATA_NS)) + .or_else(|| doc.descendants().find(|n| is(n, "datasets")))?; + let data = datasets.children().find(|n| is(n, "data"))?; + Some(to_node(data, 0)) +} + /// The data path a SOM expression names: `form[0].#subform[0].name[1]` → `[(form, 0), (name, 1)]`. /// Unnamed containers (`#subform`) have no data node. pub fn som_to_path(som: &str) -> DataPath { diff --git a/crates/xfa/src/data/write.rs b/crates/xfa/src/data/write.rs index 4667036fe5..554685e1ad 100644 --- a/crates/xfa/src/data/write.rs +++ b/crates/xfa/src/data/write.rs @@ -82,6 +82,11 @@ pub fn remove_data_instance(doc: &mut Document, path: &DataPath) -> Result) let mut run: Vec = Vec::new(); // Instances queued in this run, by parent and name: asking for more only queues the rest. let mut queued: HashMap<(&DataPath, &str), usize> = HashMap::new(); - let (mut ensured, mut capped) = (0usize, false); + let (mut ensured, mut ensured_bytes, mut capped, mut capped_bytes) = (0usize, 0usize, false, false); let flush = |run: &mut Vec, steps: &mut Vec>, removes: &mut Vec>| { if !run.is_empty() { let mut seen = std::collections::HashSet::new(); @@ -117,12 +122,24 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) } DataOp::Instances { parent, name, count } => { let done = queued.entry((parent, name.as_str())).or_insert(0); + // The path (its segments' buffers and their text) is held by the queued step, which a + // clone then `push` can leave with up to twice the room, and cloned again into + // `flush`'s dedup set, beside the label and the step itself. + let segment = std::mem::size_of::<(String, usize)>(); + let path_bytes = + parent.iter().fold(segment.saturating_add(name.len()), |n, (p, _)| n.saturating_add(segment).saturating_add(p.len())); + let per_instance = path_bytes.saturating_mul(3).saturating_add(name.len()).saturating_add(16 + std::mem::size_of::()); while *done < (*count).min(MAX_INDEX) { if ensured >= MAX_ENSURED { capped = true; break; } + if ensured_bytes.saturating_add(per_instance) > MAX_ENSURED_BYTES { + capped_bytes = true; + break; + } ensured += 1; + ensured_bytes += per_instance; #[cfg(test)] ENSURED.with(|n| n.set(n.get() + 1)); let k = *done; @@ -154,6 +171,11 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) if capped { written.warnings.push(format!("XFA data instances not written past {MAX_ENSURED}: the rows were added and removed too often")); } + if capped_bytes { + written + .warnings + .push(format!("XFA data instances not written past {MAX_ENSURED_BYTES} bytes of names: the rows were added and removed too often")); + } Ok(written) } diff --git a/crates/xfa/src/tests.rs b/crates/xfa/src/tests.rs index 682887e673..c1abcce190 100644 --- a/crates/xfa/src/tests.rs +++ b/crates/xfa/src/tests.rs @@ -7114,6 +7114,99 @@ fn a_container_is_as_wide_as_a_turned_draw_stands() { } } +/// The static fixture with its text field (`form1[0].page1[0].name[0]`) flagged Password and +/// `levels` unnamed fields between `form1[0]` and `page1[0]`, which add no SOM segment. +fn static_password_doc(levels: usize, data: &str) -> Document { + let mut doc = Document::open(Arc::new(static_shell(data))).unwrap(); + doc.update_dict(pdfcraft_cos::ObjRef::new(8, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1 << 13))).unwrap(); + let mut next = pdfcraft_cos::ObjRef::new(7, 0); + for _ in 0..levels { + let mut d = Dict::new(); + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(next)])); + next = doc.add(Object::Dict(d)); + } + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(next)]))).unwrap(); + doc +} + +/// #3944: a Password-flagged AcroForm field of a static XFA form nested deeper than 32 levels (as +/// deep as the form's own field walk reaches) has its typed value cleared from the datasets. +#[test] +fn a_deeply_nested_acroform_password_field_is_cleared_from_the_datasets() { + let data = "hunter2"; + let mut doc = static_password_doc(40, data); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); + assert!(!String::from_utf8_lossy(&xfa_bytes(&doc)).contains("hunter2")); +} + +/// A field tree deeper than the walk reaches can't be shown to hold no password field: clearing +/// refuses (so the save does) rather than leave a typed password in the datasets. +#[test] +fn a_field_tree_too_deep_to_check_for_passwords_refuses_to_clear() { + let mut doc = static_password_doc(80, "hunter2"); + assert!(has_password_values(&doc)); + assert!(clear_password_values(&mut doc).is_err()); + // A shallow form without a typed value has nothing to clear. + let mut doc = static_password_doc(0, ""); + assert!(!has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 0); +} + +/// A datasets packet of an XFA array may use the `xfa` prefix its preamble declares: the typed +/// password in it is still found and cleared, as the datasets writer reads that packet too. +#[test] +fn a_password_in_a_datasets_packet_using_the_preamble_prefix_is_cleared() { + let mut doc = static_password_doc(0, ""); + let pre = r#""#; + let tpl = r#""#; + let ds = "hunter2"; + let parts: Vec = [("preamble", pre), ("template", tpl), ("datasets", ds), ("postamble", "")] + .into_iter() + .flat_map(|(name, text)| { + let r = doc.add(Object::Stream(pdfcraft_cos::Stream::flate(Dict::new(), text.as_bytes()))); + [Object::String(pdfcraft_cos::PdfString::literal(name.as_bytes().to_vec())), Object::Ref(r)] + }) + .collect(); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + let mut acro = c.get(b"AcroForm").and_then(Object::as_dict).cloned().unwrap(); + acro.set(b"XFA".to_vec(), Object::Array(parts)); + c.set(b"AcroForm".to_vec(), Object::Dict(acro)); + }) + .unwrap(); + assert!(crate::data::parse_datasets(ds).is_none(), "the packet doesn't parse alone"); + assert!(has_password_values(&doc)); + assert_eq!(clear_password_values(&mut doc).unwrap(), 1); + assert!(!has_password_values(&doc)); +} + +/// Every sibling a SOM index skips owns a copy of the element name: the in-memory builder refuses +/// a path whose copies would pass the name-byte bound the writer holds itself to, and creates +/// nothing for it. +#[test] +fn the_in_memory_data_builder_bounds_copied_name_bytes() { + let mut root = DataNode::default(); + // 1 KiB x 9,999 skipped siblings is about 10 MB, over the 4 MiB bound. + let long = "n".repeat(1 << 10); + root.apply(&DataOp::Value { path: vec![(long.clone(), 9_999)], text: "x".into() }); + assert!(root.children.is_empty()); + // Within the bound, the same shape still builds (and keeps indexes). + root.apply(&DataOp::Value { path: vec![(long, 100)], text: "x".into() }); + assert_eq!(root.children.len(), 101); +} + +/// Each queued instance owns a copy of its parent path: a long ancestor stops the queue at the +/// byte allowance, reported as capped, instead of being multiplied by the instance count. +#[test] +fn queued_instances_are_bounded_by_the_path_bytes_they_copy() { + let mut doc = Document::new_empty(); + let parent: DataPath = vec![("a".repeat(16 << 20), 0), ("b".repeat(16 << 20), 0)]; + let written = crate::data::write_data_ops(&mut doc, &[DataOp::Instances { parent, name: "row".into(), count: 100 }], None).unwrap(); + assert!(written.warnings.iter().any(|w| w.contains("not written past")), "{:?}", written.warnings); +} + /// A page break splits an empty multi-line field that grows with its text (XFA 3.3 pp. 291–292: /// text splits between lines, and a field's `keep intact` is `none`): it starts where it is, its /// widget runs to the bottom of the content area, and the rest of its box, closed on every side, diff --git a/docs/acroform-script-workflows.md b/docs/acroform-script-workflows.md index 49666af3d0..0393b6e932 100644 --- a/docs/acroform-script-workflows.md +++ b/docs/acroform-script-workflows.md @@ -99,7 +99,7 @@ resets it, Revert, or a full save) has no timers. Fields that page actions and timers show, hide or give another icon (`display`, `buttonSetIcon`) change only how the document is shown, through view-only render overrides -(hayro-interpret patches 104 and 105). They aren't saved, printed or exported, and they don't +(hayro-interpret patches 105 and 106). They aren't saved, printed or exported, and they don't mark the document as changed. Buttons and the console see the fields as shown, and a `display` or icon a button writes into the document takes over from the view's. A value or a reset from a page action or timer is one undoable edit. diff --git a/docs/gaps.md b/docs/gaps.md index 1a1c04a9da..6f2ab13219 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -250,7 +250,7 @@ update the numbers. Read Out Loud; full keyboard operation of PdfCraft itself and a screen-reader audit. - **Evidence:** all 32 checker rules, figure alt text, tag property and sibling order edits, and a bounded auto-tag preview exist. Apply writes valid structure and parent-tree links for - text-only files; it refuses mixed content and existing tags; + text-only files, tagging headings by size and weight; it refuses mixed content and existing tags; `a11y.keyboard-only` (P0) planned, `a11y.app-screen-reader` partial. - **Impact:** most untagged files still need another editor to build their structure; PdfCraft itself is not yet usable without a mouse. diff --git a/docs/roadmap.md b/docs/roadmap.md index 7ca83dedd9..abc6d13b9f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -296,7 +296,7 @@ total matches the parity estimate. | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded, text-only auto-tag proposal tool for one page. Marking a figure decorative rewrites only that stream (#1324). An XFA radio item whose value contains a space is written back unchanged (#1319). Done since: #2569: Growable multiline XFA text reflows after edits, with undo/redo, save/reopen and print coverage; oversized fields still become read-only page text. Structured-data synchronization warnings retain accepted whitespace without relayout from stale data. Done since: #2572: Every repeated page-master button resolves its own field to the shared XFA click script, including print buttons. Done since: #2570: Dynamic XFA rich-text fields carry exData/maxLength into AcroForm MaxLen for reported and enforced Unicode character limits, retained after save/reopen. Done since: #2589: Active XFA skips PDF-container fallback scripts while retaining requested messages and saved scripts; ordinary/static/unlaid-out documents and explicit console/field calls keep their existing behavior. Missing: writing proposed tags and parent tree, field autotag, visual Tags/Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Reading colours: pending page backgrounds follow the configured colour (#2059). | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers
Done since: CLI run and MCP reject missing, empty, repeated and option-valued --root arguments before any tool/script execution or MCP initialization. A single-hyphen directory name remains valid; synthetic CLI regressions preserve a would-be overwritten file. `text_find` also caps the per-page match ranges it materializes at the remaining result limit plus one continuation match. CLI robustness scans visit each directory once through overlapping roots and symlink aliases. | -| M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded text-only auto-tag proposal tool; text-only Apply writes marked content, paragraph tags and a parent tree, then opens the Tags panel for review. Missing: mixed-content and field autotag, heading/list/table/figure classification, Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Done since: CLI info, text and render use dynamic XFA layout pages, with synthetic count/text/pixel and copy-permission regressions (#1645).
Reading colours: pending page backgrounds follow the configured colour (#2059). | +| M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded text-only auto-tag proposal tool; text-only Apply writes marked content, heading and paragraph tags and a parent tree, then opens the Tags panel for review. Missing: mixed-content and field autotag, list/table/figure classification, Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Done since: CLI info, text and render use dynamic XFA layout pages, with synthetic count/text/pixel and copy-permission regressions (#1645).
Reading colours: pending page backgrounds follow the configured colour (#2059). | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers
Done since: CLI run and MCP reject missing, empty, repeated and option-valued --root arguments before any tool/script execution or MCP initialization. A single-hyphen directory name remains valid; synthetic CLI regressions preserve a would-be overwritten file. `text_find` also caps the per-page match ranges it materializes at the remaining result limit plus one continuation match. | | M14 | 1.0 polish: performance, localization, installers | 120–250 | 15% | 80–150 | Local customization: adjustable, floating or side-docked hotbar with optional full-side strip and persistent tool picker; freehand highlighting is grouped with Highlight. Estimates unchanged. Done: PhotoCraft's translation system (`i18n/`: TSV catalogs, `tl!`, command-id and plural entries, system-language detection, strict catalog tests); every dialog, panel and notice goes through `tl!`. Catalogs of ≈ 2,000–2,140 entries for Japanese, Simplified and Traditional Chinese, Russian, Bulgarian, German, Spanish, French, Telugu, Hungarian, Ukrainian and Arabic (logical catalog; explicit right-to-left labels and mirrored stock chrome; end-to-end RTL workflows unqualified); the 2,674-entry Brazilian Portuguese catalog covers the current key union; Czech covers the menus. Italian has 2,528 entries: every key of the other catalogs that the code looks up. Signed and notarized macOS builds, signed Windows installers (x64, x86, ARM64), AppImage, deb, rpm, Flatpak and FreeBSD packages. #307 (bounded large-PDF memory): shared resource indexes and parser metadata, lazy inspection, bounded caches; a 9,156-page file inspects in ≈ 1.8 s at ≈ 500 MiB instead of exceeding 10 GiB. Edits let go of the previous working file before writing the next (a 1 GiB file peaks at two copies while editing, not three; gap 13). Shortcut display uses localized modifier/key names across menus, toolbar, palette, Help and tooltips (#496). Lithuanian adds a 2,578-entry catalog and date names, reviewed by a native-speaking contributor. Norwegian Bokmål adds a catalog and date names (terminology reviewed by a native speaker). Validated candidate (macOS raster probe; Windows runtime unverified): the Windows policy changes only embedded JetBrains Mono; horizontal alpha-edge energy per ink improves 2.78%, 3.87% and 1.59% at 100%, 125% and 150%, and is down 0.79% at 200%. Inter is pixel-identical; layout hashes, glyph coverage and the 8 MiB atlas are unchanged. Timing and RSS ranges overlap, with no improvement claimed. Estimates unchanged. Candidate #2383 clamps GPU requests to supported limits while retaining the adapter texture extent; synthetic V3D/downlevel and Windows GPU-required tests pass, but Raspberry Pi startup is unverified. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation Linux/Unix UI fallback fonts also use Fontconfig directories, including Nix store paths, with bounded discovery and the existing candidate allowlist; relative includes and digit-prefixed config filtering have isolated regressions. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation XDG desktops use the configuration base for preferences with non-overwriting migration and original-data preservation (#1395). Traditional Chinese accessibility statuses, rule semantics and Taiwan form/table terms have catalog regressions. Long main-menu lists scroll in short windows (#3373). Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean, file-backed input and global memory budgets, performance budgets, keyboard-only operation. Candidate #3558: bounded-worker macOS per-file PDFCraft association recovery after desktop open/save/sign; metadata-only watcher handling and upstream #3042 shared-writer preservation; recovery parser limits do not narrow save limits. Native workflow qualified on macOS 27.0.1; other versions and physical SMB/FUSE volumes remain unqualified. Estimates unchanged. Missing: end-to-end RTL workflow qualification, a Chinese web UI face, Hindi/Indonesian/Korean/Vietnamese, file-backed input and global memory budgets, performance budgets, keyboard-only operation. | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers | diff --git a/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml b/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml index d781ad6c7c..8d2fb49192 100644 --- a/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml +++ b/packaging/linux/flatpak/ai.storyteller.pdfcraft.bundle.yml @@ -17,6 +17,8 @@ runtime-version: '25.08' sdk: org.freedesktop.Sdk command: pdfcraft finish-args: + # The shared updater inspects and updates this installation via flatpak-spawn --host. + - --talk-name=org.freedesktop.Flatpak - --filesystem=xdg-data/ArtCraft/versions:ro # Display: Wayland first; X11 only when no Wayland compositor is running. - --socket=wayland diff --git a/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml b/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml index fba62fe382..2c5d6d88c6 100644 --- a/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml +++ b/packaging/linux/flatpak/ai.storyteller.pdfcraft.yml @@ -18,6 +18,8 @@ sdk-extensions: - org.freedesktop.Sdk.Extension.rust-stable command: pdfcraft finish-args: + # The shared updater inspects and updates this installation via flatpak-spawn --host. + - --talk-name=org.freedesktop.Flatpak - --filesystem=xdg-data/ArtCraft/versions:ro # Display: Wayland first; X11 only when no Wayland compositor is running. - --socket=wayland diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 399e4378d1..babc7c6f3c 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5920,7 +5920,7 @@ status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] evidence = ["crates/ui-egui/tests/it/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/it/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::the_renderer_check_refuses_text_still_shown_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_does_not_survive_in_info_or_xmp", "crates/redact/src/tests.rs::metadata_is_cleared_when_the_removed_text_was_cut_short", "crates/redact/src/tests/metadata_fail_closed.rs::an_xmp_stream_whose_filter_data_is_corrupt_goes", "crates/redact/src/tests/elsewhere.rs::places_apply_leaves_that_still_hold_the_removed_text_are_counted_and_kept", "crates/engine/src/tests.rs::applied_redactions_report_a_bookmark_that_still_holds_the_removed_text", "crates/automation/tests/it/automation.rs::redact_apply_reports_places_that_still_hold_the_removed_text", "crates/ui-egui/tests/it/redact.rs::applying_says_where_the_removed_text_is_still_held", "crates/redact/src/tests/metadata_matching.rs::a_heading_tracked_with_tj_numbers_matches_the_metadata", "crates/redact/src/tests/metadata_matching.rs::unicode_forms_of_the_removed_text_match", "crates/redact/src/tests/metadata_matching.rs::a_utf16le_string_is_searched_as_utf16le", "crates/redact/src/tests/identification.rs::the_identification_of_a_removed_packet_is_written_back", "crates/automation/tests/it/automation.rs::redact_apply_reports_the_identification_written_back", "crates/ui-egui/tests/it/redact.rs::applying_says_the_pdfa_identification_was_written_back"] -notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" +notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known, or with bytes the content parser can't read in its content or in a form under a mark, which can hide text other readers draw. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" [[feature]] id = "protect.redact-glyph-removal" @@ -7563,8 +7563,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] -evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary"] -notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." +evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::one_bit_images_past_80_million_pixels_are_stored_as_group_4", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] +notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), except images and masks already of one bit a pixel whose rows hold no more bytes than 80 million pixels of eight bits (a byte for each 8 samples of a row and one for any left over, so up to 640 million pixels when wide, and a strip one sample wide no taller than at eight bits), which the monochrome step averages and stores again as it does smaller ones (with JBIG2 chosen, those past 2^28 pixels stay unless averaged), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side or 2^28 in all, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." [[feature]] id = "optimize.only-if-smaller" @@ -8641,6 +8641,18 @@ tools = [] evidence = [] notes = "" +[[feature]] +id = "print.color-handling" +title = "Print colour handling (app-managed colours, printer profile)" +area = "L" +tier = "P1" +milestone = "M11" +status = "partial" +commands = [] +tools = ["doc_print", "print_color_profiles"] +evidence = ["crates/print/src/colour.rs::black_point_compensation_keeps_dark_shadows_apart", "crates/print/src/colour.rs::rgb_profiles_convert_the_values_and_keep_alpha", "crates/print/src/colour.rs::installed_profiles_are_rgb_and_grey_ones", "crates/engine/src/tests.rs::app_managed_colour_converts_the_printed_pictures", "crates/automation/tests/it/automation.rs::doc_print_with_app_managed_colour"] +notes = "doc_print color_management app (#6269): the pages print as images (dpi, grayscale apply), converted from sRGB into an RGB or grey ICC printer profile (a file, or a name from print_color_profiles: the system's colour folders on Windows, macOS and Linux) with a rendering intent and black point compensation (default on), through craft-cms; the default leaves colour to the printer as before. The driver's own colour management should be off for such a job. Still missing against Acrobat: the Print dialog's Color Handling choice, CMYK printer profiles, vector pages converted without rendering them, and the document's output intent as the source of DeviceCMYK colours." + [[feature]] id = "print.advanced-print-separations" title = "Print separations and in-RIP separations" @@ -9146,8 +9158,8 @@ milestone = "M12" status = "partial" commands = ["a11y.autotag"] tools = ["accessibility_autotag_preview", "accessibility_autotag_apply"] -evidence = ["crates/automation/tests/it/automation.rs::autotag_preview_proposes_text_without_changing_the_pdf", "crates/automation/tests/it/automation.rs::autotag_apply_writes_reviewable_tags_and_survives_save_reopen", "crates/ui-egui/tests/it/accessibility.rs::automatic_text_tagging_opens_reviewable_tags"] -notes = "Auto-tag writes real marked content, a Document structure element, P tags and a parent tree for text-only PDFs, with undo and Tags-panel review. It refuses mixed content, forms, annotations and existing tags. The preview separately proposes headings. Images, tables, lists, links, fields and headings are not classified or tagged by Apply." +evidence = ["crates/automation/tests/it/automation.rs::autotag_preview_proposes_text_without_changing_the_pdf", "crates/automation/tests/it/automation.rs::autotag_apply_writes_reviewable_tags_and_survives_save_reopen", "crates/automation/tests/it/automation.rs::autotag_apply_tags_headings_by_size_across_pages", "crates/a11y/src/autotag_roles.rs::levels_rank_styles_largest_first_and_never_skip_a_level", "crates/ui-egui/tests/it/accessibility.rs::automatic_text_tagging_opens_reviewable_tags"] +notes = "Auto-tag writes real marked content, a Document structure element, H1-H6 and P tags and a parent tree for text-only PDFs, with undo and Tags-panel review. A text object is a heading when it is short and larger or bolder than the body text; levels rank those styles across the document and never skip a level. It refuses mixed content, forms, annotations and existing tags. The preview still proposes headings by its own single-line rule, so it can differ from Apply. Running headers and footers can still become headings (#7065); invisible text (render mode 3) is ignored. Images, tables, lists, links and fields are not classified or tagged by Apply." [[feature]] id = "a11y.autotag-fields" @@ -10050,8 +10062,8 @@ milestone = "M12" status = "partial" commands = [] tools = ["model_render"] -evidence = ["crates/automation/tests/it/model_render.rs::a_prc_model_is_drawn_over_the_background", "crates/automation/tests/it/model_render.rs::an_orbit_frames_the_model_from_a_named_view_and_its_moves", "crates/prc-render/src/nav/orbit_tests.rs::a_pressed_point_off_centre_keeps_its_place_as_the_view_turns", "crates/annot/src/model3d/tests.rs::a_3dref_leads_to_the_stream_it_names", "crates/prc-render/tests/model.rs::a_synthetic_prc_renders_its_nearer_triangle_over_the_farther"] -notes = "headless only: model_render draws a page's PRC model (through a /3DRef too) to a PNG, framed from a chosen direction, or by orbit navigation (named views about a chosen up axis, turn, zoom about a picture point, pan) from prc-render's GUI-free nav module, with headlight shading. Missing: the 3D view in the app (no activation, no on-screen navigation; nav is not wired to a viewer yet), walk and fly navigation, U3D (an error), the annotation's /3DV views, lighting schemes and /RM render modes, materials and textures, compressed tessellation, picking, cross sections, the model tree, and measurements" +evidence = ["crates/automation/tests/it/model_render.rs::a_prc_model_is_drawn_over_the_background", "crates/automation/tests/it/model_render.rs::an_orbit_frames_the_model_from_a_named_view_and_its_moves", "crates/prc-render/src/nav/orbit_tests.rs::a_pressed_point_off_centre_keeps_its_place_as_the_view_turns", "crates/annot/src/model3d/tests.rs::a_3dref_leads_to_the_stream_it_names", "crates/prc-render/tests/model.rs::a_synthetic_prc_renders_its_nearer_triangle_over_the_farther", "crates/prc-render/tests/model.rs::a_compressed_tessellation_is_rebuilt_and_rendered", "crates/automation/tests/it/model_render.rs::a_compressed_prc_model_is_rebuilt_and_drawn"] +notes = "headless only: model_render draws a page's PRC model (through a /3DRef too) to a PNG, framed from a chosen direction, or by orbit navigation (named views about a chosen up axis, turn, zoom about a picture point, pan) from prc-render's GUI-free nav module, with headlight shading. Missing: the 3D view in the app (no activation, no on-screen navigation; nav is not wired to a viewer yet), walk and fly navigation, U3D (an error), the annotation's /3DV views, lighting schemes and /RM render modes, materials and textures, compressed tessellations that store their normals (those whose normals are recalculated are rebuilt), picking, cross sections, the model tree, and measurements" [[feature]] id = "misc.3d-add" diff --git a/vendor/README.md b/vendor/README.md index 4cc90971c3..028e1490b0 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -55,7 +55,7 @@ Other resource categories retain their existing parent-chain lookup. | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| -| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (61) stroke geometry by ISO 32000-2 §8.5.3.2 and §8.4.3: a degenerate subpath (a single-point closed path, or points that all coincide) is stroked as a dot of the line width with round caps and not at all with butt or projecting square caps (upstream nudged a zero-length `l` so square caps drew a square, and `x y m h` lost its `h` because kurbo's `is_empty` counts only segments); a segment after `h` begins a new subpath with a `MoveTo` to the point `h` reached (§8.5.2.1, Table 58), so `h` then an `l` back to that point is a degenerate subpath of its own; a dash array that is all zero or has a negative or non-finite entry is kept as `[]`, a solid line (`[0 0]` became hundredth-of-a-point dashes, a faint half-covered line); a negative or non-finite line width (`w`, `/LW`) is kept as 0, so no device sees one below zero; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (65) a fill or clip whose path has no width or no height (but not both) marks the device pixels the line touches (ISO 32000-2 §10.7.4: "A zero-width or zero-height rectangle paints a line 1 pixel wide", and a clip is "the set of pixels that would be included by a fill operation"): whole pixels when the line stays within one device row or column, a band one device pixel wide when a rotated or skewed CTM carries it across rows and columns, built per subpath so the gap between two subpaths on one line stays unpainted (`degenerate_fill_area`). A zero-height clip such as `10 10 100 0 re W n` hid everything painted after it, and the fill fell back to a thinnest-line stroke that half-covered two rows on a pixel boundary. A point paints as before, and a path with area, however thin, is filled as the shape it is (the thinnest-line fallback took a side under 1/256 unit as zero, so a thin rectangle that the CTM scales up to whole pixels was drawn as a hairline across its bounding box); (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; so is an image whose JBIG2 soft mask or `/Mask` one of the decoder's limits refuses, or PdfCraft's bound on the pixels of an image (`image_size_ok`), where a mask the file damaged still leaves its image whole (#6722), and an image drawn on its own, a stencil mask included, that a limit refuses is still a failed image (`ImageXObject::masks_picture`): `decode_raster` leaves the image out when its mask reported `ContentTruncated`, decoded or kept by the cache (91), and a size or count the file damaged can read as past a limit and is taken for one, the safe side; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888); (76) a CID of an Arial the PDF doesn't embed (`CIDFontType2`, not of an Adobe character collection; `ArialMT`, `Arial-BoldMT`, `Arial,Bold` and the other two faces, not Arial Narrow, Black or Unicode MS) that no Unicode route draws names, through its `/CIDToGIDMap`, the Arial glyph of the standard Macintosh order (read-fonts' `post::DEFAULT_GLYPH_NAMES`, its first 258 glyphs), and a CFF substitute draws its glyph of that name (.notdef when it has none) instead of the CID taken as its own glyph index (#5889); (77) a standard 14 font drawn with its bundled stand-in (`StandardFontBlob::Cff`) reads each glyph's own advance from its charstring (`CffFontBlob::advance_width`), so a stand-in glyph wider or narrower than the standard metrics is fitted to them as the OpenType stand-ins already were (ISO 32000-2 §9.6.2.2): with no advance the outline was drawn as it is, and the stand-ins' € (WinAnsiEncoding code 128, Annex D Table D.2), 667 to 722 units wide against the metrics' 556 (Helvetica) and 500 (Times), overlapped the next glyph. Symbol and ZapfDingbats (Annex D.5, D.6) are fitted the same way, the check mark `4` a check box draws for its `/MK /CA` caption among them; (78) `TilingPattern::instance` numbers each setting of a tiling pattern (`scn`/`SCN`), its copies keeping the number, and `TilingPattern::is_coloured` reports `/PaintType 1`, so a renderer can draw a coloured cell once for every fill of one setting (hayro patch (30)); (79) text extraction from a symbolic embedded TrueType font with no `/Encoding` (and no `/ToUnicode`) reads a code through the `post` table name of the glyph its `cmap` selects (`EmbeddedKind::symbolic_text`), when that name has a Unicode value by the glyph-name rules of `glyph_name_text`, which are the Adobe Glyph List Specification's (ISO 32000-2 §9.10.2) and the editor's in pdfcraft-fonts (a `.suffix` is dropped, `_` separates the characters of a ligature, `uni` with one or more groups of exactly four uppercase hex digits and `u` with four to six are code points; lowercase digits, other lengths and surrogate or out-of-range values name nothing, so `uni20ac` reads as WinAnsi), instead of reading the code as WinAnsi: FrameMaker files distilled with Distiller 3 number their codes in the Macintosh glyph order, 29 below ASCII, so `Solid` was extracted, edited and exported to Word as `6ROLG` (#3962). Codes with no glyph, or whose glyph has no such name, still read as WinAnsi; glyph selection is unchanged; (80) an `/Indexed` colour space that breaks ISO 32000-2 §8.6.6.3 is drawn as most readers draw it: a lookup table shorter than m × (hival + 1) bytes keeps the entries it holds in full and an index past them is drawn black after conversion, in any base space (zero bytes would be white in DeviceCMYK, Separation and DeviceN, and blue in Lab); a `hival` past 255 is taken as 255, and a real `hival` is truncated toward zero. An image drawn so reports `InterpreterWarning::MalformedIndexedImage`, surfaced as `RenderWarning::MalformedIndexedImage`, and a fill or stroke colour in such a space reports `ColorFallback`. An image whose lookup is missing, not a string or a stream, or a stream that does not decode (or decodes to nothing although it holds data) is not drawn, with the same warning; a colour set in such a space keeps the upstream DeviceGray fallback. Upstream treated all of these as an unreadable space and fell back to DeviceGray, painting the indices near-black, even ones whose entries were present. A `hival` below zero or not a number has no reading: the space falls back as upstream, and an image in it reports the warning. A lookup string given by an indirect reference is read (upstream consumed the reference trying it as a stream and found no table). The warning is taken from the space as built, so no table is read twice: inline images naming the space through a resource read its table once however many there are (`InterpreterCache::indexed_lookup_reads`); (82) a soft mask (or `/Mask` image) larger than its image and more than twice as large on a side as the device pixels its image is drawn at is averaged down to that size, rounded up to a power of two, in `ImageXObject::decoded_soft_mask`, and cached there (`DecodedImageKey::shrunk_to`, with the image's raster keyed by the same rounded size); every size is made from one cached intermediate mask at most 4096 on a side, so the full mask is decoded once however many sizes it is drawn at: it bypassed the decoded-image cache and was decoded, copied and staged at full size each time it was drawn. A 2 × 2 image with a 34,862 × 4,332 mask drawn 40 times on one slide (pdf.js `issue16263.pdf`) took about 3.6 s a draw and minutes a page; it now decodes the mask once and draws the page in about 2.5 s. A mask no larger than twice its drawn size keeps its resolution, which the renderer filters as before; `InterpreterCache::decoded_mask_decodes` counts the masks among the decodes; (83) an embedded TrueType program whose `head.indexToLocFormat` holds neither 0 nor 1 is read with the `loca` format its table's length gives (`with_loca_format_fixed`): the glyph reader took anything but 1 as short offsets, so a font marked 256 had its long offsets misread and drew boxes (pdf.js `issue2537r.pdf`); (84) a Type 0 font of an Adobe character collection (Japan1, GB1, CNS1, Korea1) reads a code its `/ToUnicode` has nothing for, or every code when it has none, through the collection's UCS2 CMap, code to CID to Unicode, embedded or not (ISO 32000-2 §9.10.2; `Type0Font::ucs2`, parsed the first time a code needs it): only fonts that are not embedded took that route, so Japanese and Chinese pages whose fonts are embedded without a `/ToUnicode`, or with one that maps nothing, had no text to select, find or copy; (85) a Type 0 font's `/ToUnicode` written as a CID CMap (`begincidchar` / `begincidrange`, destinations as numbers, as Word writes it) gives each number as the code's Unicode value for text, as glyph lookup (`map_via_unicode`) already read it and as pdftotext and PDFium extract it; ISO 32000-2 §9.10.3 has `beginbfchar` / `beginbfrange` there, and such text extracted nothing (pdf.js `issue9367`); (86) form fields' widgets are drawn after the page's other annotations, each group in `/Annots` order (`interpret_page`), as MuPDF and PDFium draw them; ISO 32000-2 doesn't order the two: a check box listed before a filled square comment that covers it was hidden under the comment (pdf.js `issue13003.pdf`); (87) a Type 3 font keeps each glyph procedure it decoded (`Type3::decoded_proc`), so a glyph shown again isn't inflated again: a font used as a fill pattern drew one glyph about 650,000 times on a page and decoded its procedure for each; a procedure over 64 KiB isn't kept, and the fonts of one `InterpreterCache` keep at most 32 MiB together (`MAX_CACHED_TYPE3_PROCS`), each kept procedure counting its bytes, its name and 64 bytes besides, as fonts live as long as the cache; a procedure holds its bytes within the page's content budget while the glyph is painted, kept or not (`context::hold_content`), so a page skips the same glyphs once the budget is spent, and one decoded with no more than 64 KiB of budget left, which the budget may have cut short, isn't kept; `InterpreterCache::type3_proc_decodes` counts the procedures decoded; (88) the rule of (76) also reads Times New Roman (`TimesNewRomanPSMT`, `TimesNewRomanPS-BoldMT`, `TimesNewRoman,Bold` and the other faces, not Times or Times New Roman MT Std), whose first 258 glyphs follow that order too: in a public form whose embedded Times New Roman subsets keep their glyph indexes, 941 of 942 `/ToUnicode` entries below 258 name that order's character, and its non-embedded `TimesNewRoman,Bold` drew "Salarié(e) du régime général" as "Salari²*g+f w't²i lo g'ì²p²tcn" (#6449); (89) a form XObject already being drawn is not drawn inside itself (`Context::enter_form`, reported as `ContentTruncated`, §8.10.1): a name in its content that reached it again, through its own resources or the page resources a form with its own still falls back on, drew it until the page's nested-paint budget was spent (PDFBOX-5782, 38 s); the forms being drawn are kept per page and thread, so the tiling patterns, Type 3 glyphs, soft masks and groups painted inside a form, which are interpreted in contexts of their own, see it too; (90) a Type 3 font without `/Widths` (or `/FirstChar`/`/LastChar`) advances each glyph by the width its glyph procedure sets with `d0` or `d1` (`Type3::proc_width`, ISO 32000-2 9.6.4) instead of 0, read once per code; a font with `/Widths` is unchanged, a code outside `/FirstChar`..`/LastChar` keeping width 0 (Table 110) (PDFBOX-2943: a misspelt `/Widths` key drew every glyph on one spot), and selecting such a font reports `InterpreterWarning::Type3WidthsMissing` (`Font::type3_widths_missing`, checked in `Context::resolve_font`); (91) the decoded-image cache keeps a failed decode for its key, as the warnings it reported, so a picture or stencil mask that doesn't decode (damaged data, a JPX or JBIG2 stream its decoder refuses) isn't decoded again at every draw, tile and render that shares the cache, and each draw still reports what the decode did (`DecodedImageCache::get_or_insert_with`, `ImageXObject::reporting_to`); a failure counts 128 bytes towards the cache's budget, and one the page's JBIG2 allowance may have caused (73) isn't kept (the hayro-syntax comment on `charge_page_reading` says so), nor one a JBIG2 mask's decoding work refused, while a JBIG2 mask refused by a limit its stream reaches by itself is kept, with the image it takes out (`note_stream_limited`), so a page drawing that image a hundred times decodes it once; (92) the text of a glyph whose name is not in the glyph list reads `uniXXXX` and `uXXXX` to `uXXXXXX` by the Adobe Glyph List Specification (`agl_unicode_from_name`: `uni` and exactly four uppercase hex digits, `u` and four to six, no surrogate, nothing past U+10FFFF), so `uni65`, `u65`, `uni20ac` and `uni+065` read as nothing instead of `e` and `€`; upstream's lenient `unicode_from_name` stays for finding a glyph in a font (TrueType `cmap` lookup, accented standard-font letters), so drawing is unchanged; (93) a CID-keyed CFF font keeps the font dicts it can read: a font dict without a Private DICT (one a subsetter emptied) leaves only the glyphs FDSelect gives it missing, where one such dict dropped the whole font and its text drew with a fallback (ISO 32000-2 §9.9 Table 124, Adobe TN #5176); (94) a font dictionary whose `/Subtype` is missing or the empty name (`/Subtype /`) is read as Type 1, so its `/BaseFont` picks the face (ISO 32000-2 Table 109 requires `/Subtype`; other readers fall back this way): a `/Courier-Bold` was drawn in the Helvetica fallback (PDFBOX-1988). Any other unknown subtype is still refused; (98) a non-embedded font named `SymbolMT`, or `Symbol`/`SymbolMT` with a `,` or `-` style suffix (`Symbol,Bold`, `Symbol-Bold`; `SYMBOL_STYLES`, so `Symbol-Helvetica` stays another font), is the standard Symbol font and draws through its built-in encoding, so a TrueType `SymbolMT` declaring `/WinAnsiEncoding` draws `mu` for code 0x6D, not `m` (ISO 32000-2 §9.6.2.2); (99) the mesh sampler walks each row of a triangle only across the columns where that row's sample points can lie in it, not its whole box (a row with none still costs a visit): thin triangles of the fixed 19 × 19 patch grid lying across the pixel grid (a narrow curved band of tensor patches) have boxes many times their area, and a mesh painted through a path the size of its own box spent the visit budget before its last patches and left them unpainted; (100) a Type 1 or Type 3 font glyph name that only numbers its glyph (`C39`, `MT72`, `G4E`: one to three letters, then decimal or hexadecimal digits) and names the very code it is shown with reads as that code's Latin-1 character (controls excepted; in a Type 3 font, which already reads codes 32-126 by the code, above 126), in a font where every such name names its own code (`numbered_names_name_their_codes`, worked out once per font); a font with any numbered name at another code (`/C1` at 233, review on #3239; `/Differences [1 /C70 …]`) or at a C0 control code (a subset counting its glyphs from code 1, `/Differences [1 /C1 /C2 …]`, where `/C65` is the 65th glyph used) keeps reading them as nothing (#3170); (101) a symbolic TrueType font with no encoding reads a code as Mac OS Roman in text extraction when its Macintosh Roman (1, 0) `cmap` subtable selects the glyph the code draws (ISO 32000-2 §9.6.5.4) and that glyph sits where the standard Macintosh glyph order (`post` version 1.0) puts the code's Mac OS Roman character, for a font whose `post` table names no glyphs, that names no base encoding (an unknown name included) and that isn't Symbol or ZapfDingbats (§9.6.6.1), after `/Differences` and before the WinAnsi last resort; Windows subsets in the same order keep their WinAnsi reading, and glyph selection is unchanged; (102) `OutlineGlyph::vertical_glyph_width`: a vertical-writing Type 0 glyph's width w0 from `/W` or `/DW`, its extent across the column (ISO 32000-2 §9.7.4.3), which `advance_width` (the horizontal advance, 0 in vertical writing) does not give; (81) a reply of type R (a markup annotation with an `/IRT` and an `/RT` other than `/Group`; R is the default, so a missing, unknown or malformed `/RT` reads as R; a widget, link or pop-up with `/IRT` still draws) is not drawn, whatever its appearance or flags: ISO 32000-2 Table 172 says such replies "shall not" be displayed individually but in their comment's thread (files from editors that give every reply a note-icon appearance showed a stack of icons, #5631); a `/RT /Group` member still draws; (103) `InterpreterSettings::view_auto_state` (on by default): the default configuration's auto state for the View event (`/AS` usage applications with `/Event /View` and `/View` in `/Category`) sets each managed group from its `/Usage /View /ViewState` before `ocg_overrides`, ON only if every recommendation is ON (ISO 32000-2 §8.11.4.4); `/Zoom`, `/User` and `/Language` give no recommendation; at most 256 applications, 4096 `/OCGs` entries and 16 categories each are read; off for print renders; (104) `InterpreterSettings::annotation_overrides`: the viewer shows or hides an annotation by object in place of its Hidden and NoView flags (scripted animation frames), and an `/Annots` entry that isn't a dictionary is skipped instead of ending the list; (105) `InterpreterSettings::appearance_overrides`: the viewer draws an annotation with another form XObject as its normal appearance, in its own `/Rect` (nothing, when the override names an object that isn't a form: a cleared icon) (`buttonSetIcon` from page actions and timers) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (61) `stroke_geometry::{degenerate_subpaths_draw_a_round_dot_or_nothing, degenerate_subpaths_leave_the_rest_of_the_path_alone, a_segment_after_h_begins_a_new_subpath, all_zero_dash_arrays_draw_a_solid_line, invalid_dash_arrays_reach_a_device_as_solid, valid_dash_patterns_still_dash, negative_line_widths_reach_a_device_as_zero, negative_line_widths_draw_the_thinnest_line_in_the_stroke_colour}` (pdfcraft-render integration tests); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (65) `it::zero_area_fill_clip` (`crates/render/tests/it/zero_area_fill_clip.rs`); (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn`, `a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `a_jbig2_image_drawn_on_its_own_that_a_limit_refuses_is_a_failed_image`, `a_limit_refusal_says_nothing_of_the_next_image`, `a_jpx_picture_whose_jbig2_mask_a_limit_refuses_stays_out_at_every_size` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render); (76) `raster::tests::non_embedded_arial::{an_arial_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (77) `euro_width::the_euro_fits_its_standard_advance`, `euro_width::two_euros_stay_apart`, `euro_width::a_slanted_euro_fits_its_standard_advance`, `symbol_dingbats_width::{symbol_glyphs_keep_their_standard_advances, zapf_dingbats_glyphs_keep_their_standard_advances}` (pdfcraft-render `tests/it`), `dingbat_check_box::a_check_box_caption_check_is_drawn_at_its_standard_advance` (pdfcraft-engine `tests/it`); (78) `raster::tests::{fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}` (pdfcraft-render); (79) `raster::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names`, `pdf::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names` (pdfcraft-fonts, the editor's reading by the same rules) ; (80) `crates/render/tests/it/indexed_lookup.rs` (`cargo test -p pdfcraft-render --test it indexed_lookup`), `raster::tests::indexed_lookup_cache`; (82) `raster::tests::a_soft_mask_far_larger_than_where_it_is_drawn_is_shrunk_once`, `raster::tests::an_averaged_down_soft_mask_follows_its_samples_and_its_drawn_size`, `raster::tests::decoded_images_serve_every_size_they_are_drawn_at` (pdfcraft-render); (83) `raster::tests::truetype_loca_format::a_truetype_font_with_an_out_of_range_loca_format_draws_its_glyphs` (pdfcraft-render); (84) `text::tests::an_embedded_adobe_collection_font_reads_through_its_ucs2_cmap`, `every_adobe_collection_reads_through_its_own_ucs2_cmap` (GB1, CNS1, Korea1, Identity-V, `/H` over CFF) and `a_font_outside_the_adobe_collections_gets_no_text_from_a_ucs2_cmap` (pdfcraft-render); (85) `text::tests::{a_tounicode_written_as_cid_ranges_gives_text, a_tounicode_bf_entry_wins_over_a_cid_entry_for_the_same_code}` (pdfcraft-render); (86) `widgets_above_annotations::a_widget_is_drawn_above_the_comments_on_its_page` (pdfcraft-render `tests/it`); (87) `raster::tests::type3_proc_cache::{a_type3_glyph_shown_again_decodes_its_procedure_once, a_procedure_over_64_kib_is_decoded_for_each_copy, the_procedures_one_cache_keeps_stop_at_32_mib, fonts_with_the_same_glyph_name_keep_their_own_procedures}` (pdfcraft-render); (88) `raster::tests::non_embedded_arial::{a_times_new_roman_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (89) `raster::tests::a_form_that_draws_itself_is_drawn_once`, `raster::tests::a_form_that_reaches_itself_through_a_pattern_or_a_glyph_is_drawn_once` (pdfcraft-render); (90) `raster::tests::type3_proc_widths::{a_type3_font_without_widths_advances_by_its_glyph_procedures, a_code_outside_first_char_to_last_char_still_has_width_zero, widths_the_font_gives_are_kept, a_code_without_a_usable_procedure_width_still_advances_nothing, a_type3_font_without_widths_is_reported}` (pdfcraft-render); (91) `raster::tests::failed_image_decodes::{a_picture_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_that_does_not_decode_is_decoded_once_at_each_size, a_stencil_mask_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_whose_soft_mask_does_not_decode_draws_unmasked_decoded_once, a_picture_that_decodes_is_still_decoded_once_and_drawn_every_time, a_picture_left_out_for_the_pages_allowance_draws_on_the_next_page, a_picture_whose_jbig2_mask_a_limit_refuses_is_decoded_once_however_often_it_is_drawn, a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw}` (pdfcraft-render); (92) `text::tests::agl_unicode_from_name_rule_reads_text` (pdfcraft-render); (93) `cid_cff_unusable_font_dict::{glyphs_of_a_usable_font_dict_draw_beside_an_unusable_one, a_program_with_no_usable_font_dict_draws_no_square}` (pdfcraft-render integration tests); (94) `raster::tests::a_font_without_a_subtype_is_drawn_by_its_base_font` (pdfcraft-render); (98) `symbol_mt_builtin_encoding::{symbol_mt_draws_symbols_through_the_built_in_encoding, a_name_that_only_starts_with_symbol_is_not_symbol, symbol_mt_applies_its_differences, an_embedded_symbol_mt_draws_its_own_program}` (pdfcraft-render integration tests); (99) `raster::tests::thin_patches_of_a_mesh_in_a_tight_path_are_all_painted` (pdfcraft-render); (100) `text::tests::numbered_glyph_names_read_as_their_own_code`, `text::tests::numbered_type3_glyph_names_read_as_their_own_code`, `text::tests::glyph_names_outside_the_glyph_list_still_read_as_text` (pdfcraft-render); (101) `raster::tests::symbolic_truetype_mac_roman_text::{a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman, winansi_codes_of_a_font_in_the_macintosh_order_read_as_before, a_symbolic_truetype_font_without_a_1_0_subtable_reads_as_before, a_1_0_subtable_selecting_other_glyphs_than_the_drawn_ones_is_not_read, a_font_that_names_its_glyphs_is_not_read_by_their_order, a_symbol_or_zapfdingbats_font_in_the_macintosh_order_gets_no_mac_roman_reading, differences_naming_notdef_clear_the_mac_roman_reading_for_its_code, a_base_encoding_name_this_reader_does_not_know_keeps_the_mac_roman_reading_off}` (pdfcraft-render), `pdf::tests::a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman` (pdfcraft-fonts); (102) `text::tests::tracked_vertical_latin_stays_one_line` (pdfcraft-render); (81) `raster::tests::replies_are_not_drawn_but_their_comment_and_group_members_are`, `raster::tests::an_unknown_or_malformed_rt_is_a_reply_and_a_widget_with_irt_still_draws` (hayro-interpret); (103) `raster::tests::the_view_auto_state_sets_a_groups_state_on_screen`, `raster::tests::layer_toggles_and_print_renders_are_not_the_view_auto_state` and `raster::tests::malformed_view_auto_state_is_ignored` (pdfcraft-render); (104, 105) `raster::tests::annotation_and_appearance_overrides_change_only_the_view` (pdfcraft-render) | +| hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (61) stroke geometry by ISO 32000-2 §8.5.3.2 and §8.4.3: a degenerate subpath (a single-point closed path, or points that all coincide) is stroked as a dot of the line width with round caps and not at all with butt or projecting square caps (upstream nudged a zero-length `l` so square caps drew a square, and `x y m h` lost its `h` because kurbo's `is_empty` counts only segments); a segment after `h` begins a new subpath with a `MoveTo` to the point `h` reached (§8.5.2.1, Table 58), so `h` then an `l` back to that point is a degenerate subpath of its own; a dash array that is all zero or has a negative or non-finite entry is kept as `[]`, a solid line (`[0 0]` became hundredth-of-a-point dashes, a faint half-covered line); a negative or non-finite line width (`w`, `/LW`) is kept as 0, so no device sees one below zero; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (65) a fill or clip whose path has no width or no height (but not both) marks the device pixels the line touches (ISO 32000-2 §10.7.4: "A zero-width or zero-height rectangle paints a line 1 pixel wide", and a clip is "the set of pixels that would be included by a fill operation"): whole pixels when the line stays within one device row or column, a band one device pixel wide when a rotated or skewed CTM carries it across rows and columns, built per subpath so the gap between two subpaths on one line stays unpainted (`degenerate_fill_area`). A zero-height clip such as `10 10 100 0 re W n` hid everything painted after it, and the fill fell back to a thinnest-line stroke that half-covered two rows on a pixel boundary. A point paints as before, and a path with area, however thin, is filled as the shape it is (the thinnest-line fallback took a side under 1/256 unit as zero, so a thin rectangle that the CTM scales up to whole pixels was drawn as a hairline across its bounding box); (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; so is an image whose JBIG2 soft mask or `/Mask` one of the decoder's limits refuses, or PdfCraft's bound on the pixels of an image (`image_size_ok`), where a mask the file damaged still leaves its image whole (#6722), and an image drawn on its own, a stencil mask included, that a limit refuses is still a failed image (`ImageXObject::masks_picture`): `decode_raster` leaves the image out when its mask reported `ContentTruncated`, decoded or kept by the cache (91), and a size or count the file damaged can read as past a limit and is taken for one, the safe side; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888); (76) a CID of an Arial the PDF doesn't embed (`CIDFontType2`, not of an Adobe character collection; `ArialMT`, `Arial-BoldMT`, `Arial,Bold` and the other two faces, not Arial Narrow, Black or Unicode MS) that no Unicode route draws names, through its `/CIDToGIDMap`, the Arial glyph of the standard Macintosh order (read-fonts' `post::DEFAULT_GLYPH_NAMES`, its first 258 glyphs), and a CFF substitute draws its glyph of that name (.notdef when it has none) instead of the CID taken as its own glyph index (#5889); (77) a standard 14 font drawn with its bundled stand-in (`StandardFontBlob::Cff`) reads each glyph's own advance from its charstring (`CffFontBlob::advance_width`), so a stand-in glyph wider or narrower than the standard metrics is fitted to them as the OpenType stand-ins already were (ISO 32000-2 §9.6.2.2): with no advance the outline was drawn as it is, and the stand-ins' € (WinAnsiEncoding code 128, Annex D Table D.2), 667 to 722 units wide against the metrics' 556 (Helvetica) and 500 (Times), overlapped the next glyph. Symbol and ZapfDingbats (Annex D.5, D.6) are fitted the same way, the check mark `4` a check box draws for its `/MK /CA` caption among them; (78) `TilingPattern::instance` numbers each setting of a tiling pattern (`scn`/`SCN`), its copies keeping the number, and `TilingPattern::is_coloured` reports `/PaintType 1`, so a renderer can draw a coloured cell once for every fill of one setting (hayro patch (30)); (79) text extraction from a symbolic embedded TrueType font with no `/Encoding` (and no `/ToUnicode`) reads a code through the `post` table name of the glyph its `cmap` selects (`EmbeddedKind::symbolic_text`), when that name has a Unicode value by the glyph-name rules of `glyph_name_text`, which are the Adobe Glyph List Specification's (ISO 32000-2 §9.10.2) and the editor's in pdfcraft-fonts (a `.suffix` is dropped, `_` separates the characters of a ligature, `uni` with one or more groups of exactly four uppercase hex digits and `u` with four to six are code points; lowercase digits, other lengths and surrogate or out-of-range values name nothing, so `uni20ac` reads as WinAnsi), instead of reading the code as WinAnsi: FrameMaker files distilled with Distiller 3 number their codes in the Macintosh glyph order, 29 below ASCII, so `Solid` was extracted, edited and exported to Word as `6ROLG` (#3962). Codes with no glyph, or whose glyph has no such name, still read as WinAnsi; glyph selection is unchanged; (80) an `/Indexed` colour space that breaks ISO 32000-2 §8.6.6.3 is drawn as most readers draw it: a lookup table shorter than m × (hival + 1) bytes keeps the entries it holds in full and an index past them is drawn black after conversion, in any base space (zero bytes would be white in DeviceCMYK, Separation and DeviceN, and blue in Lab); a `hival` past 255 is taken as 255, and a real `hival` is truncated toward zero. An image drawn so reports `InterpreterWarning::MalformedIndexedImage`, surfaced as `RenderWarning::MalformedIndexedImage`, and a fill or stroke colour in such a space reports `ColorFallback`. An image whose lookup is missing, not a string or a stream, or a stream that does not decode (or decodes to nothing although it holds data) is not drawn, with the same warning; a colour set in such a space keeps the upstream DeviceGray fallback. Upstream treated all of these as an unreadable space and fell back to DeviceGray, painting the indices near-black, even ones whose entries were present. A `hival` below zero or not a number has no reading: the space falls back as upstream, and an image in it reports the warning. A lookup string given by an indirect reference is read (upstream consumed the reference trying it as a stream and found no table). The warning is taken from the space as built, so no table is read twice: inline images naming the space through a resource read its table once however many there are (`InterpreterCache::indexed_lookup_reads`); (82) a soft mask (or `/Mask` image) larger than its image and more than twice as large on a side as the device pixels its image is drawn at is averaged down to that size, rounded up to a power of two, in `ImageXObject::decoded_soft_mask`, and cached there (`DecodedImageKey::shrunk_to`, with the image's raster keyed by the same rounded size); every size is made from one cached intermediate mask at most 4096 on a side, so the full mask is decoded once however many sizes it is drawn at: it bypassed the decoded-image cache and was decoded, copied and staged at full size each time it was drawn. A 2 × 2 image with a 34,862 × 4,332 mask drawn 40 times on one slide (pdf.js `issue16263.pdf`) took about 3.6 s a draw and minutes a page; it now decodes the mask once and draws the page in about 2.5 s. A mask no larger than twice its drawn size keeps its resolution, which the renderer filters as before; `InterpreterCache::decoded_mask_decodes` counts the masks among the decodes; (83) an embedded TrueType program whose `head.indexToLocFormat` holds neither 0 nor 1 is read with the `loca` format its table's length gives (`with_loca_format_fixed`): the glyph reader took anything but 1 as short offsets, so a font marked 256 had its long offsets misread and drew boxes (pdf.js `issue2537r.pdf`); (84) a Type 0 font of an Adobe character collection (Japan1, GB1, CNS1, Korea1) reads a code its `/ToUnicode` has nothing for, or every code when it has none, through the collection's UCS2 CMap, code to CID to Unicode, embedded or not (ISO 32000-2 §9.10.2; `Type0Font::ucs2`, parsed the first time a code needs it): only fonts that are not embedded took that route, so Japanese and Chinese pages whose fonts are embedded without a `/ToUnicode`, or with one that maps nothing, had no text to select, find or copy; (85) a Type 0 font's `/ToUnicode` written as a CID CMap (`begincidchar` / `begincidrange`, destinations as numbers, as Word writes it) gives each number as the code's Unicode value for text, as glyph lookup (`map_via_unicode`) already read it and as pdftotext and PDFium extract it; ISO 32000-2 §9.10.3 has `beginbfchar` / `beginbfrange` there, and such text extracted nothing (pdf.js `issue9367`); (86) form fields' widgets are drawn after the page's other annotations, each group in `/Annots` order (`interpret_page`), as MuPDF and PDFium draw them; ISO 32000-2 doesn't order the two: a check box listed before a filled square comment that covers it was hidden under the comment (pdf.js `issue13003.pdf`); (87) a Type 3 font keeps each glyph procedure it decoded (`Type3::decoded_proc`), so a glyph shown again isn't inflated again: a font used as a fill pattern drew one glyph about 650,000 times on a page and decoded its procedure for each; a procedure over 64 KiB isn't kept, and the fonts of one `InterpreterCache` keep at most 32 MiB together (`MAX_CACHED_TYPE3_PROCS`), each kept procedure counting its bytes, its name and 64 bytes besides, as fonts live as long as the cache; a procedure holds its bytes within the page's content budget while the glyph is painted, kept or not (`context::hold_content`), so a page skips the same glyphs once the budget is spent, and one decoded with no more than 64 KiB of budget left, which the budget may have cut short, isn't kept; `InterpreterCache::type3_proc_decodes` counts the procedures decoded; (88) the rule of (76) also reads Times New Roman (`TimesNewRomanPSMT`, `TimesNewRomanPS-BoldMT`, `TimesNewRoman,Bold` and the other faces, not Times or Times New Roman MT Std), whose first 258 glyphs follow that order too: in a public form whose embedded Times New Roman subsets keep their glyph indexes, 941 of 942 `/ToUnicode` entries below 258 name that order's character, and its non-embedded `TimesNewRoman,Bold` drew "Salarié(e) du régime général" as "Salari²*g+f w't²i lo g'ì²p²tcn" (#6449); (89) a form XObject already being drawn is not drawn inside itself (`Context::enter_form`, reported as `ContentTruncated`, §8.10.1): a name in its content that reached it again, through its own resources or the page resources a form with its own still falls back on, drew it until the page's nested-paint budget was spent (PDFBOX-5782, 38 s); the forms being drawn are kept per page and thread, so the tiling patterns, Type 3 glyphs, soft masks and groups painted inside a form, which are interpreted in contexts of their own, see it too; (90) a Type 3 font without `/Widths` (or `/FirstChar`/`/LastChar`) advances each glyph by the width its glyph procedure sets with `d0` or `d1` (`Type3::proc_width`, ISO 32000-2 9.6.4) instead of 0, read once per code; a font with `/Widths` is unchanged, a code outside `/FirstChar`..`/LastChar` keeping width 0 (Table 110) (PDFBOX-2943: a misspelt `/Widths` key drew every glyph on one spot), and selecting such a font reports `InterpreterWarning::Type3WidthsMissing` (`Font::type3_widths_missing`, checked in `Context::resolve_font`); (91) the decoded-image cache keeps a failed decode for its key, as the warnings it reported, so a picture or stencil mask that doesn't decode (damaged data, a JPX or JBIG2 stream its decoder refuses) isn't decoded again at every draw, tile and render that shares the cache, and each draw still reports what the decode did (`DecodedImageCache::get_or_insert_with`, `ImageXObject::reporting_to`); a failure counts 128 bytes towards the cache's budget, and one the page's JBIG2 allowance may have caused (73) isn't kept (the hayro-syntax comment on `charge_page_reading` says so), nor one a JBIG2 mask's decoding work refused, while a JBIG2 mask refused by a limit its stream reaches by itself is kept, with the image it takes out (`note_stream_limited`), so a page drawing that image a hundred times decodes it once; (92) the text of a glyph whose name is not in the glyph list reads `uniXXXX` and `uXXXX` to `uXXXXXX` by the Adobe Glyph List Specification (`agl_unicode_from_name`: `uni` and exactly four uppercase hex digits, `u` and four to six, no surrogate, nothing past U+10FFFF), so `uni65`, `u65`, `uni20ac` and `uni+065` read as nothing instead of `e` and `€`; upstream's lenient `unicode_from_name` stays for finding a glyph in a font (TrueType `cmap` lookup, accented standard-font letters), so drawing is unchanged; (93) a CID-keyed CFF font keeps the font dicts it can read: a font dict without a Private DICT (one a subsetter emptied) leaves only the glyphs FDSelect gives it missing, where one such dict dropped the whole font and its text drew with a fallback (ISO 32000-2 §9.9 Table 124, Adobe TN #5176); (94) a font dictionary whose `/Subtype` is missing or the empty name (`/Subtype /`) is read as Type 1, so its `/BaseFont` picks the face (ISO 32000-2 Table 109 requires `/Subtype`; other readers fall back this way): a `/Courier-Bold` was drawn in the Helvetica fallback (PDFBOX-1988). Any other unknown subtype is still refused; (98) a non-embedded font named `SymbolMT`, or `Symbol`/`SymbolMT` with a `,` or `-` style suffix (`Symbol,Bold`, `Symbol-Bold`; `SYMBOL_STYLES`, so `Symbol-Helvetica` stays another font), is the standard Symbol font and draws through its built-in encoding, so a TrueType `SymbolMT` declaring `/WinAnsiEncoding` draws `mu` for code 0x6D, not `m` (ISO 32000-2 §9.6.2.2); (99) the mesh sampler walks each row of a triangle only across the columns where that row's sample points can lie in it, not its whole box (a row with none still costs a visit): thin triangles of the fixed 19 × 19 patch grid lying across the pixel grid (a narrow curved band of tensor patches) have boxes many times their area, and a mesh painted through a path the size of its own box spent the visit budget before its last patches and left them unpainted; (100) a Type 1 or Type 3 font glyph name that only numbers its glyph (`C39`, `MT72`, `G4E`: one to three letters, then decimal or hexadecimal digits) and names the very code it is shown with reads as that code's Latin-1 character (controls excepted; in a Type 3 font, which already reads codes 32-126 by the code, above 126), in a font where every such name names its own code (`numbered_names_name_their_codes`, worked out once per font); a font with any numbered name at another code (`/C1` at 233, review on #3239; `/Differences [1 /C70 …]`) or at a C0 control code (a subset counting its glyphs from code 1, `/Differences [1 /C1 /C2 …]`, where `/C65` is the 65th glyph used) keeps reading them as nothing (#3170); (101) a symbolic TrueType font with no encoding reads a code as Mac OS Roman in text extraction when its Macintosh Roman (1, 0) `cmap` subtable selects the glyph the code draws (ISO 32000-2 §9.6.5.4) and that glyph sits where the standard Macintosh glyph order (`post` version 1.0) puts the code's Mac OS Roman character, for a font whose `post` table names no glyphs, that names no base encoding (an unknown name included) and that isn't Symbol or ZapfDingbats (§9.6.6.1), after `/Differences` and before the WinAnsi last resort; Windows subsets in the same order keep their WinAnsi reading, and glyph selection is unchanged; (102) `OutlineGlyph::vertical_glyph_width`: a vertical-writing Type 0 glyph's width w0 from `/W` or `/DW`, its extent across the column (ISO 32000-2 §9.7.4.3), which `advance_width` (the horizontal advance, 0 in vertical writing) does not give; (81) a reply of type R (a markup annotation with an `/IRT` and an `/RT` other than `/Group`; R is the default, so a missing, unknown or malformed `/RT` reads as R; a widget, link or pop-up with `/IRT` still draws) is not drawn, whatever its appearance or flags: ISO 32000-2 Table 172 says such replies "shall not" be displayed individually but in their comment's thread (files from editors that give every reply a note-icon appearance showed a stack of icons, #5631); a `/RT /Group` member still draws; (103) `InterpreterSettings::view_auto_state` (on by default): the default configuration's auto state for the View event (`/AS` usage applications with `/Event /View` and `/View` in `/Category`) sets each managed group from its `/Usage /View /ViewState` before `ocg_overrides`, ON only if every recommendation is ON (ISO 32000-2 §8.11.4.4); `/Zoom`, `/User` and `/Language` give no recommendation; at most 256 applications, 4096 `/OCGs` entries and 16 categories each are read; off for print renders; (104) an additive colour (`DeviceRGB`, `CalRGB`, `Lab`, three-component ICCBased) painted in an isolated `DeviceCMYK` group has ink tints, the ink that draws back to it through the CMYK profile (`cmyk_inverse::srgb_to_cmyk`: black `0.4 · k₀² + 0.6 · k₀⁸` with `k₀ = 1 − max(r, g, b)`, then cyan, magenta and yellow by at most 12 Newton steps; the last colour is cached per thread), so the group blends in CMYK instead of falling back to RGB; (105) `InterpreterSettings::annotation_overrides`: the viewer shows or hides an annotation by object in place of its Hidden and NoView flags (scripted animation frames), and an `/Annots` entry that isn't a dictionary is skipped instead of ending the list; (106) `InterpreterSettings::appearance_overrides`: the viewer draws an annotation with another form XObject as its normal appearance, in its own `/Rect` (nothing, when the override names an object that isn't a form: a cleared icon) (`buttonSetIcon` from page actions and timers) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (61) `stroke_geometry::{degenerate_subpaths_draw_a_round_dot_or_nothing, degenerate_subpaths_leave_the_rest_of_the_path_alone, a_segment_after_h_begins_a_new_subpath, all_zero_dash_arrays_draw_a_solid_line, invalid_dash_arrays_reach_a_device_as_solid, valid_dash_patterns_still_dash, negative_line_widths_reach_a_device_as_zero, negative_line_widths_draw_the_thinnest_line_in_the_stroke_colour}` (pdfcraft-render integration tests); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (65) `it::zero_area_fill_clip` (`crates/render/tests/it/zero_area_fill_clip.rs`); (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn`, `a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `a_jbig2_image_drawn_on_its_own_that_a_limit_refuses_is_a_failed_image`, `a_limit_refusal_says_nothing_of_the_next_image`, `a_jpx_picture_whose_jbig2_mask_a_limit_refuses_stays_out_at_every_size` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render); (76) `raster::tests::non_embedded_arial::{an_arial_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (77) `euro_width::the_euro_fits_its_standard_advance`, `euro_width::two_euros_stay_apart`, `euro_width::a_slanted_euro_fits_its_standard_advance`, `symbol_dingbats_width::{symbol_glyphs_keep_their_standard_advances, zapf_dingbats_glyphs_keep_their_standard_advances}` (pdfcraft-render `tests/it`), `dingbat_check_box::a_check_box_caption_check_is_drawn_at_its_standard_advance` (pdfcraft-engine `tests/it`); (78) `raster::tests::{fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}` (pdfcraft-render); (79) `raster::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names`, `pdf::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names` (pdfcraft-fonts, the editor's reading by the same rules) ; (80) `crates/render/tests/it/indexed_lookup.rs` (`cargo test -p pdfcraft-render --test it indexed_lookup`), `raster::tests::indexed_lookup_cache`; (82) `raster::tests::a_soft_mask_far_larger_than_where_it_is_drawn_is_shrunk_once`, `raster::tests::an_averaged_down_soft_mask_follows_its_samples_and_its_drawn_size`, `raster::tests::decoded_images_serve_every_size_they_are_drawn_at` (pdfcraft-render); (83) `raster::tests::truetype_loca_format::a_truetype_font_with_an_out_of_range_loca_format_draws_its_glyphs` (pdfcraft-render); (84) `text::tests::an_embedded_adobe_collection_font_reads_through_its_ucs2_cmap`, `every_adobe_collection_reads_through_its_own_ucs2_cmap` (GB1, CNS1, Korea1, Identity-V, `/H` over CFF) and `a_font_outside_the_adobe_collections_gets_no_text_from_a_ucs2_cmap` (pdfcraft-render); (85) `text::tests::{a_tounicode_written_as_cid_ranges_gives_text, a_tounicode_bf_entry_wins_over_a_cid_entry_for_the_same_code}` (pdfcraft-render); (86) `widgets_above_annotations::a_widget_is_drawn_above_the_comments_on_its_page` (pdfcraft-render `tests/it`); (87) `raster::tests::type3_proc_cache::{a_type3_glyph_shown_again_decodes_its_procedure_once, a_procedure_over_64_kib_is_decoded_for_each_copy, the_procedures_one_cache_keeps_stop_at_32_mib, fonts_with_the_same_glyph_name_keep_their_own_procedures}` (pdfcraft-render); (88) `raster::tests::non_embedded_arial::{a_times_new_roman_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (89) `raster::tests::a_form_that_draws_itself_is_drawn_once`, `raster::tests::a_form_that_reaches_itself_through_a_pattern_or_a_glyph_is_drawn_once` (pdfcraft-render); (90) `raster::tests::type3_proc_widths::{a_type3_font_without_widths_advances_by_its_glyph_procedures, a_code_outside_first_char_to_last_char_still_has_width_zero, widths_the_font_gives_are_kept, a_code_without_a_usable_procedure_width_still_advances_nothing, a_type3_font_without_widths_is_reported}` (pdfcraft-render); (91) `raster::tests::failed_image_decodes::{a_picture_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_that_does_not_decode_is_decoded_once_at_each_size, a_stencil_mask_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_whose_soft_mask_does_not_decode_draws_unmasked_decoded_once, a_picture_that_decodes_is_still_decoded_once_and_drawn_every_time, a_picture_left_out_for_the_pages_allowance_draws_on_the_next_page, a_picture_whose_jbig2_mask_a_limit_refuses_is_decoded_once_however_often_it_is_drawn, a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw}` (pdfcraft-render); (92) `text::tests::agl_unicode_from_name_rule_reads_text` (pdfcraft-render); (93) `cid_cff_unusable_font_dict::{glyphs_of_a_usable_font_dict_draw_beside_an_unusable_one, a_program_with_no_usable_font_dict_draws_no_square}` (pdfcraft-render integration tests); (94) `raster::tests::a_font_without_a_subtype_is_drawn_by_its_base_font` (pdfcraft-render); (98) `symbol_mt_builtin_encoding::{symbol_mt_draws_symbols_through_the_built_in_encoding, a_name_that_only_starts_with_symbol_is_not_symbol, symbol_mt_applies_its_differences, an_embedded_symbol_mt_draws_its_own_program}` (pdfcraft-render integration tests); (99) `raster::tests::thin_patches_of_a_mesh_in_a_tight_path_are_all_painted` (pdfcraft-render); (100) `text::tests::numbered_glyph_names_read_as_their_own_code`, `text::tests::numbered_type3_glyph_names_read_as_their_own_code`, `text::tests::glyph_names_outside_the_glyph_list_still_read_as_text` (pdfcraft-render); (101) `raster::tests::symbolic_truetype_mac_roman_text::{a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman, winansi_codes_of_a_font_in_the_macintosh_order_read_as_before, a_symbolic_truetype_font_without_a_1_0_subtable_reads_as_before, a_1_0_subtable_selecting_other_glyphs_than_the_drawn_ones_is_not_read, a_font_that_names_its_glyphs_is_not_read_by_their_order, a_symbol_or_zapfdingbats_font_in_the_macintosh_order_gets_no_mac_roman_reading, differences_naming_notdef_clear_the_mac_roman_reading_for_its_code, a_base_encoding_name_this_reader_does_not_know_keeps_the_mac_roman_reading_off}` (pdfcraft-render), `pdf::tests::a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman` (pdfcraft-fonts); (102) `text::tests::tracked_vertical_latin_stays_one_line` (pdfcraft-render); (81) `raster::tests::replies_are_not_drawn_but_their_comment_and_group_members_are`, `raster::tests::an_unknown_or_malformed_rt_is_a_reply_and_a_widget_with_irt_still_draws` (hayro-interpret); (103) `raster::tests::the_view_auto_state_sets_a_groups_state_on_screen`, `raster::tests::layer_toggles_and_print_renders_are_not_the_view_auto_state` and `raster::tests::malformed_view_auto_state_is_ignored` (pdfcraft-render); (104) `raster::tests::isolated_cmyk_groups::an_rgb_colour_in_an_isolated_cmyk_group_blends_as_its_ink` and `::an_rgb_colour_with_a_non_finite_component_has_no_tints` (pdfcraft-render); (105, 106) `raster::tests::annotation_and_appearance_overrides_change_only_the_view` (pdfcraft-render) | | hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Each target row averages the whole source rows it covers (contiguous, cheap) and is then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms); working a row at a time (`area_shrink_rows`) keeps its work memory to a few rows. An image with a soft mask, a stencil's ink included, is premultiplied a source row at a time as it is averaged (`area_shrink_premultiplied`), never as a full-resolution RGBA copy: a 10718 × 13701 stencil shown at page size peaked at 1.16 GB with that copy and peaks at 1.03 GB without it, as with nearest-neighbour. A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged by more than 1% on either side (`SMOOTHED_IMAGE_SCALE`: a scan drawn at its resolution can come out a few ulps above 1, and blurred by half a pixel at a half-pixel offset), also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms; (30) a stencil mask painted with a pattern is drawn under a clip of its device rectangle, padded by two pixels: the pattern in an isolated layer, cut to the stencil by a layer holding it composited with `Compose::DestIn` (drawn without a Type 3 glyph's image rules, as the sub-render it replaces was), instead of through a canvas-sized luminance mask rendered for every draw; and a coloured tiling pattern's cell pixmap, repeated or shown once under patch (20), is drawn once per setting of the pattern and cell scale (`Renderer::tiling_cell`, `TilingCells`, eight entries per renderer, keyed by hayro-interpret patch (78)'s `TilingPattern::instance`) instead of once per fill, where every cell was kept until the page was composited. pdf.js issue 8078 paints 73,047 16 × 16 stencils with one pattern whose cell is 3.5 MB at 36 dpi: it ran out of memory after 5 minutes and now renders in about 5 s. For its tests, `offscreen_draws` counts on each thread the tiling cells drawn and the pixels of every pixmap drawn offscreen (all made by `offscreen_pixmap`), so a page's cost is checked by count, not by time; (31) `RenderSettings::max_coverage` keeps the rasterizer to that many coverage values for the page (vello_cpu patch 1), and a page whose paths it left out says so with `InterpreterWarning::ContentTruncated` | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::a_shrunk_stencil_mask_keeps_thin_strokes`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched`, `raster::tests::a_scan_drawn_a_rounding_error_above_its_resolution_stays_sharp`; (30) `raster::tests::{a_patterned_stencil_mask_paints_only_where_it_inks, a_patterned_stencil_mask_under_a_soft_mask_is_masked_once, a_patterned_stencil_mask_in_a_type3_glyph_keeps_thin_strokes, many_patterned_stencil_masks_draw_no_page_sized_masks, fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}`; (31) `raster::tests::coverage_budget::a_page_past_its_coverage_leaves_the_rest_out_and_says_so` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most as many symbol pixels per symbol dictionary as the largest page bitmap holds, 2^28 (a fuzzed dictionary decoded for minutes; each symbol now pays for its pixels out of the stream's decoding work and is held in the retained bitmaps, and at the earlier 2^26 the soft mask of a scanned drawing, one dictionary of 7,559 symbols, was refused: #6722), and a refusal by one of the decoder's limits is told apart from damaged data (`DecodeError::is_limit`, `OverflowError::BitmapLimit`); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change; (13) a bitmap filled with a default pixel of 1 (a page, or an intermediate text or halftone region) had 1s past its width in the last word of each row, and a refinement, which reads its reference a word at a time, read them as pixels where the standard has 0 (6.3.5.2): refining such a reference not a multiple of 32 wide decoded to other pixels than were coded, release builds too. `Bitmap::clear_padding` clears those bits when a bitmap is filled with 1s, as cutting symbols and patterns from a collective bitmap already did. Ghostscript reads the test's page and text streams as they decode now; (14) a region's location is unsigned (5.4.4), so one past `i32::MAX` is off the page, where the decoder took it for a signed one: a region there was drawn back onto the page from the left or the top (at 0xFFFFFFFC, four pixels left of it), and a refinement there, of the page or of a region at the same place (7.4.7.5), was refused as an overflow, and the page with it. `Bitmap::combine_region` keeps such a location past the page. A refinement of another region refines it where it is, as Table 35 has it (GRREFERENCEDX and GRREFERENCEDY 0), whatever the two locations say; a refinement of the page, see (16); (15) a refinement whose reference is offset right of it (GRREFERENCEDX above 0, which a refined text instance has from a wider bitmap, Table 12, and a refined symbol from an RDX above 0, Table 18) read the reference's first 32 columns as 0: `RefinementContextGatherer::maybe_reload_buffers` emptied its buffers left of the reference and didn't load them again at its first column. Such a refinement decoded to other pixels than were coded, or failed and lost the page. Left of the reference the buffers now keep the words of its first column each row starts with; with an offset of 0 or less nothing changes; (16) a refinement of the page refines "the current contents of the page buffer, restricted to the area of the page buffer specified by this segment's region segment information field" (7.4.7.4), at an offset of 0 (Table 35): `Bitmap::area` copies that area a word at a time, 0 off the page and past its width whatever the padding of its rows holds, and every refinement region reads its reference at offset 0. The page around the area was read too, so a refinement of the page decoded to other pixels than were coded, and one wider or taller than the page was refused. And `RefinementContextGatherer::start_row` loads the reference's words a column left of where a row starts in it, as the reload would, not from column 0: with the reference offset 32 or more left of the refinement (GRREFERENCEDX below −31, as a refined text instance has from a narrower bitmap or an RDX below −31, and a refined symbol from such an RDX), each row's first context read 0 for the reference column it starts at, and from 33 left for the one before it too, and the refinement decoded to other pixels or failed; (17) `Image::new_embedded` reads data that starts with a JBIG2 file header (the ID string `0x97 4A 42 32 0D 0A 1A 0A`, flags and an optional page count, D.4) past it, in the organization the header declares (sequential or random access), its segments counted together with the globals' as (8) has them: ISO 32000-1 7.4.7 has `JBIG2Decode` data in the embedded organization (D.3), but some producers write the file header too, and Acrobat and PDFBox draw such streams. The header's ID string was read as a segment header and the image failed (`unknown or reserved segment type`), so the page drew white | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding (about a second at the earlier 2^26 bound; not run at 2^28), `raster::tests::a_symbol_dictionary_holds_as_many_pixels_as_a_page`, and for `is_limit` and `BitmapLimit` `error::pdfcraft_limit_tests`, `bitmap::pdfcraft_limit_tests` and `gray_scale::pdfcraft_limit_tests` (hayro-jbig2); (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing`; (13) `jbig2::decode::tests::a_refinement_reads_nothing_past_its_references_width` (pdfcraft-filters), and in the crate `tests::a_bitmap_of_1s_has_nothing_past_its_width`; (14) `jbig2::decode::tests::a_region_located_past_i32_max_is_off_the_page` (pdfcraft-filters), and in the crate `tests::a_region_located_past_i32_max_is_off_the_page` and `tests::a_refinement_of_a_region_decodes_the_same_wherever_they_are`; (15) `jbig2::decode::tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column` (pdfcraft-filters), and in the crate `tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column`; (16) `jbig2::decode::tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` (pdfcraft-filters), and in the crate `tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` and `tests::a_page_area_is_the_page_inside_it_and_0_outside` ; (17) `jbig2::decode::tests::a_stream_with_a_jbig2_file_header_decodes_past_it`, `jbig2::decode::tests::a_truncated_jbig2_file_header_is_an_error`, `jbig2::decode::tests::segments_after_a_jbig2_file_header_count_with_the_globals` (pdfcraft-filters), `raster::tests::a_jbig2_image_with_a_file_header_draws` (pdfcraft-render) | | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365); (7) object parsing builds at most 4,194,304 objects (every array element and dictionary value at any depth, `parser::MAX_OBJECT_NODES`, the same cap as `pdfcraft_cos`) per top-level object, an object stream reads at most 1,000,000 index entries (cos's `/N` clamp), and one load builds at most 8,388,608 objects from all of its object streams together, shared across rayon workers (a decoded stream of `0 0 0 ...` inside the 256 MiB decompression limit built ~120M objects, 10+ GB, and aborted). A lookup parses only the member it wants, so it isn't skipped behind a large member, and a load parses only the members it would add: none its cross-reference table places in another stream (stale copies) and none already loaded as an ordinary object or from an earlier stream. Members the table doesn't list at all are still parsed (a rebuilt table lists no compressed objects) and share the budget, so they can spend it before listed members of the same stream; a load expands its object streams one at a time in object-number order, a stream's members in parallel only when the budget holds all of them and otherwise in index order, so a spent budget skips the same members on every load (#5083) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog`; (7) `inspect::tests::{object_stream_members_over_the_node_cap_are_skipped, object_stream_index_entries_past_the_clamp_are_ignored, object_streams_of_one_load_share_their_node_budget, object_streams_loaded_in_parallel_share_their_node_budget, a_malformed_object_stream_member_refunds_its_reservation, a_tight_budget_builds_object_stream_members_in_index_order, a_lookup_in_an_object_stream_parses_only_the_member_it_wants}` (pdfcraft-render; `NodeBudget` and `ObjectStream::new_within` are public so these can use a small budget) | diff --git a/vendor/hayro-interpret/src/cmyk_inverse.rs b/vendor/hayro-interpret/src/cmyk_inverse.rs new file mode 100644 index 0000000000..74804f92a7 --- /dev/null +++ b/vendor/hayro-interpret/src/cmyk_inverse.rs @@ -0,0 +1,118 @@ +//! PdfCraft patch (104): an sRGB colour as the process ink that draws back to it through the +//! CMYK profile pages are drawn with, for an additive colour painted into a `DeviceCMYK` +//! blending space (ISO 32000-2 §11.3.4). +//! +//! Black is `K = 0.4 · k₀² + 0.6 · k₀⁸` with `k₀ = 1 − max(r, g, b)`, light in the mid tones +//! and full for black; cyan, magenta and yellow are then solved for by Newton's method at that +//! black, clamped to the ink range. A colour inside the profile's gamut draws as itself; one +//! outside it lands where the clamped solve stops. The curve matches Acrobat's output for the +//! greys, black and red measured in a CMYK page group, plain and under a blend mode; +//! §10.4.2.4's black generation and undercolour removal is for a device without a profile. + +use moxcms::{ColorProfile, Layout, TransformF32Executor, TransformOptions}; +use std::cell::Cell; +use std::sync::{Arc, LazyLock}; + +use crate::color::CMYK_PROFILE; + +/// The weight of `k₀²` in the black ink; the rest is on `k₀⁸`. +const BLACK_GENERATION: f32 = 0.4; +/// Newton steps; each costs four profile evaluations. +const MAX_STEPS: usize = 12; +/// Close enough: well under one 8-bit level. +const TOLERANCE: f32 = 0.5 / 255.0; +/// The finite-difference step for the Jacobian. +const STEP: f32 = 0.02; + +static FORWARD: LazyLock>> = LazyLock::new(|| { + ColorProfile::new_from_slice(CMYK_PROFILE) + .ok()? + .create_transform_f32( + Layout::Rgba, + &ColorProfile::new_srgb(), + Layout::Rgb, + TransformOptions::default(), + ) + .ok() +}); + +/// The process ink `[C, M, Y, K]` (0 = no ink) that the CMYK profile draws as `rgb`; for a +/// colour outside its gamut, where the clamped solve stops. `None` when the +/// profile can't be evaluated or `rgb` isn't finite. +/// +/// The last colour converted on this thread is kept, as a run of fills (the glyphs of a +/// line of text) usually shares one colour. +pub(crate) fn srgb_to_cmyk(rgb: [f32; 3]) -> Option<[f32; 4]> { + if !rgb.iter().all(|v| v.is_finite()) { + return None; + } + let rgb = rgb.map(|v| v.clamp(0.0, 1.0)); + let key = rgb.map(f32::to_bits); + if let Some((last, ink)) = LAST.get() + && last == key + { + return Some(ink); + } + let ink = solve_ink(rgb)?; + LAST.set(Some((key, ink))); + Some(ink) +} + +thread_local! { + static LAST: Cell> = const { Cell::new(None) }; +} + +fn solve_ink(rgb: [f32; 3]) -> Option<[f32; 4]> { + let forward: &TransformF32Executor = FORWARD.as_ref()?.as_ref(); + let k0 = 1.0 - rgb[0].max(rgb[1]).max(rgb[2]); + let k = BLACK_GENERATION * k0 * k0 + (1.0 - BLACK_GENERATION) * k0.powi(8); + let eval = |cmy: [f32; 3]| draw(forward, [cmy[0], cmy[1], cmy[2], k]); + + let mut cmy = rgb.map(|v| 1.0 - v); + for _ in 0..MAX_STEPS { + let drawn = eval(cmy)?; + let residual: [f32; 3] = std::array::from_fn(|i| drawn[i] - rgb[i]); + if residual.iter().all(|r| r.abs() < TOLERANCE) { + break; + } + // Column `j` of the Jacobian: how the drawn colour moves with ink `j`. + let mut columns = [[0.0_f32; 3]; 3]; + for (j, column) in columns.iter_mut().enumerate() { + let step = if cmy[j] <= 1.0 - STEP { STEP } else { -STEP }; + let mut moved = cmy; + moved[j] += step; + let shifted = eval(moved)?; + *column = std::array::from_fn(|i| (shifted[i] - drawn[i]) / step); + } + let Some(delta) = solve(columns, residual.map(|r| -r)) else { + break; + }; + cmy = std::array::from_fn(|i| (cmy[i] + delta[i]).clamp(0.0, 1.0)); + } + Some([cmy[0], cmy[1], cmy[2], k]) +} + +fn draw(forward: &TransformF32Executor, cmyk: [f32; 4]) -> Option<[f32; 3]> { + let mut out = [0.0_f32; 3]; + forward.transform(&cmyk, &mut out).ok()?; + out.iter().all(|v| v.is_finite()).then_some(out) +} + +/// `x` with `A x = b`, `A` given by its columns (Cramer's rule). `None` when `A` is +/// singular or the result isn't finite. +fn solve(a: [[f32; 3]; 3], b: [f32; 3]) -> Option<[f32; 3]> { + let det = |[c0, c1, c2]: [[f32; 3]; 3]| { + c0[0] * (c1[1] * c2[2] - c2[1] * c1[2]) - c1[0] * (c0[1] * c2[2] - c2[1] * c0[2]) + + c2[0] * (c0[1] * c1[2] - c1[1] * c0[2]) + }; + let d = det(a); + if !d.is_finite() || d.abs() < 1e-9 { + return None; + } + let x: [f32; 3] = std::array::from_fn(|j| { + let mut m = a; + m[j] = b; + det(m) / d + }); + x.iter().all(|v| v.is_finite()).then_some(x) +} diff --git a/vendor/hayro-interpret/src/color.rs b/vendor/hayro-interpret/src/color.rs index 4021b31e04..4a7a05f1b6 100644 --- a/vendor/hayro-interpret/src/color.rs +++ b/vendor/hayro-interpret/src/color.rs @@ -1552,9 +1552,10 @@ impl Color { /// its opacity, for blending in a `DeviceCMYK` transparency group (ISO 32000-2 §11.3.4). /// /// `DeviceCMYK` gives its components. `DeviceGray` is black ink alone, `K = 1 − gray` - /// (§10.4.2.2). `None` for every other space, ICCBased CMYK included: such a colour has no - /// tints without a colour conversion this crate does not make (an ICCBased colour keeps - /// its own profile), and the caller draws it as before. + /// (§10.4.2.2). PdfCraft patch (104): an additive colour (`DeviceRGB`, `CalRGB`, `Lab`, a + /// three-component ICCBased colour) is the ink that draws back to it through the CMYK + /// profile. `None` for every other space, ICCBased CMYK included (it keeps its own + /// profile), and the caller draws it as before. pub fn ink_tints(&self) -> Option<([f32; 4], f32)> { let c = |i: usize| { self.components @@ -1566,11 +1567,27 @@ impl Color { let tints = match self.color_space.0.as_ref() { ColorSpaceType::DeviceCmyk => [c(0), c(1), c(2), c(3)], ColorSpaceType::DeviceGray => [0.0, 0.0, 0.0, 1.0 - c(0)], + ColorSpaceType::DeviceRgb | ColorSpaceType::CalRgb(_) | ColorSpaceType::Lab(_) => { + self.ink_from_srgb()? + } + ColorSpaceType::ICCBased(icc) if icc.0.number_components == 3 => { + self.ink_from_srgb()? + } _ => return None, }; Some((tints, self.opacity.clamp(0.0, 1.0))) } + fn ink_from_srgb(&self) -> Option<[f32; 4]> { + // `to_rgba` turns a non-finite component into some finite colour, which is not the + // colour's ink, so such a colour has no tints. + if self.components.iter().any(|c| !c.is_finite()) { + return None; + } + let [r, g, b, _] = self.to_rgba().components(); + crate::cmyk_inverse::srgb_to_cmyk([r, g, b]) + } + /// Create a color from RGBA. pub fn from_rgba(rgba: AlphaColor) -> Self { let c = rgba.components(); @@ -1592,9 +1609,11 @@ pub fn cmyk_to_srgb8(input: &[u8], output: &mut [u8]) -> Option<()> { CMYK_TRANSFORM.convert_u8(input, output) } -static CMYK_TRANSFORM: LazyLock = LazyLock::new(|| { - ICCProfile::new(include_bytes!("../assets/CGATS001Compat-v2-micro.icc"), 4).unwrap() -}); +/// The CMYK profile pages are drawn with. +pub(crate) const CMYK_PROFILE: &[u8] = include_bytes!("../assets/CGATS001Compat-v2-micro.icc"); + +static CMYK_TRANSFORM: LazyLock = + LazyLock::new(|| ICCProfile::new(CMYK_PROFILE, 4).unwrap()); pub(crate) trait ToRgb { fn convert_sample(&self, input: &[f32], output: &mut [u8], manual_scale: bool) -> Option<()> { diff --git a/vendor/hayro-interpret/src/interpret/mod.rs b/vendor/hayro-interpret/src/interpret/mod.rs index daabc6e3ce..0e341b124e 100644 --- a/vendor/hayro-interpret/src/interpret/mod.rs +++ b/vendor/hayro-interpret/src/interpret/mod.rs @@ -117,11 +117,11 @@ pub struct InterpreterSettings { /// `ocg_overrides`. On by default: that is the state for viewing. Off for output made for /// printing, which the View event is not. pub view_auto_state: bool, - /// PdfCraft patch (104): viewer overrides for annotations (object number, generation, shown), + /// PdfCraft patch (105): viewer overrides for annotations (object number, generation, shown), /// applied in place of their Hidden and NoView flags: what a document's scripts show and /// hide while it is read (frames of an animation), without changing the document. pub annotation_overrides: Arc>, - /// PdfCraft patch (105): annotations the viewer draws with a form XObject as their normal + /// PdfCraft patch (106): annotations the viewer draws with a form XObject as their normal /// appearance (object number and generation of the annotation, then of the form), in their /// own `/Rect`: a push button a document's script gave another icon (`buttonSetIcon`) while /// it is read, without changing the document. An override naming an object that isn't a form @@ -234,7 +234,7 @@ pub fn interpret_page<'a>( // annotations, each group in `/Annots` order, as other readers draw them: a filled // comment listed after a check box covered the box. ISO 32000-2 doesn't order the two. // Two passes over the array, so nothing is held for every entry of a long one. - // PdfCraft patch (104): each annotation with its object, so the viewer can show or hide + // PdfCraft patch (105): each annotation with its object, so the viewer can show or hide // it by its object (`annotation_overrides`). An entry that isn't a dictionary is // skipped; it no longer ends the list. let xref = context.xref; @@ -252,7 +252,7 @@ pub fn interpret_page<'a>( // Annotation should be hidden (Hidden = bit 2, NoView = bit 6). // PdfCraft patch: NoView was not honoured upstream. - // PdfCraft patch (104): the viewer's override, when it has one, decides instead. + // PdfCraft patch (105): the viewer's override, when it has one, decides instead. let overridden = obj.and_then(|r| { context .settings @@ -295,7 +295,7 @@ pub fn interpret_page<'a>( // PdfCraft patch: `/N` may be a stream or a dictionary of appearance states selected // by `/AS` (checkboxes, radio buttons, …). Upstream only handled the stream form. - // PdfCraft patch (105): the viewer may draw it with another form instead + // PdfCraft patch (106): the viewer may draw it with another form instead // (`appearance_overrides`). // An override naming an object that isn't a form draws nothing: a script cleared the // button's icon. diff --git a/vendor/hayro-interpret/src/lib.rs b/vendor/hayro-interpret/src/lib.rs index 01e717da93..03a0f44811 100644 --- a/vendor/hayro-interpret/src/lib.rs +++ b/vendor/hayro-interpret/src/lib.rs @@ -33,6 +33,7 @@ mod log; mod cache; mod cmyk_group; +mod cmyk_inverse; mod context; mod convert; mod device; diff --git a/vendor/hayro/src/renderer.rs b/vendor/hayro/src/renderer.rs index c8640adb14..372f7e28cc 100644 --- a/vendor/hayro/src/renderer.rs +++ b/vendor/hayro/src/renderer.rs @@ -922,8 +922,8 @@ impl TilingCells { #[derive(Clone, Copy, Default)] struct InkState { pass: Option, - /// Something in the group has no ink tints (an RGB colour, a pattern, an image): the - /// group is drawn as before instead. + /// Something in the group has no ink tints (a pattern, an image, an ICCBased CMYK colour): + /// the group is drawn as before instead. unsupported: bool, }