diff --git a/.github/workflows/house-rules.yml b/.github/workflows/house-rules.yml new file mode 100644 index 0000000000..de64ad3ee4 --- /dev/null +++ b/.github/workflows/house-rules.yml @@ -0,0 +1,28 @@ +name: House rules + +on: + push: + branches: [main] + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + pull_request: + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: house-rules-${{ github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + uses: storytold/craft-common-actions/.github/workflows/house-rules.yml@main + with: + rust: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8549142f7..4b44d79b68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,14 @@ concurrency: cancel-in-progress: false jobs: + # Regenerates contributors/contributors.json for the release commit; every build bundles it. + credits: + permissions: + contents: read + pull-requests: read + uses: storytold/craft-common-actions/.github/workflows/refresh-contributors.yml@main release: + needs: credits # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: @@ -27,6 +34,7 @@ jobs: app-title: PdfCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 version: ${{ inputs.version }} + contributors-artifact: ${{ needs.credits.outputs.artifact-name }} secrets: APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} @@ -42,3 +50,4 @@ jobs: AZURE_SIGNING_ENDPOINT: ${{ secrets.AZURE_SIGNING_ENDPOINT }} AZURE_SIGNING_ACCOUNT: ${{ secrets.AZURE_SIGNING_ACCOUNT }} AZURE_CERT_PROFILE: ${{ secrets.AZURE_CERT_PROFILE }} + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} diff --git a/AGENTS.md b/AGENTS.md index 3ab900aa95..b68d8c8921 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,59 @@ + +## House rules + +These apply in every Craft repository. A later rule may be stricter, not weaker. The source is `standards/house-rules-core.md` in `storytold/craft-contrib`. Propose changes there, not in a copy. + +1. **A person is accountable.** Whoever runs an agent answers for what it opens and for its review comments. +2. **Check, then claim.** Before writing code, read the issue comments, search pull requests for `#N` and the keywords, and check recent commits on `main`. Comment on the issue first. Release a claim you will not finish. +3. **Verify, don't guess.** Open a definition before using it. Confirm the cause in the code, even when someone names a location. +4. **Keep fixes narrow and tested.** Fix what the issue describes, where the bug is. Add a test that fails without the fix. +5. **Review the diff before opening a pull request.** Identifiers exist as used, the test fails without the fix, nothing unrelated changed, and the description is true. +6. **Look for a better pull request.** If another fixes the same thing, close yours when it is better or equal, and say why. +7. **Report only what ran.** Say what you tested and what you could not run. Without a local build or test, say so and rely on CI before marking ready. +8. **`main` changes through a pull request.** Do not push to it. Branch protection enforces this. There is no exception. +9. **Stop and ask only about** licensing outside the allowlist, publishing, releases, a new remote, secrets, spending money, destructive git, and anything else hard to undo. Otherwise keep working. `plan/` is a gitignored notebook, not rules. Without it, do not ask for it and do not stop. +10. **Keep secrets and personal details out** of the repository, logs and requests. +11. **CI stays shared.** Workflows call `storytold/craft-common-actions`. Do not change CI in a feature or fix pull request. If CI cannot cover a change, open an issue there. +12. **Guards say why.** A check that looks removable says what it prevents. Read that before changing it. +13. **Instructions must be readable by everyone.** No notes about one machine, and no links to files a reader cannot open. `craftrules` is retired. Do not link to it. +14. **Keep pull requests moving.** Concurrent pull requests are fine; unanswered actionable feedback is not. Respond in 7 days, a reminder at 14, closure eligible at 30, from when an actionable request waits on the author. Suggestions and unconfirmed bot findings do not start the clock. A fix, explanation or agreed plan resets it; unrelated activity does not. Waiting on a reviewer or maintainer pauses it and does not count against the author. A maintainer decides closure; it is never automatic. Reuse search results, do not poll, and stop at a rate limit. +15. **No agent attribution.** A pull request does not say an agent wrote it. A repository rule may not require that. + + +## Rust Craft repositories + +These apply to native Craft apps and to shared Rust libraries. Select them by repository type. A repository that merely contains some Rust does not take this block. Rules later in a repository's file may be stricter, but not weaker. The source is `standards/house-rules-rust.md` in [storytold/craft-contrib](https://github.com/storytold/craft-contrib). + +1. **Never crash.** Non-test code does not panic and has no `unsafe`, except where the repository file names an exception. Return errors, treat input as hostile, and bound recursion and allocations. New production crate roots copy the `deny` attribute from an existing crate: `clippy::unwrap_used`, `clippy::expect_used`, `clippy::panic`, `clippy::unimplemented`, `clippy::todo`, `clippy::unreachable`. Every crash fix comes with a test that panicked before the fix. +2. **Clean room.** Learn from observed behaviour and public specifications. Do not copy another product's code, assets, strings or artwork. Check a dependency's licence before adding it. +3. **Assets and fonts.** Every asset is original or suitably licensed, and is recorded in `ATTRIBUTION.md` in the same change. Font files live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), not in the app repository. +4. **Rust only for native Craft apps.** No handwritten JavaScript or TypeScript, and no Tauri, Electron or webview shell, in those apps. Shared tooling follows its documented technical scope and maintainer-approved exceptions. Craft PR Review's existing Tauri and TypeScript UI is one such exception. It does not relax this rule for native apps. Preserve any existing web build. +5. **Time goes through `chrono`.** Do not hand-roll calendar conversion. For dates beyond what `chrono` can represent, follow [standards/time.md](https://github.com/storytold/craft-contrib/blob/main/standards/time.md). +6. **Contributor credits.** Add or edit only your own person's entry in `contributors/people.toml`, and only after asking them. Never guess a name or hand-edit `contributors.json`. + +## Licence allowlist + +Dependencies and assets use SPDX identifiers. An expression is allowed when every identifier is in this list. A repository may name a shorter list. It may not add an identifier here by editing its own copy of these rules. + +- `ISC` +- `MIT` +- `BSD-2-Clause` +- `BSD-3-Clause` +- `Apache-2.0` +- `OFL-1.1` +- `Ubuntu-font-1.0` +- `Bitstream-Vera` +- `CC0-1.0` +- `CC-BY-4.0` +- `CC-BY-SA-4.0` +- `LicenseRef-Public-Domain` + +`LicenseRef-Storyteller-Trademark` is not in this list. It covers Storyteller brand marks only, where a repository already says so. + + # AGENTS.md — rules for every agent and contributor -These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** +These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. Day-to-day working instructions are in this file; `CLAUDE.md` imports it. --- @@ -85,7 +138,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi --- -## 2. Clean-room rules (summary; details in `CLAUDE.md` and `plan/adr/0001`) +## 2. Clean-room rules (summary; `plan/adr/0001` is an optional local note) - Never read, disassemble or copy anything inside the Acrobat bundle. Only names and listings are allowed, and **never** open `Contents/Resources/JavaScripts/`. - Behaviour comes only from public specifications (ISO 32000-2, the Arlington model), public documentation and black-box observation. @@ -106,7 +159,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi PdfCraft opens files from strangers, and people trust it with their work. A malformed PDF, a bad tool argument, a corrupt settings file or a full disk must produce an error the user (or agent) can act on, never a crash and never lost work. **This outranks feature work:** don't ship a feature by adding a panic path, and fix a crash before building on top of it. The rules below are the ones that bind this repository. -- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity in `CLAUDE.md`). +- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity below). - **No panicking shortcuts.** No `unwrap()`, `expect()`, `panic!`, `unreachable!`, `todo!` or `unimplemented!` outside tests, unless the call is provably infallible (e.g. writing to a `Vec`), with a comment saying why. `assert!` only for internal invariants that input cannot reach. - **No `unsafe`.** The workspace sets `unsafe_code = "forbid"`. If a platform API needs it, use a maintained safe wrapper crate; a real exception needs the owner's sign-off. - **Treat every input-derived number as hostile.** Use `get()` instead of `[i]` or `[a..b]`. Slice strings only at char boundaries. Use checked/saturating arithmetic for lengths, offsets and counts. Guard against division by zero and NaN/inf casts, and never cast a negative value to `usize`. Cap allocations sized by the file. @@ -115,10 +168,18 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Prove it.** Every crash fix comes with a small synthetic regression test that panicked before the fix. Turn fuzzer findings (`cargo xtask fuzz`) into such tests before fixing them. - **Enforced by clippy.** Root `clippy.toml` lets tests use `unwrap`/`expect`/`panic!`. A crate that is clean carries `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`; new crates start with it. -## 5. Everything else +## 5. Page interaction, fidelity and gates - -Follow `CLAUDE.md`. It covers quality gates, layering, commit rules and how to run and look at the app. +- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd. `cos`, `filters`, `crypt` and `arlington` stay standalone. +- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state. `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. +- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim`, `crates/ui-egui/src/canvas/page_cursor.rs`) and never set it themselves. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. +- **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. Modules never read the router; they are told. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. +- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. +- **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `workspace = true` under `[lints]`. Scope `cargo clippy -p --all-targets -- -D warnings` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. +- `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. +- `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. ## Contributor credits (About window) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index a8541c1942..a8de85e07a 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -4,10 +4,11 @@ Every asset PdfCraft includes, bundles or uses to build its published material, with its author, source and licence. The policy is in [AGENTS.md](AGENTS.md) §1. The machine-readable list, with the SHA-256 of every asset, is [ATTRIBUTION.toml](ATTRIBUTION.toml). Licence texts are kept beside the assets and summarised in [NOTICE](NOTICE). -## In this repository (255) +## In this repository (258) | Asset | Title | Author | Licence | Source | Used for | |---|---|---|---|---|---| +| `docs/images/pdfcraft-ocr-verify.png` | PdfCraft screenshot: OCR Verify screen | PdfCraft contributors (generated by `cargo xtask screenshots`) | MIT OR Apache-2.0 | this repository | README: OCR review. Shows only PdfCraft UI, its Lucide icons and a synthetic scanned page built from the embedded fonts, with a real recognition (the OCR models) loaded into the Verify screen | | `assets/icons/accessibility.svg` | Lucide icon "accessibility" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/accessibility.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/align-left.svg` | Lucide icon "align-left" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/align-left.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/align-center.svg` | Lucide icon "align-center" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/align-center.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -44,7 +45,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/chevron-up.svg` | Lucide icon "chevron-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevron-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-left.svg` | Lucide icon "chevrons-left" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-left.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-right.svg` | Lucide icon "chevrons-right" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-right.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/circle-dashed.svg` | Lucide icon "circle-dashed" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dashed.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-dot.svg` | Lucide icon "circle-dot" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dot.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-help.svg` | Lucide icon "circle-help" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-help.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -173,7 +174,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/text-select.svg` | Lucide icon "text-select" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/text-select.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash-2.svg` | Lucide icon "trash-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash.svg` | Lucide icon "trash" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/type.svg` | Lucide icon "type" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/type.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/underline.svg` | Lucide icon "underline" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/underline.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/undo-2.svg` | Lucide icon "undo-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/undo-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -183,7 +184,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/x.svg` | Lucide icon "x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-in.svg` | Lucide icon "zoom-in" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-in.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-out.svg` | Lucide icon "zoom-out" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-out.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths | +| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs | | `assets/fonts/Inter-Medium.ttf` | Inter Medium | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/Inter-SemiBold.ttf` | Inter SemiBold | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/JetBrainsMono-Regular.ttf` | JetBrains Mono Regular | The JetBrains Mono Project Authors | OFL-1.1 | https://github.com/JetBrains/JetBrainsMono | UI monospace font; demo PDF code | @@ -203,6 +204,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `vendor/hayro-interpret/assets/FoxitSymbol.pfb` | FoxitSymbol (standard-14 substitute font) | Foxit Software Inc. / PDFium Authors | BSD-3-Clause | https://pdfium.googlesource.com/pdfium (via hayro-interpret 0.7.0 assets/) | Renders non-embedded standard-14 fonts (vendored hayro-interpret) | | `vendor/hayro-interpret/assets/CGATS001Compat-v2-micro.icc` | CGATS001Compat v2 micro ICC profile | Clinton Ingram (saucecontrol/Compact-ICC-Profiles) | CC0-1.0 | https://github.com/saucecontrol/Compact-ICC-Profiles | Default CMYK colour conversion (vendored hayro-interpret) | | `crates/fonts/src/std14_metrics.rs` | Standard-14 font advance widths and encodings (as Rust tables) | Adobe (Core14 AFM metrics, PDF specification encodings); tables generated by the PdfCraft contributors | MIT OR Apache-2.0 | ISO 32000-2 Annex D and the Adobe Core14 AFM metrics. Generated by joining the published WinAnsi, Symbol and ZapfDingbats encodings (code to glyph name) with the per-glyph advance widths, then spot-checked against the specification by hand in the tests of crates/fonts/src/std14.rs. Data only: the code that reads these tables lives in std14.rs, so that ordinary edits do not churn this file's hash. | Advance widths for the fourteen standard fonts, which a PDF may name without embedding. Non-visual data, not a typeface: the item AGENTS.md 1.1 permits, held first-party so generated appearance streams and the renderer agree. | +| `crates/prc/src/acof/table.rs` | PRC Double prefix-code table (as a Rust table) | Kristof Ralovich (prc-rs); the code table itself is defined by the PRC format (ISO 14739-1, working draft clause 11.17.2) | MIT | Rows taken from prc-rs src/double.rs at commit 287b671bb5b31b05efb5ec1b98f7c359b9e4dd7d (https://github.com/ralovich/prc-rs), mechanically reformatted into (code length, code, entry) tuples; checked to be a complete prefix code by the tests in crates/prc/src/acof.rs. Data only: the code that reads the table lives in acof.rs. The MIT notice is kept in the file header. | Decodes every PRC Double (coordinates, transforms, colours) in 3D annotations' PRC streams. Non-visual data. | | `vendor/hayro-cmap/assets/cmaps.brotli` | Adobe CMap resources (predefined CJK CMaps) | Adobe | BSD-3-Clause | https://github.com/adobe-type-tools/cmap-resources (via hayro-cmap 0.1.0 assets/) | Character-code mapping tables required to read CJK PDFs. Non-visual data, allowed by AGENTS.md §1.1 | | `docs/brand/artcraft-logo-white.png` | ArtCraft brand: artcraft-logo-white | ArtCraft Team | LicenseRef-Storyteller-Trademark | ArtCraft Team brand assets (provided by the project owner, 2026-10-01; identical copies in storytold/photocraft) | ArtCraft branding next to the community links (About dialog, Help menu, home screen) and in docs. ArtCraft Team trademark, not open source; may be used only unmodified within PdfCraft; forks and modified versions must remove it. | | `docs/brand/artcraft-logo-white.svg` | ArtCraft brand: artcraft-logo-white | ArtCraft Team | LicenseRef-Storyteller-Trademark | ArtCraft Team brand assets (provided by the project owner, 2026-10-01; identical copies in storytold/photocraft) | ArtCraft branding next to the community links (About dialog, Help menu, home screen) and in docs. ArtCraft Team trademark, not open source; may be used only unmodified within PdfCraft; forks and modified versions must remove it. | @@ -247,7 +249,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/thumbs-up.svg` | Lucide icon "thumbs-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/thumbs-down.svg` | Lucide icon "thumbs-down" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-down.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-plus.svg` | Lucide icon "message-square-plus" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-plus.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `crates/ui-egui/tests/data/form.pdf` | One-page AcroForm test fixture (text, check box, radio group, combo box) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: hand-written PDF objects, no third-party content | Form-filling UI tests (crates/ui-egui/tests/it/forms.rs, commands.rs) | | `assets/icons/pentagon.svg` | Lucide icon "pentagon" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pentagon.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-quote.svg` | Lucide icon "message-square-quote" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-quote.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -262,6 +264,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/superscript.svg` | Lucide icon "superscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/superscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/subscript.svg` | Lucide icon "subscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/subscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/pilcrow.svg` | Lucide icon "pilcrow" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pilcrow.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/a-large-small.svg` | Lucide icon "a-large-small" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg) | UI icon, embedded via crates/ui-egui/src/widgets/icon_data.rs | | `assets/icons/move-horizontal.svg` | Lucide icon "move-horizontal" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/move-horizontal.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | ## Compiled in through dependencies (6) @@ -311,4 +314,4 @@ Not in this repository and never downloaded by it: compiled in only when the bui | Input | Option | Title | Author | Licence | Source | Attribution | Used for | |---|---|---|---|---|---|---|---| | `craft-i18n` | `CRAFT_I18N_DIR` | Interface translations: the craft-i18n catalogs (locales//common.tsv and pdfcraft.tsv) of the 34 languages PdfCraft offers besides English | The craft-i18n translators, including every translator of PdfCraft's former in-repo catalogs (crates/ui-egui/src/i18n/*.tsv, migrated at pdfcraft@ecdd1e6) | MIT OR Apache-2.0 | https://github.com/storytold/craft-i18n (pinned by commit in craft-i18n.rev, which CI, releases and the flatpak manifests read) | https://github.com/storytold/craft-i18n/blob/main/ATTRIBUTION.md | Interface text in languages other than English, embedded by crates/ui-egui/build.rs and looked up through craft-libs' craft-i18n crate; a build without CRAFT_I18N_DIR is English only. Clean-room translations from PdfCraft's English interface labels | -| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | +| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 783ecc6c9f..608bb601a5 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -10,6 +10,17 @@ # licence: SPDX expression from the allowlist in AGENTS.md §1.2. adobe_data = true marks the closed list of # Adobe-authored non-visual technical data allowed by AGENTS.md §1.1. +[[asset]] +path = "docs/images/pdfcraft-ocr-verify.png" +title = "PdfCraft screenshot: OCR Verify screen" +author = "PdfCraft contributors (generated by `cargo xtask screenshots`)" +source = "this repository" +licence = "MIT OR Apache-2.0" +licence_file = "LICENSE-MIT" +kind = "image" +usage = "README: OCR review. Shows only PdfCraft UI, its Lucide icons and a synthetic scanned page built from the embedded fonts, with a real recognition (the OCR models) loaded into the Verify screen" +sha256 = "ea7485b151e27ad3fd2a2dac079436ab11745d42032201a029bb20f819c6af8f" + [[asset]] path = "assets/icons/accessibility.svg" title = "Lucide icon \"accessibility\"" @@ -414,7 +425,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "201e790f3a05c65b2c4b0f952c9b9e1f35cd1c5b21698b3f4be131883fe90bc8" [[asset]] @@ -1833,7 +1844,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "55aa37e0b73e7f8cd906952dbdeb96283c6595ca64aba240a17f3455cb989114" [[asset]] @@ -1943,7 +1954,7 @@ source = "https://github.com/rsms/inter" licence = "OFL-1.1" licence_file = "assets/fonts/OFL-Inter.txt" kind = "font" -usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths" +usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs" sha256 = "40d692fce188e4471e2b3cba937be967878f631ad3ebbbdcd587687c7ebe0c82" [[asset]] @@ -2156,6 +2167,17 @@ adobe_data = true usage = "Advance widths for the fourteen standard fonts, which a PDF may name without embedding. Non-visual data, not a typeface: the item AGENTS.md 1.1 permits, held first-party so generated appearance streams and the renderer agree." sha256 = "f225e0b044875b8fa6375b6b5ad935f0d99fcc1ad03ecf9f0baae2cedbfca4d2" +[[asset]] +path = "crates/prc/src/acof/table.rs" +title = "PRC Double prefix-code table (as a Rust table)" +author = "Kristof Ralovich (prc-rs); the code table itself is defined by the PRC format (ISO 14739-1, working draft clause 11.17.2)" +source = "Rows taken from prc-rs src/double.rs at commit 287b671bb5b31b05efb5ec1b98f7c359b9e4dd7d (https://github.com/ralovich/prc-rs), mechanically reformatted into (code length, code, entry) tuples; checked to be a complete prefix code by the tests in crates/prc/src/acof.rs. Data only: the code that reads the table lives in acof.rs. The MIT notice is kept in the file header." +licence = "MIT" +licence_file = "licenses/prc-rs-LICENSE.txt" +kind = "data" +usage = "Decodes every PRC Double (coordinates, transforms, colours) in 3D annotations' PRC streams. Non-visual data." +sha256 = "8e705c5cde6a3ef778e8aca69759e38845b228b3f7f12949e587b19855d2114c" + [[bundled]] crate = "epaint_default_fonts" version = "0.37.0" @@ -2982,7 +3004,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "ef1fec17d88794ba6c34c5cf63bdadbd234b3d4dffbcc3d55468b47e3f398472" [[asset]] @@ -3120,7 +3142,7 @@ source = "https://github.com/storytold/craft-fonts (pinned by commit in .github/ licence = "OFL-1.1" attribution = "https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md" kind = "font" -usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" +usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" [[asset]] path = "crates/sign/tests/data/x509-rsa-sha1.pdf" @@ -3189,6 +3211,17 @@ kind = "icon" usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" sha256 = "d344d9f9222f6c1b698c0c82a753a2212d9bd7785f06c00cd6e4910fe2fee8f8" +[[asset]] +path = "assets/icons/a-large-small.svg" +title = "Lucide icon \"a-large-small\"" +author = "Lucide Contributors (portions Cole Bemis, Feather)" +source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg)" +licence = "ISC AND MIT" +licence_file = "assets/icons/LICENSE-lucide.txt" +kind = "icon" +usage = "UI icon, embedded via crates/ui-egui/src/widgets/icon_data.rs" +sha256 = "d4985fc71174d108e322c3b25cb6abe2837a115fe3f239a1918645db436743ec" + [[asset]] path = "assets/icons/move-horizontal.svg" title = "Lucide icon \"move-horizontal\"" diff --git a/CLAUDE.md b/CLAUDE.md index 5c7034b7a4..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,65 +1 @@ -# PdfCraft — instructions for agents - -PdfCraft is a clean-room, open-source, Rust-native PDF application targeting Adobe Acrobat Pro parity. It runs natively on macOS, Windows, Linux and FreeBSD, and on the web via WASM. It is the sibling of `../photocraft` (a Photoshop-class editor) and follows the same conventions. - -## Start every session here -1. Read `plan/STATUS.md`: the current milestone, the next unchecked task and any blockers. `ROADMAP.md` is the one-page summary (stage, numbers, progress log) and `docs/roadmap.md` the milestone table; at the end of every session update the milestone row and add a line to the ROADMAP.md progress log. - - Read `docs/gaps.md` (the ranked gaps: Acrobat interop, the open-issue backlog, the renderer and a fidelity harness, editing existing content, Pro workflows, 1.0 polish) and `docs/target-app-parity.md` (the numbers and how they are measured) before choosing work. Prefer the gaps over new P2/P3 features, and keep both documents true when things change (`docs/` progress docs carry a status line and a revision history). - - "Shipped" in `parity/` means "exists and tested", not "as good as Acrobat". Don't mark a feature shipped on a generic test, and say in its notes what is still missing. -2. Read that task in `plan/execution-plan.md` §3, the relevant section of `plan/architecture.md`, and the README of the crate you're touching. -3. Follow the **autonomous operation protocol** in `plan/execution-plan.md` §7 (orient → plan → implement + test → verify → record → commit). Don't stop to ask unless §7 lists the decision as the user's. - -`plan/` is gitignored (local-only, like PhotoCraft). The machine-readable parity checklist lives in `parity/` (committed). - -## Non-negotiables -- **Assets: read `AGENTS.md` §1 before adding or showing any icon, image, font or document.** No assets from Adobe products, ever. Only openly licensed or contributor-original assets are allowed, each with an entry in `ATTRIBUTION.toml`. `cargo xtask assets` enforces this. `AGENTS.md` overrides this file. -- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts).** Never commit font files here (`AGENTS.md` §1.4). It is the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts && CRAFT_FONTS_DIR=../craft-fonts cargo test --workspace` embeds the Japanese fonts (UI fallback, Japanese text in edited PDFs) and runs their tests, which otherwise skip. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Clean-room.** - - Never read, disassemble or copy anything inside the Acrobat bundle (names and listings only). **Never open `Contents/Resources/JavaScripts/`.** - - Behaviour comes from public docs, specs (ISO 32000-2, the Arlington model) and black-box observation (`plan/acrobat/`). - - Never copy GPL/AGPL code. MuPDF, Ghostscript, Poppler, veraPDF and DSS run only as external oracle processes. - - See `plan/README.md` §Clean-room and `plan/adr/0001`. -- **Privacy.** When observing Acrobat, use synthetic fixtures only. Never capture the Home view, recent files or account info. Capture by window id (`plan/acrobat/tools/`). Never commit Acrobat outputs, corpus files or personal data. -- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd (`plan/architecture.md` §3). `cos`/`filters`/`crypt`/`arlington` stay standalone. -- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state (`hover_pos`, `interact_pos`, `latest_pos`, `press_origin`, button reads); `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. -- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim(Layer::…, icon)`, `crates/ui-egui/src/canvas/page_cursor.rs`), never set it (`set_cursor_icon`, `cursor_icon =`, `on_hover_cursor`); `document_area` sets it once, from the claim on the highest layer the active tool reaches. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. -- **Page presses.** A page module decides what of its own is under a point in one hit test, which returns a `Hit` (owner, layer, cursor, tip; `crates/ui-egui/src/canvas/page_router.rs`) and gives its cursor, its tip and its press. It acts on a press only when the press is its own or, for the active tool, nobody's: the page view tells it which (`PagePress`: its own, nobody's, or another page's), and it never tests the press point again or asks another module whether it wants it. Escape cancels any press the pages hold (`cancel_on_escape`), and that press is ignored until the button is up. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. -- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. -- **Never panic.** Every PDF, script, MCP call, settings file and keystroke is untrusted input, and none of it may crash the app or lose the user's work. This outranks feature work. Return `Result` (the crate's error enum) and propagate with `?`, or fall back leniently and record it. In non-test code: - - No `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` unless it is provably infallible, with a comment saying why. - - No indexing or slicing with input-derived positions (use `get`, and slice strings only at char boundaries). - - Use checked or saturating arithmetic on input-derived numbers, and guard against division by zero and NaN/inf casts. - - Cap allocations sized by input, and bound recursion with depth limits or seen-sets. - - Handle lock poisoning. - - No `unsafe` (`unsafe_code = "forbid"`). - - Every crash fix gets a synthetic regression test. See `AGENTS.md` §4. -- **Rust only** in the product and build (`xtask`). No handwritten JS/TS. -- **Time goes through `chrono`** (workspace dependency). Never hand-roll UTC/calendar conversion (days-to-date algorithms, leap-year math); a crate owns only its output format string. The one exception is range extension: past the years chrono can represent, shift the date by whole 400-year Gregorian cycles (146,097 days, after which the calendar repeats), let chrono compute the month and day, and add the cycles back to the year (splitting seconds into hours, minutes and seconds is plain arithmetic). -- **Review every update as you make it, at four zooms.** Not an end-of-task pass — a per-update habit, so a defect never gets built on. **Clause:** checked arithmetic, `get()` not `[i]`, char-boundary slicing, the right error through `?`, `<` against `<=`, the 1-based/0-based seam, capped allocations. **Function:** is it total — name an input that panics; walk empty, zero, max, malformed, cyclic; is the return type honest. **File/module:** two sources of truth, dead or duplicated code, is the public surface still smallest and the doc comment still true. **Project:** layering, wasm, undo/redo, save fidelity, encryption and permissions, a matching `automation` tool or registry entry, and whether a `parity/` status or note just became untrue. Scale the zoom to the change, and fix what a level turns up before widening. - - Tests mirror the zooms: property → unit → crate integration → end-to-end. - - **Prove a test fails before the fix by actually reverting the fix.** A test nobody has seen fail is not evidence. This has caught two worthless tests in this repo: one confinement test passed with path resolution switched off entirely, because the tools failed on a nonexistent document id before reaching the path. - - **Rebasing is a review step, not a mechanical one.** Read every hunk you resolve, and the auto-merged hunks in the files you touched too — `main` moves fast enough that a file can change by hundreds of lines under a branch. -- **External PRs are held to safety, security and completeness before style.** Provenance first (no Adobe-derived asset, no committed font, no copied GPL/AGPL code, behaviour from the spec and not from reading another implementation), then no new panic path or `unsafe`, then `--root` confinement on every path including ones reached by nested dispatch, opt-in-only MCP and control channel, no new outbound network, then a headless `automation` tool, tests at the right zoom and an honest `parity/` entry. Provenance is the part a contributor cannot be expected to know, and the only part that cannot be undone after merge. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check once `xtask ci` exists (M0). - - New workspace crates must include `[lints] workspace = true` in `Cargo.toml` to inherit `[workspace.lints]`. Production crate roots must declare the required crate-level lint attributes before modules or other crate items; crate-level `//!` documentation may precede them. - - **Scope builds and tests to the crates you touched** — `cargo clippy -p -j 2 --all-targets` and `cargo test -p -j 2`, under a `timeout`. A full `--workspace` build has locked up a development machine's CPU; it is CI's job and CI runs it on every PR. Run the workspace sweep locally only when you have a reason to and the machine is free. -- **Check for existing work before starting a fix or feature.** Search the issues and PRs, open and closed, with `gh issue list -R storytold/pdfcraft --state all --search ""` and the same with `gh pr list` (try a few phrasings), and check whether a merged fix is already on `main` (`git merge-base --is-ancestor origin/main`). If one is, the report is likely from an older build: say so rather than fixing it again. If an open PR covers it, review or build on that PR instead of opening a competing one. If an issue exists, reference it in the PR (`Fixes #123`). -- **Commits and PRs:** `main` is the only integration branch and is **never pushed to directly**. Branch from an up-to-date `main` as `/` (`feat/`, `fix/`, `docs/`, `chore/`, `refactor/`), one task id per commit (e.g. `M1.4: xref stream reader`), commit only green states, and open a PR into `main` using `.github/pull_request_template.md`. Merge only when CI is green. - - **`main` moves fast enough that PRs rot.** It took 396 commits in the three days to 2026-10-12, and most open PRs are conflicting at any given time. Rebase your own branch the same day `main` moves under it, and ask the author to rebase theirs — never rewrite a contributor's branch. Conflict risk tracks how *contested* the files are, not how large the change is: tooling, tests and docs are nearly free, while `crates/fonts`, `ui-egui` and the i18n catalogues are where several people collide at once. - -## Running and looking at the app -- `cargo run -p pdfcraft -- ` opens the desktop app. -- `cargo run -p pdfcraft-cli -- run --script steps.json --root DIR` drives the engine headlessly through the automation tools (`pdfcraft-cli tools` lists them). Use it, together with `page_render`, to check engine changes. `pdfcraft-cli mcp` is the opt-in MCP server (AGENTS.md §3). -- `cargo xtask fuzz --time 300` mutation-fuzzes open/render/edit/save in child processes. Findings land in `fuzz-out/findings/` (git-ignored; never commit corpus-derived files). Turn every real finding into a small synthetic regression test before fixing it. -- `cargo xtask parity [--partial]` reports Acrobat-parity progress from `parity/acrobat-features.toml`. Update the entry when a feature ships. -- `cargo xtask demo-pdf` builds `dist/demo/pdfcraft-showcase.pdf` (needs Chrome) for visual checks. -- For UI work, **look at the result**. Either launch `pdfcraft --control FILE doc.pdf` and use `pdfcraft-cli ui --control FILE screenshot --out x.png` (plus `inspect`, `click`, `key`, `type`, `command`, `set`), or take a headless shot with `cargo run -p pdfcraft-ui-egui --example shot`. Compare against `plan/acrobat/02-ui-ux.md`. Control-channel tests use kittest (`crates/ui-egui/tests/control.rs`). -- Parallel agents: use a separate `CARGO_TARGET_DIR` per agent and separate git worktrees. - -## Current bootstrap debt (tracked in STATUS.md) -- `pdfcraft-render` renders through the `hayro` crate directly and inspects documents through `lopdf`. Both get replaced by `cos` / `model` / the DisplayList device (M1–M2, ADR-0004). - -## CI workflows -- Workflows in `.github/workflows/` call the shared workflows in [`storytold/craft-common-actions`](https://github.com/storytold/craft-common-actions) at `@main`, with their inputs and secrets. No other jobs: if no shared workflow does what's needed, it doesn't go here. -- Don't add or change CI in a feature or fix PR. If the existing workflows can't build or test a change, link an existing craft-common-actions issue or PR, or open a new issue, or a PR only for a small fix that is safe for every app and does not increase CI time. -- Some workflow files here still run their own jobs. Don't add to them. Moving a job into craft-common-actions is its own task, never part of another PR, and the job stays here until a shared workflow runs it. +@AGENTS.md diff --git a/Cargo.lock b/Cargo.lock index 4df93ed3fe..6710dddeea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -1303,6 +1312,26 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-web-api" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +dependencies = [ + "js-sys", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + +[[package]] +name = "craft-web-host" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "craft-winprint-gdi" version = "0.1.0" @@ -4277,6 +4306,8 @@ dependencies = [ "pdfcraft-model", "pdfcraft-organize", "pdfcraft-platform", + "pdfcraft-prc", + "pdfcraft-prc-render", "pdfcraft-render", "pdfcraft-xfa", "png 0.18.1", @@ -4322,6 +4353,7 @@ version = "0.18.1" dependencies = [ "chrono", "hayro-syntax", + "js-sys", "pdfcraft-crypt", "pdfcraft-filters", "pdfcraft-oracle", @@ -4329,6 +4361,7 @@ dependencies = [ "proptest", "serde_json", "thiserror 2.0.21", + "wasm-bindgen", ] [[package]] @@ -4505,6 +4538,7 @@ version = "0.18.1" dependencies = [ "ocrs", "pdfcraft-fonts", + "png 0.18.1", "rten", "tempfile", "thiserror 2.0.21", @@ -4551,6 +4585,24 @@ dependencies = [ "zbus", ] +[[package]] +name = "pdfcraft-prc" +version = "0.18.1" +dependencies = [ + "flate2", + "proptest", + "thiserror 2.0.21", +] + +[[package]] +name = "pdfcraft-prc-render" +version = "0.18.1" +dependencies = [ + "flate2", + "pdfcraft-prc", + "thiserror 2.0.21", +] + [[package]] name = "pdfcraft-preflight" version = "0.18.1" @@ -4578,6 +4630,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4586,8 +4639,10 @@ dependencies = [ "pdfcraft-fonts", "pdfcraft-forms", "pdfcraft-model", + "pdfcraft-render", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] @@ -4632,9 +4687,11 @@ dependencies = [ "p521", "pbkdf2", "pdfcraft-annot", + "pdfcraft-content", "pdfcraft-cos", "pdfcraft-fonts", "pdfcraft-model", + "pdfcraft-testkit", "rc2", "ripemd", "rsa", @@ -4682,6 +4739,7 @@ dependencies = [ "pdfcraft-engine", "pdfcraft-fonts", "pdfcraft-model", + "pdfcraft-ocr", "pdfcraft-platform", "pdfcraft-render", "pdfcraft-xfa", @@ -4709,10 +4767,13 @@ dependencies = [ name = "pdfcraft-web" version = "0.18.1" dependencies = [ + "craft-web-api", + "craft-web-host", "eframe", "egui", "js-sys", "pdfcraft-ui-egui", + "serde_json", "wasm-bindgen-futures", "web-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 4b98f47c28..bf6a4e3146 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,8 @@ pdfcraft-engine = { path = "crates/engine", version = "0.18.1" } pdfcraft-platform = { path = "crates/platform", version = "0.18.1" } pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.18.1" } pdfcraft-automation = { path = "crates/automation", version = "0.18.1", default-features = false } +pdfcraft-prc = { path = "crates/prc", version = "0.18.1" } +pdfcraft-prc-render = { path = "crates/prc-render", version = "0.18.1" } # pdf hayro = "0.7.1" diff --git a/LICENSES.toml b/LICENSES.toml index d9ceb3180e..0617ca5d57 100644 --- a/LICENSES.toml +++ b/LICENSES.toml @@ -88,6 +88,13 @@ destination = "licenses/egui-emoji-icon-font-mit-license.txt" sha256 = "b9d2c1d909aa149996fd4c91dcb92b2362a04431640c1d200959da94caf8cde1" origin = "https://raw.githubusercontent.com/emilk/egui/0.36.2/crates/epaint_default_fonts/fonts/emoji-icon-font-mit-license.txt" +[[document]] +source = "licenses/prc-rs-LICENSE.txt" +licences = ["MIT"] +destination = "licenses/prc-rs-LICENSE.txt" +sha256 = "ddb846513d92153d8fa16b18ed3e7c86953a582f0d2f49f02b29f1bd569dac58" +origin = "https://raw.githubusercontent.com/ralovich/prc-rs/287b671bb5b31b05efb5ec1b98f7c359b9e4dd7d/LICENSE" + [[document]] source = "vendor/hayro-interpret/LICENSE-APACHE" licences = ["Apache-2.0"] diff --git a/README.md b/README.md index 3c1b67936a..f488d160cc 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Real-world typography: world scripts, vertical Japanese, colour emoji, gradients ### Fearless -Every save appends your changes and leaves the original bytes untouched. Writes are atomic, undo runs deep, and nothing is lost if you close by mistake. +Ordinary saves append your changes and leave the original bytes untouched. Edits to damaged objects whose xref and header generations disagree use a full rewrite, keeping both accepted references connected to the same repaired object. Writes are atomic, undo runs deep, and nothing is lost if you close by mistake. @@ -253,6 +253,18 @@ Documents restricted by their author show a clear notice, and PdfCraft honours t +## Scan pages, then read the words back before trusting them + +**Scan & OCR ▸ Recognize text** turns a scan into searchable text — the current page, a range, or a folder of files written as searchable copies — with optional cleanup before reading (auto-rotate, deskew, denoise, binarize; word boxes map back to the original scan) and a second engine: an installed Tesseract joins the built-in one, merged per your strategy. A searchable result applies as one undoable step; the **Editable text** output opens the **OCR Verify screen** instead, where each word carries the engine's confidence as a colour, suspects are one keypress away, and corrections, a user dictionary and Re-OCR of a region or a page happen before Accept writes the new document. **Scan & OCR ▸ Correct recognized text** reviews any recognition there before it is applied. + +

+ The OCR Verify screen: a scanned note with each recognized word boxed in its confidence colour, the info strip reporting a 95% average confidence and one low-confidence word, and the confidence legend with its thresholds below +
+ The OCR Verify screen: words coloured by confidence — the legend spells the thresholds out — reviewed and corrected before anything is applied. +

+ +--- + ## Every tool, one keystroke away Press ⌘K to search every tool and command, or browse the **All tools** catalogue. Tools that are still in development are marked with the milestone that will ship them. diff --git a/ROADMAP.md b/ROADMAP.md index 6675b1edd8..3f4d570b95 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -21,10 +21,17 @@ > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M2, PR #4172: extend on-screen CJK fallback to Combine thumbnails and figure previews; recognise FangSong and subset-prefixed Founder names; estimates unchanged) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (typed small Edit font sizes and Fill & Sign text/date colour, #2444/#2496; estimates unchanged) +> **Last reviewed:** 2026-10-20 · **Last updated:** 2026-10-20 · **Change:** patch (M1, #1143: repaired aliases survive preflight, snapshots and normalized saves; retained materialized revisions report generation disagreements; estimates unchanged) + +> **Last reviewed:** 2026-10-18 · **Last updated:** 2026-10-18 · **Change:** minor (M1, #1143: explicit live xref/header generation repair policy and bounded diagnostics; estimates unchanged) + +> **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M1: refresh object-generation reference identity against current upstream; estimates unchanged) **Stage: alpha** · next: beta, ~25% (ready for real work 50% → 75%) and ~400–700 h away -> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) +> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M2: scans keep their strokes at screen size and are smoothed when zoomed; M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) +> **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) +> **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (web: same-document host mode on the canvas, `data-host="page"`; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M11: image optimization accounts for page-local UserUnit; estimates unchanged) @@ -223,14 +230,21 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged. +- **2026-11-03 (M11, shared JBIG2 dictionary):** With Monochrome Images set to JBIG2, Optimize has up to 16 JBIG2 images share one symbol dictionary (`/JBIG2Globals`), the letters found on more than one page stored once, when that is smaller (`pdfcraft_filters::encode_jbig2_shared`). Eight pages of 10 pt text rendered at 300 dpi take 41% less in Helvetica and 42% in Times than coded alone, 10% in Courier; groups of 16 save 45%, of 32 only 47%. Only images redaction can store again without the dictionary share one (image masks and DeviceGray named directly, at most 64 Mi pixels), and redacting one stores the rest of its group so, which the group bounds to 15. + - **2026-11-03 (M5, #5850 review):** Resolve indirect IT when recognizing typewriter contents and Fill & Sign marks. Comment listing and property lookup retain their existing direct-Subtype behavior. A real double-click Edit composer preserves the typewriter’s text as written; ordinary comment edits keep trimming. Regressions cover direct Subtype with indirect intent, save/reopen, raw posted contents and same-index comments on different pages. The existing live-movement test now commits its setup with an outside click rather than Enter. Estimates unchanged. - **2026-11-03 (M5, #5850):** Fill & Sign Enter inserts a line break in a growing editor; clicking outside commits and Escape cancels. Nonempty text keeps leading, internal and trailing line breaks. The headless fill_sign_add path sizes its typewriter box for the same wrapped lines so saved appearances show every row. Production-order keyboard and engine save/reopen/render regressions exercise the UI and headless callers. Estimates unchanged. - **2026-11-03 (M7, #5835):** In image crop mode, a right-click on the image, its crop box or its handles keeps the crop and opens no menu, so the menu no longer switches as the pointer moves. A right-click elsewhere clears the selection and opens the page menu. UI tests cover all four page rotations, overlapping images, a 45° image, and a check that the Done/Cancel/Remove Crop pills at a page edge still keep the crop. +- **2026-11-02 (M12, #1297):** `model_add` embeds a U3D or PRC file, recognised by its header, as a 3D annotation with a chosen activation and a poster (an image or PDF page fitted in the box, or a labelled frame); an older file's catalog `/Version` is raised to what the model needs. Views, a rendered poster and a UI command are later steps. Estimates unchanged. + - **2026-11-02 (M10, #5743):** macOS print jobs name their title and owner (`com.apple.print.JobInfo.PMJobName`, `PMJobOwner`), as the system print dialog does, because a Kyocera filter crashes on a job without them. Checked by a test and through `doc_print` against a stand-in `lp`; not yet on a physical printer. +- **2026-11-02 (M14, #5737, #5460):** Preferences can turn off the update check at start. It stays on by default, and Help ▸ Check for updates still asks. + - **2026-11-01 (M7, #5657):** Header/footer, watermark and background additions and replacements stage the complete call before committing. Synthetic later-page failures preserve exact incremental bytes, object count, dirty state and allocation cursor; success controls retain other marks and shared/inherited resources. Estimates unchanged. - **2026-11-01 (M14, #5572):** Preferences ▸ Documents and view sets how many recent files to keep, from 0 to 50 with 12 by default. At 0 Home has no Recent section and File ▸ Open Recent is disabled. The list is cut when Preferences closes, so the number can pass through 0 on its way to another. @@ -257,8 +271,10 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-26 (M2, PR #4172):** Reconciled current main, preserving its Japanese serif/weight selection and both branches' progress records. Installed CJK viewer and preview fallback remains display-only; estimates unchanged. - **2026-10-29 (M7, #2517):** Widening a paragraph rejoins a wrap with the number of spaces that wrap consumed, including a typed run and none for a Chinese or Japanese word split. The count survives an incremental save and a full rewrite, and a width-only `text_edit` uses it. Estimates unchanged. +- **2026-10-29 (M2):** Scanned pages look like the paper again. An image that doesn't ask for /Interpolate was shrunk by nearest-neighbour (since #635), which kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes; it is now area-averaged to whole device pixels (on a 13-page 300 dpi scan every sample at 96–192 dpi is within 1 of the exact area average; it was up to 255 off). Enlarged, an image of page-scan size is smoothed bilinearly, so a zoomed scan has no staircases; icons, QR codes and pixel art keep hard pixels. Estimates unchanged. - **2026-10-26 (M11, JBIG2 symbols):** Lossless JBIG2 now stores each distinct mark of a black-and-white image once, as a symbol, and places its exact copies, when that is smaller than the generic region: on six pages of rendered 300 dpi text 28% of the generic region's size and 17% of Group 4's, and the same bits back through two independent decoders (hayro-jbig2 and Poppler). Scans, whose letters are seldom identical, gain little until refinement coding; a dictionary shared between pages (`/JBIG2Globals`) is next. Estimates unchanged. - **2026-10-25 (M5, comment replies):** One reply can be deleted from the Comments panel without its thread. On the selected card each reply has its own "…" menu (Edit, Copy text, Delete); Delete takes that reply with its pop-up and the replies to it, as `comment_delete` already did headlessly, and the comment and its other replies stay. The card stays selected, and the panel's editor stays open, on what they were on though the delete moved it in `/Annots`. Undo and Redo close the editor, since either can move what it was open on. A locked reply can't be deleted. Estimates unchanged. +- **2026-10-25 (web, same-document host mode):** A page that loads the PdfCraft web module into its own document (no iframe) can opt into the host bridge with `data-host="page"` on the app's canvas — the exact value `page`; any other value keeps the normal app with downloads, unless a same-origin framed `?host=parent` selects the parent bridge (unframed or cross-origin, `?host=parent` also keeps the normal app). It is the `?host=parent` bridge with the transport swapped: the same five messages travel as non-bubbling `CustomEvent`s on the canvas (`detail` holds the parent messages' fields without `type`; the host answers with a `pdfcraft:open` event), so saves reach the host instead of downloading and dirty and start-up-failure reporting are unchanged. Unit tests cover the opt-in rule; no parity or estimate changes. - **2026-10-24 (M10, #4158 partial):** Word, HTML, RTF and PowerPoint export keep a block that continues another on its line (an italic case name, a link, a superscript) as a run in its own style inside that paragraph, with the source's word space, instead of a paragraph of its own. Style changes that wrap onto the next line still split paragraphs (the reading blocks keep style boundaries). Estimates unchanged. - **2026-10-24 (M3, window move on Wayland, #3970 follow-up):** After a move or resize by PdfCraft's own title bar or edges, the button release given back to egui goes before the pointer leaving the window, so hover no longer sticks at the old place until the pointer moves. Found by CodeRabbit in review of #3970; a unit test covers it. Estimates unchanged. - **2026-10-24 (M14, #4253):** The annotation quick bar remembers Keep tool selected across restarts, including unpinning. Old and malformed settings keep safe defaults; the opt-in UI control channel reads and changes the same preference. Estimates unchanged. @@ -287,6 +303,11 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md Estimates unchanged. - **2026-10-21 (M3, window resize and move on Wayland):** With PdfCraft's own title bar on GNOME/Wayland, a second resize by a window edge, or a second move by the empty tab strip, works straight after the first; it waited for a click inside the window, because the compositor keeps the button's release from egui. A unit test covers the release given back; checked by hand in a test build. Estimates unchanged. - **2026-10-24 (M5, move preview):** A comment being moved is drawn where it goes while it is dragged (a rectangle or oval showed only its blue frame): the comment is rendered alone once as the drag starts and laid over the page, which shows it where it was until the move is applied. Engine `Document::comment_layer`; synthetic engine and UI regressions. Estimates unchanged. +- **2026-10-24 (M5, border shape):** Text boxes and Fill & Sign's typed text take an oval or a circle for their border (Comment Properties ▸ Border shape, Fill & Sign's toolbar): the text box keeps its place and the shape goes around it, the text centred, a circle snug around the text (its box no wider than the text needs); stored as /RD and /PCBorderShape and drawn in the appearance. comment_add / comment_edit / fill_sign_add `border_shape`. One new label in every catalog. Synthetic annot, UI and automation regressions. Estimates unchanged. +- **2026-10-24 (M5, Escape in Fill & Sign):** Escape leaves a Fill & Sign tool as it leaves a comment tool: the first press closes the open type box, the next goes back to Select, so the Text tool no longer opens a box at every click with no way out by the keyboard. A synthetic UI regression covers the Text and Checkmark tools. Estimates unchanged. +- **2026-10-23 (M5, ortho):** Shift while drawing holds a line or arrow level or plumb, whichever is nearer (a CAD drawing's ortho mode), and each new side of connected lines, a polygon or a cloud from the corner before; dragging an end or corner with Shift is held the same way (it turned to 45° steps). `measure::snap::ortho`; `comment_add ortho`. Estimates unchanged. +- **2026-10-24 (M3, arrow keys on a wide page):** → / ← scroll a page wider than the window (a plan zoomed in) sideways a line at a time and turn the page at its edge, as ↓ / ↑ do in single-page view; ⌘→ / ⌘← turn it at once, and a page that fits across turns as before (#185). A synthetic viewer regression covers both. Estimates unchanged. +- **2026-10-24 (M5, part opacity):** Comment Properties sets a text box's text, border and fill, and a shape's line and fill, each at an opacity of its own (a see-through fill with an opaque line on a plan); stored as /CA, /ca (PDF 2.0) and /PCTextOpacity and drawn with separate graphics states, so other viewers show it. Make Properties Default and the settings carry it; comment_add / comment_edit `fill_opacity`, `text_opacity`. Four new labels in every catalog. Synthetic annot, UI and automation regressions. Estimates unchanged. - **2026-10-22 (M4, #3916 partial):** Document edits and Undo/Redo cancel an Organize page drag. Pointer-release regressions check exact page order and save/reopen. - **2026-10-22 (M10, #3752 paper):** CUPS jobs ask for the chosen paper in every size. The 13 sizes the Print dialog sent no paper for (A6, A0–A2, ARCH, ANSI C–E) use PWG 5101.1 names, checked against a synthetic PPD with `cupsfilter`, and `doc_print` to a printer sends the paper too. The macOS tray half (Auto Select instead of the PPD's default tray) is a craft-print change that needs a pin bump here. Estimates unchanged. @@ -294,6 +315,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-22 (M14, macOS association recovery candidate):** Successful desktop open/save/sign can queue recovery of an existing per-file binding to the running PDFCraft bundle through NSWorkspace. Metadata-only watcher changes retain undo and avoid false overwrite prompts; real same-length/same-mtime edits remain protected. Rebased on 0.12.0 and reuses #3042's shared atomic metadata preservation; the bounded recovery parser does not narrow save limits. Prior native Preview TEST-signature and Finder workflows passed on macOS 27.0.1; other versions remain unqualified. Estimates unchanged. - **2026-10-13 (M7, free image rotation):** A selected page image has a rotate knob (Shift: 15 degree steps) and `image_edit` a `rotate_by` action; the angle is written as one undoable `cm`. Snapping to page edges while moving is in #1740; resize snapping is not done. Estimates unchanged. +- **2026-10-20 (M9 #79 review, M2, M8):** text extraction reads a marked-content /ActualText that marks no glyph where it paints (paths and images, cut to what is visible, along the direction they are drawn), so words drawn as outlines can be found, copied and exported; OCR and scan field detection still count such a page as having no text of its own. Redaction strips /ActualText, /Alt, /E and /T from inline marked content around paint it clips, as it already did around paint it removed, so a partly redacted outlined word leaves no name in the file. Named-layer recognition also needs the layer form to draw n1 and n3. Estimates unchanged. - **2026-10-20 (M4, initial view):** Window and interface options read indirect boolean ViewerPreferences values. Synthetic regressions cover true/false, direct/indirect preference dictionaries and save/reopen; malformed entries retain false defaults. Estimates unchanged. - **2026-10-20 (M3, #3535):** Escape with a menu open closes the menu and keeps the text selection; the next Escape clears it, as before. One keypress used to do both. A synthetic UI regression checks it. Estimates unchanged. - **2026-10-16 (M3, close prompt keyboard):** The "Save changes?" prompt opens with Save chosen (Enter saves, as before) and the arrow keys and Tab / Shift+Tab move the choice to Cancel or Don't save, shown with a focus ring; Enter or Space press the chosen button and Escape still cancels. Before, Enter always saved whatever was Tab-focused. Estimates unchanged. @@ -308,6 +330,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M11, #3507):** Image merging keeps trailing comments, split operands and graphics state across content streams. Synthetic tool tests check saved text, rendered regions and exact image rows. - **2026-10-17 (M8, gap 6, #2662):** A filled cover over a long line counts when it hides half of any one word, not only 40% of the line's words or box, so a black box over one number in a long line — or an unapplied redaction mark over it — no longer audits clean while the number is still extractable; a sliver under half of every word it touches and under 40% of the line is still not flagged +- **2026-10-13 (M10, OCR review, batch and two engines):** Scan & OCR gains a review step, a second engine and Unicode output. Recognition can clean the raster first (opt-in auto-rotate, deskew, denoise, binarize; word boxes map back to the scan) and read with the built-in ocrs engine, an installed tesseract program run as a fenced external process (a validated location that fails closed instead of falling back to the PATH, allow-listed languages, a private temp directory, timeout and output caps), or both: Automatic asks tesseract when the ocrs reading is weak or of unknown confidence, and a ROVER merge keeps the more confident word (an unscored ocrs word yields only to a tesseract word of 70 or more). The OCR Verify screen (Scan & OCR ▸ Correct recognized text) reviews a recognition before it is applied: confidence-coloured word boxes, corrections, a per-language user dictionary (a plain-text file in the preferences folder, so a word added from any document stays on disk after it closes), Re-OCR of a page or a region, Accept/Reject per page or whole. Pages accepted one by one each apply once; a region re-read moves the review marks of the words it keeps, and a whole-page re-read clears that page's marks (undoable in the panel). Output is searchable (an invisible layer, one undo step) or a NEW editable document of visible text. Words beyond WinAnsi are drawn in Type3 faces from craft-fonts with a ToUnicode map in the editable output (characters no bundled face has appear as '?'); the searchable layer writes them as Type3 fonts with empty character programs and a ToUnicode map, so extraction reads them back with no fonts and no substitutions; redaction removes that layer like any other text (tests in pdfcraft-redact; the end-to-end engine test needs the craft-fonts build input and is skipped without it). Batch runs skip pages that already have text unless forced, and skip whole files that have text only with the opt-in skip-files flag (off by default; Force OCR overrides both); they refuse two inputs of the same file name and existing outputs before reading anything, and report low-confidence pages at the settings' threshold. Headless: ocr_recognize (engine, strategy, preprocessing, region; the editable output goes to an `out` that must not exist yet, and `out` is refused for a searchable run), ocr_words, ocr_review_apply, ocr_dictionary, ocr_status and ocr_recognize_files. All 21 interface catalogs carry the new strings; the Lithuanian ones were added without native review. Reading with every cleaning step off passes the render through untouched, as on main (the 300 dpi normalization runs only when a cleaning step is on: enlarging a 72 dpi render 4x on its own made the models read "due: €250" as "ue: ?250"). Still missing: the deep scan clean-up options (descreen, background and edge-shadow removal), scan images in the editable output, matching the scanned typeface, and recognition quality beyond the tested English set. - **2026-10-13 (M10, #1757):** Office export keeps each dotted-leader contents entry in one paragraph, with its page number, instead of inferring table cells from its separate text objects. - **2026-10-22 (M6, calculation scripts):** The Calculate tab and a rename read simplified field notation only from a comment, where Acrobat writes it: a string that spells `BVCALC … EVCALC` is no longer taken for the notation, or rewritten by a rename. A `/` after the condition of `if`, `while`, `for` or `with` starts a regular expression, so a rename no longer rewrites a name inside one, and a quote inside one no longer hides the `getField` calls after it. Both found in review of #3550. A `//` comment ends at `\r`, U+2028 or U+2029 as well as `\n`, so in a script with `\r` line endings the code after one is no longer hidden from a rename, or taken for the notation. Estimates unchanged. - **2026-10-20 (M6, calculation order page):** Prepare a form ▸ Calculation order, a page of its own, lists the calculated fields in the order they calculate with their formulas and values, moved with Up/Down arrows (as Acrobat's Set Field Calculation Order), Alt+↑/↓ or by dragging. A card says what is wrong with the order and holds Fix order; a loop has a card of its own that opens one of its formulas. On the page, each calculated field shows its step, and arrows show what the field in focus reads and what reads it. Estimates unchanged. @@ -318,6 +341,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M12, #3673):** Character encoding accepts Identity-H/V fonts with Adobe CJK collections and recognizes predefined non-Identity CMaps independently. Unknown CMaps and foreign registries remain refused; synthetic checker and read-only automation regressions passed after failing before the fix. Per-glyph mapping validation and embedded-font extraction remain separate. Estimates unchanged. - **2026-10-19 (M6, renames keep formulas):** Renaming a field renames it in every calculation that reads it, in the same undo step: `AFSimple_Calculate` field lists, simplified field notation (the comment and the JavaScript after it) and `getField` calls naming it in one string literal; the rest of each script is kept as written. A sum used to drop the renamed field, notation read it as 0 and a script kept a stale value. Field Properties says how many formulas changed and `form_set_props` lists them as `formulas_updated`. Names built at run time and other events' scripts are not rewritten. Estimates unchanged. - **2026-10-19 (M6, text field table):** Prepare a form ▸ Text field table drags out an area and adds rows and columns of named text fields as one undoable edit, laid out on the page as displayed, with a live preview, per-column name patterns and widths, and refusal before anything is added when a cell would be too small or a name taken; `form_add_field_table` does the same headlessly. `form.multiple-copies` is partial: copying an existing field into a grid is still missing. Estimates unchanged. +- **2026-10-17 (research):** Added a [PDF workflow research reference](docs/research/pdf-workflows/README.md): dated public-source locators, 20 workflow research domains, a pinned feature mapping and interaction evidence. Reference documentation; no feature statuses, milestone priorities or estimates changed. - **2026-10-16 (M7, #2516):** A paragraph number from `text_paragraphs` keeps naming the paragraph it listed after edits regroup the page: `text_edit`, `text_edit_layout` and `page_render`'s `hide_text_block` find it again by its text and box, and refuse one that was merged into a neighbour (the edited paragraph included) instead of rewriting the paragraph that took its number (`stale_paragraphs` lists the numbers gone). The inline editor's switch-paragraph click holds the same key. No stable identity across save and reopen yet (#2508). - **2026-10-19 (M7, #3591):** A background added to a page that already has one is painted over the old one instead of going under it and staying hidden. Behind the content, backgrounds come first, in the order added, and then behind-content watermarks, so a watermark added after a background no longer disappears. Synthetic regressions in `crates/edit` check the order. Estimates unchanged. @@ -329,9 +353,15 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M3, #3541/#3620):** Comment hover follows the individual marked lines instead of the enclosing multiline rectangle, and overlapping comments follow paint order. Synthetic UI regressions cover all four text-markup kinds, page/view rotations, neighboring highlights, legacy rectangle fallback and unchanged PDF bytes. Estimates unchanged. - **2026-10-20 (M10, #3532 partial):** Vector printing retains existing widget artwork when its appearance lacks Subtype. A print-owned copy adds Form dispatch metadata. The source stream and all explicit subtypes stay unchanged. Synthetic source/output raster checks cover resources, selected/shared appearances, placement, flags and content modes. Appearance regeneration and physical printer validation remain outside this fix. Estimates unchanged. +- **2026-10-20 (M5, several comments):** Comment Properties opened on one of several selected comments, the context menu's Colour and the quick bar's colour change all of them as one undo step, each where the property applies (a font size to text boxes, a fill to shapes, a width to lines and shapes); one that stays locked keeps its look. Copied Fill & Sign ticks, crosses, dots and lines hang on the pointer as themselves rather than as empty boxes, and copied text boxes show their text (`FillMark::strokes` is shared by the appearance and the outline). Estimates unchanged. +- **2026-10-19 (M5, comments between documents):** Comments copy and paste, within and between documents: ⌘C with comments selected copies them as XFDF (each with its replies) to the app's and the system clipboard, ⌘V / Edit ▸ Paste comments / the page menu pastes them onto the current page as new comments under names of their own (`Edit::PasteComments`, `pdfcraft_xfdf::import_comments` with a target page and fresh names, so nothing is replaced and pasting twice gives two sets; `export_xfdf_comments` writes a selection with its replies). Several comments select at once (⌘-click, ⇧-click, on the page and in the panel); Delete removes them as one step and a drag moves them together. Connected lines, polygons and clouds finish from the right-click menu's Complete (Undo last point, Cancel drawing beside it), as other editors do. Tools `comments_copy` / `comments_paste`. Estimates unchanged. - **2026-10-20 (M5, arrow keys move comments):** With a comment selected, the arrow keys move it a screen pixel and ⌘/Ctrl or Shift and an arrow ten, instead of scrolling the view or turning the page; up is up on screen at any zoom and rotation, and each press is one undo step. Text markup and locked comments stay put. Estimates unchanged. +- **2026-10-20 (M8, #3647):** Apply prunes fully removed tiling patterns, then checks retained pattern resources for text, images and unreadable content before reporting success. Shared patterns, Form-owned and inherited patterns, and cells kept by another page cause an atomic refusal when safe removal cannot be shown. Synthetic saved-file and automation regressions cover refusal, preserved marks and undo, and vector-only controls. Estimates unchanged. + +- **2026-10-20 (M1, PR #1143 review):** Repair-aware dictionary/preflight edits and snapshot restoration preserve both accepted generations of a damaged live slot. Such edits use the existing full-save fallback; classic and packed rewrites coalesce aliases so page-tree parent links retain one object identity. Explicit generation replacement remains single-generation. Materialized xref diagnostics inspect retained primary and hybrid rows before folding, retaining the 256-object cap plus summary and discarding abandoned-index notes. Synthetic red/green controls cover preflight, undo restoration, equal values, explicit narrowing, encryption packing and four save variants. Estimates unchanged. + - **2026-10-19 (M14, #3373):** The main menu and its long submenus scroll in short windows, keeping their commands reachable. Estimates unchanged. - **2026-10-19 (M3, #3387):** Save-prompt shortcuts yield to later consent and utility dialogs; a synthetic UI regression checks that an edited document stays open. Estimates unchanged. @@ -376,6 +406,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-16 (M10, #1201 partial):** Lossless image export applies each component's Decode pair. Indexed values round and clip before palette lookup. Color-key masks keep raw samples. The existing 16-bit precision limit remains. JPEG and mask Decode, ICC transforms, and calibrated colors remain outside this fix. Estimates unchanged. - **2026-10-16 (M3, #2739):** Wrapped revision buttons stay above Binding in Document Properties. A synthetic regression checks control bounds and opens a saved revision at two viewport widths. Estimates unchanged. - **2026-10-19 (M7, tracked text):** Editable paragraphs and object labels distinguish repeated glyph tracking from word gaps in TJ arrays and separately positioned glyphs. Synthetic regressions cover character spacing, horizontal scaling, rotated baselines, larger word gaps, and rows of one-glyph words or cells (inferred tracking is capped at half an em, so `a b c` and digit or Y/N columns keep their spaces). Estimates unchanged. +- **2026-10-16 (M9, #79):** a signature appearance that follows the named-layer convention (its /N draws one Form that holds and draws n1 and n3) shows the signature's real state: n1, n3 and n4 are redrawn in the display copy with the panel's icons and a caption in the interface language (Helvetica; else shaped in embedded Inter, or in a craft-fonts face for Arabic, Japanese and Telugu; else English, as Chinese always is). Exports, both print modes and the CLI's render show the redrawn state; saves never do. Every signature keeps its badge, since a document can clip or cover its own layers. Estimates unchanged. +- **2026-10-16 (M9):** the page validation overlay moves to the widget's bottom-right corner as it lies on the page (so it turns with the page and the view) and takes a quarter of the widget's shorter side, at most 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field too small for a legible mark keeps its outline, a rectangle the screen cannot place is not drawn at all, and the whole widget answers the pointer. Estimates unchanged. - **2026-10-16 (M5, #2669, item 5):** The Comments panel groups replies by name and counts all page roots once per frame. Estimates unchanged. - **2026-10-16 (M7, #2140):** Header & footer (and Bates numbering, which uses the same dialog) can shrink the page content so the header and footer don't cover it: the original content is scaled evenly into the band they leave free, centred across, also on rotated pages; Remove and Update take the scaling away, and margins that leave no room are refused. Dialog checkbox with a preview of the band, `shrink` on `doc_header_footer`, all catalogs. Annotations and added text and images are not moved yet. Estimates unchanged. - **2026-10-20 (M14, Spanish catalog gaps):** `es.tsv` gains the 143 `(context, English)` keys the other catalogs carried and it was missing, so Spanish covers the whole bundled key union like Lithuanian and Vietnamese: calendar months and weekdays, the `certificate` context, X.509 key usages, signature-revision and digital-ID lines, password validation, eight plural messages, the combine-from-folder strings, the rendering and stale-file notices and the shortcut-spelled tooltips. Counts in `localization-parity.md` re-measured against the current union (`vi` restated against it for the same reason). Status stays `partial`: no translated help, and OS details stay outside the catalog. Estimates unchanged. @@ -396,6 +428,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-15 (community, M6, #2608):** In Prepare a form the selected field moves with the arrow keys: a point per press, ten with Shift, as the object nudge and dragging it do. Each press is one undo step ("Change field properties"), and the keys no longer turn the page while a field is selected. Estimates unchanged. - **2026-10-15 (M3, closable community card):** The ArtCraft community card on Home has a close button that hides it for the rest of the run; it returns on the next launch, and its links stay in the top bar's Discord button and the Help menu. A UI regression covers closing and the fresh-launch default. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. +- **2026-10-18 (M1, PR #1143 repair policy):** A live in-file reference resolves when its generation matches the xref row or a same-number header at that indexed offset. Neither-match and free slots remain null, including when old object bytes remain. Opening records bounded xref/header disagreements without parsing object bodies or expanding compact xref rows. Synthetic controls cover read order, clones, overlays, dictionary edits, full classic/packed saves and an independent reader of incremental output. **Correction to the 2026-10-15 note below:** its async-opening-test sentence was not evidenced by this COS change; no fresh async-opening or GUI validation is claimed. The full-reference stream-length recursion key is defensive hardening, not an independently isolated regression fix. Estimates unchanged. +- **2026-10-15 (M1, PR #1143 refresh):** COS reference lookups and full saves distinguish complete object references, so stale generations cannot expose replacement content. Snapshot restoration also keeps edits to objects with nonzero generations before incremental saves. Synthetic regressions cover caches, overlays, snapshots, stream lengths and classic/packed saves. Current upstream async-opening test corrections are included. Estimates unchanged. - **2026-10-15 (M14, #2383 candidate):** Device-limit requests now clamp unsupported maxima and minimum alignments before enabling the adapter's full 2D texture extent. This addresses the reported V3D request of 16 shader variables on an adapter supporting 15. Synthetic regressions fail before the clamp and pass afterward, along with the existing HiDPI case and Windows GPU-required application tests. Raspberry Pi startup remains unverified; milestone and overall estimates are unchanged. - **2026-10-12 (M6):** XFDF field import and data-file extraction join all text nodes in a value, preserving text after XML comments and processing instructions. Synthetic regressions cover Unicode, CDATA, empty values and PDF save/reopen. Estimates unchanged. - **2026-10-12 (M7, picture right-click menu):** The Edit text & images menu of a picture gains Cut Image, Redact Image (a redaction mark over the picture, applied from Redact a PDF), Create Link… and Recognize Text… (for that page), in all 21 catalogs. The menu was drawn only while the pointer was over the picture, so its items outside the picture (most of a logo's menu, and Delete below a page) could not be clicked; an open menu now stays drawn. Still missing from Acrobat's menu: Arrange (bring to front, send backward), Remove crop, Draw freehand and Edit image using… (opening an external editor). @@ -763,6 +797,7 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-11-01 | patch | M7 #5657: header/footer, watermark and background operations commit all selected pages together; later-page errors preserve marks, content, shared/inherited resources and pending edits; estimates unchanged. | | 2026-10-28 | patch | M2, PR #4172: integrate main 5af10276 and retain both roadmap histories; viewer-only CJK and print/output font policies and estimates unchanged. | | 2026-10-27 | minor | M10: OCR stages SHA-256-verified ONNX models after the upstream RTEN URL brownout; discovery prefers a complete ONNX pair per directory, falls back to a complete RTEN pair, and never mixes formats or directories; language/web limits and estimates unchanged. | +| 2026-10-20 | minor | M9 #79 review, M2, M8: /ActualText that marks no glyph extracts where it paints, as far as it is visible; redaction strips /ActualText around paint it clips; named-layer recognition needs the layer form to draw n1 and n3. Estimates unchanged. | | 2026-10-23 | minor | M8, #3205 follow-up: phone-pattern redaction includes the +39 country code on labelled Italian geographic numbers with a slash; dates, protocol identifiers and whole-run guards remain. Estimates unchanged. | | 2026-10-23 | minor | M11: image downsampling measures physical resolution after page-local UserUnit scaling, including shared nested forms; synthetic measurement and optimization regressions. Estimates unchanged. | | 2026-10-23 | minor | M10: Word table cells use the grid columns they span, including cells after a merged cell; synthetic export regression and CLI reproduction. Estimates unchanged. | @@ -774,8 +809,11 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-10-20 | minor | M6, #3502: form detection preserves comments across content streams; estimates unchanged. | | 2026-10-20 | minor | M10, #3532 partial: vector printing retains existing appearances with absent Subtype; estimates unchanged. | | 2026-10-20 | minor | M11 #3507: image merging preserves comments and state across content streams. Estimates unchanged. | +| 2026-10-18 | minor | M1, #1143: a live reference may match the xref generation or a same-number header at its indexed offset; neither-match and free slots remain null. Bounded repair diagnostics and synthetic cache, free-slot, edit and classic/packed save regressions; estimates unchanged. | | 2026-10-18 | minor | M5, PR #1971: nested source `/Page` and `/Pages` dictionaries export as null without traversing page content or the page tree. Synthetic saved/reopened three-page regression retains unrelated metadata and aliases. FDF import semantics, copied-graph limits and estimates unchanged. | | 2026-10-15 | minor | M5: budget retained FDF entries before cloning; discovery keeps lazy array owners and annotation positions, with alias/cache-eviction stress fixtures. Limits cover the copied graph; source parsing/page discovery retain existing COS/parser limits, with no total source-memory bound. FDF import semantics and estimates unchanged. | +| 2026-10-16 | minor | M9 #79: named-layer signature appearances show the signature's real state in the display copy, exports and both print modes, never in saves; captions follow the interface language. Estimates unchanged. | +| 2026-10-16 | minor | M9: the page validation overlay moves to the widget's bottom-right corner as it lies on the page and stops growing past 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field under 32 screen points keeps its outline. Estimates unchanged. | | 2026-10-15 | minor | M7: the Edit text caret is as tall as the row it is on in a region with several sizes; estimates unchanged. | | 2026-10-18 | minor | M4, PR #1919 review: repeated page indices apply each rotation to the latest page-local value, preserving indirect/inherited angles. Synthetic adjacent/interleaved duplicates retain siblings and page contents through incremental and full saves. Estimates unchanged. | | 2026-10-18 | minor | M3, #3149: page replacement cancels render demand collected for the previous geometry; all viewer raster upload paths reject empty or oversized textures before cache eviction. Synthetic regressions reproduce the reported 1880×2350 upload at a 2048 limit, retain cached images and unsaved edits, and resume bounded backdrops and tiles. Estimates unchanged. | @@ -784,6 +822,7 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-10-14 | minor | M1: raw-deflate fallback preserves decoding-limit errors in strict and tolerant decoding; synthetic filter and COS stream regressions. Estimates unchanged. | | 2026-10-16 | minor | M14: Czech catalog completed (2,674 entries) and "tag" terminology unified. Estimates unchanged. | | 2026-10-13 | minor | M9: separate page validation overlays preserve signature artwork, and snapshot tabs reuse instead of multiplying; baked captions remain unchanged (#79 open); estimates unchanged | +| 2026-10-15 | minor | M1: refresh object-generation reference identity, retaining current upstream progress and regression tests. Estimates unchanged. | | 2026-10-16 | minor | M10 #1201 partial: lossless image export applies component Decode pairs and Indexed mapping. Estimates unchanged. | | 2026-10-15 | minor | M6: CSV export and spreadsheet merging quote tabs, and import distinguishes quoted tabs from TSV delimiters without changing field identities. Estimates unchanged. | | 2026-10-13 | minor | M4: rotation queries and edits resolve indirect page-tree values; inherited, negative and large rotations verified through both saves. Estimates unchanged. | diff --git a/apps/pdfcraft-cli/src/commands.rs b/apps/pdfcraft-cli/src/commands.rs index 9d0b2057f0..3118c901b4 100644 --- a/apps/pdfcraft-cli/src/commands.rs +++ b/apps/pdfcraft-cli/src/commands.rs @@ -366,8 +366,9 @@ pub(crate) fn render(args: &[String]) -> Result<(), CliError> { Some("pam") => None, _ => return Err(format!("render: --out {out}: use a .png, .jpg, .tif or .pam name").into()), }; - // The app renders a display copy that draws comments missing their /AP streams; the CLI - // must agree with it (and with independent viewers), while keeping the file it reads intact. + // The app renders a display copy that draws comments missing their /AP streams and the state + // of named-layer signatures (here with no trusted certificates); the CLI must agree with it + // (and with independent viewers), while keeping the file it reads intact. let bytes = read(path)?; let password = flag(args, "--password"); // The renderer takes only the user password of R2–R4 files; `Session::open` and the display diff --git a/apps/pdfcraft-cli/tests/it/batch_exit.rs b/apps/pdfcraft-cli/tests/it/batch_exit.rs new file mode 100644 index 0000000000..6dfb9f61e9 --- /dev/null +++ b/apps/pdfcraft-cli/tests/it/batch_exit.rs @@ -0,0 +1,111 @@ +//! The exit codes the CLI has today: `ExitCode::SUCCESS` (0) or `ExitCode::FAILURE` (1). +//! +//! Pinned here so that the exit-code contract is a visible, reviewed change when it comes: a +//! failed file, an outside-root path and a usage error all exit 1 today. + +use serde_json::Value; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const BIN: &str = env!("CARGO_BIN_EXE_pdfcraft-cli"); + +/// A fresh folder for one test, so the tests can run in parallel and leave nothing behind. +fn workdir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("pdfcraft-cli-exit-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// `batch` with `args`, run from the working directory `dir`. +fn batch(dir: &Path, args: &[&str]) -> Output { + Command::new(BIN).current_dir(dir).arg("batch").args(args).output().unwrap() +} + +/// `run --script --report` from the working directory `dir`: one JSON document for the run. +fn run_report(dir: &Path, script: &Path) -> Output { + Command::new(BIN).current_dir(dir).args(["run", "--script"]).arg(script).arg("--report").output().unwrap() +} + +/// The one JSON document a command prints on stdout. +fn reported(output: &Output) -> Value { + serde_json::from_slice(&output.stdout).unwrap() +} + +#[test] +fn batch_exits_0_when_every_file_succeeds() { + let dir = workdir("batch-ok"); + std::fs::write(dir.join("page.txt"), "Exit code page").unwrap(); + let output = batch(&dir, &["page.txt", "--steps", "convert", "--out-dir", "out"]); + assert_eq!(output.status.code(), Some(0), "{}", String::from_utf8_lossy(&output.stderr)); + let report = reported(&output); + assert_eq!(report["completed"], 1, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert!(dir.join("out/page.pdf").exists(), "the converted page is written"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn batch_exits_1_when_one_file_fails() { + let dir = workdir("batch-fail"); + std::fs::write(dir.join("bad.pdf"), "broken").unwrap(); + let output = batch(&dir, &["bad.pdf", "--steps", "convert", "--out-dir", "out"]); + assert_eq!(output.status.code(), Some(1), "{}", String::from_utf8_lossy(&output.stderr)); + let report = reported(&output); + assert_eq!(report["failed"], 1, "{report}"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn batch_usage_error_exits_1_and_creates_nothing() { + let dir = workdir("batch-usage"); + std::fs::write(dir.join("page.txt"), "Exit code page").unwrap(); + // Baseline: a usage error exits 1 today. A later change to the exit-code contract will + // update this assertion on purpose. + let output = batch(&dir, &["page.txt", "--out-dir", "out"]); + assert_eq!(output.status.code(), Some(1), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(output.stdout.is_empty(), "a usage error prints to stderr, not a report on stdout"); + assert!(!output.stderr.is_empty(), "a usage error says what is wrong"); + assert!(!dir.join("out").exists(), "a refused batch creates no output"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn batch_outside_root_exits_1_and_creates_nothing() { + let dir = workdir("batch-outside-root"); + let output = Command::new(BIN) + .current_dir(&dir) + .args(["batch", "../outside.pdf", "--steps", "convert", "--out-dir", "out", "--root"]) + .arg(&dir) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(1), "{}", String::from_utf8_lossy(&output.stderr)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("is outside the allowed directory"), "the input is refused by the root: {stderr}"); + assert!(output.stdout.is_empty(), "a refused batch prints no report"); + assert!(!dir.join("out").exists(), "a refused batch creates no output"); + std::fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn run_script_report_exits_0_when_ok_and_1_when_a_step_fails() { + let dir = workdir("run-report"); + let script = dir.join("steps.json"); + let blank = serde_json::json!({ "tool": "doc_create", "args": { "from": "blank", "width": 100, "height": 100 } }); + let steps = serde_json::json!([blank.clone(), { "tool": "no_such_tool", "args": {} }]); + std::fs::write(&script, steps.to_string()).unwrap(); + + let failing = run_report(&dir, &script); + assert_eq!(failing.status.code(), Some(1), "{}", String::from_utf8_lossy(&failing.stderr)); + let report = reported(&failing); + assert_eq!(report["ok"], false, "{report}"); + assert_eq!(report["steps"][1]["status"], "error", "{report}"); + + std::fs::write(&script, serde_json::json!([blank]).to_string()).unwrap(); + let good = run_report(&dir, &script); + assert_eq!(good.status.code(), Some(0), "{}", String::from_utf8_lossy(&good.stderr)); + let report = reported(&good); + assert_eq!(report["ok"], true, "{report}"); + assert_eq!(report["steps"][0]["status"], "ok", "{report}"); + std::fs::remove_dir_all(&dir).unwrap(); +} diff --git a/apps/pdfcraft-cli/tests/it/main.rs b/apps/pdfcraft-cli/tests/it/main.rs index f0b87c97a7..30a6e25882 100644 --- a/apps/pdfcraft-cli/tests/it/main.rs +++ b/apps/pdfcraft-cli/tests/it/main.rs @@ -3,6 +3,7 @@ mod audit; mod automation_roots; mod batch; +mod batch_exit; mod check; mod edit; mod extract; diff --git a/apps/pdfcraft-web/Cargo.toml b/apps/pdfcraft-web/Cargo.toml index ad8243d594..ad5b62229c 100644 --- a/apps/pdfcraft-web/Cargo.toml +++ b/apps/pdfcraft-web/Cargo.toml @@ -11,10 +11,15 @@ workspace = true [dependencies] pdfcraft-ui-egui = { workspace = true } +# The host messages (storytold/craft-libs): one contract and one bridge for the web builds. +# The page command API (storytold/craft-libs#413): window.pdfcraft over the UI control channel. +craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } +craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } eframe = { workspace = true, features = ["default_fonts", "glow", "persistence", "accesskit"] } egui = { workspace = true, features = ["default_fonts", "monochrome_emoji_fonts"] } +serde_json = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" -web-sys = { version = "0.3", features = ["Window", "Document", "Element", "HtmlCanvasElement", "HtmlElement", "Location", "Url", "UrlSearchParams", "Response", "Event", "MessageEvent", "EventTarget"] } +web-sys = { version = "0.3", features = ["Window", "Document", "Element", "HtmlCanvasElement", "HtmlElement", "Location", "Url", "UrlSearchParams", "Response", "Event", "MessageEvent", "EventTarget", "CustomEvent", "CustomEventInit"] } js-sys = "0.3" diff --git a/apps/pdfcraft-web/src/api.rs b/apps/pdfcraft-web/src/api.rs new file mode 100644 index 0000000000..3f1d265377 --- /dev/null +++ b/apps/pdfcraft-web/src/api.rs @@ -0,0 +1,127 @@ +//! `window.pdfcraft`: the page command API (`craft-web-api`, storytold/craft-libs#413) over the app's +//! UI control channel (`pdfcraft_ui_egui::control`), opt-in with `?control`. +//! +//! The crate owns the JavaScript side (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, +//! `screenshot`, as promises with a timeout). This module is PdfCraft's part: a [`Request`] that +//! sends each call through the app's [`ControlClient`] and waits for the reply, and the table that +//! names PdfCraft's methods: its control channel drives the UI, so `commands` is `ui.commands`, +//! `execute` runs a registry command with `ui.command`, `docInspect` is `ui.state` and `uiInspect` +//! is `ui.inspect`. +//! +//! The channel keeps the accessibility tree of every frame, so it is off unless the page asks for it +//! (`?control`), as FigCraft's is. PdfCraft's control requests cannot be withdrawn, so a call the page +//! gave up on may still run (or be rejected) when the app reaches it, as it does for a desktop client that timed out. + +use serde_json::{Value, json}; + +/// The reply of a control request as the protocol's `{ok, result}` / `{ok: false, error}` object. +fn reply_object(reply: Result) -> Value { + match reply { + Ok(result) => json!({"ok": true, "result": result}), + Err(error) => json!({"ok": false, "error": error}), + } +} + +/// The params of `ui.command`. `execute(command, params)` sends `{command, params}`; `ui.command` +/// takes `{id}` (and `params` for `ui.panel`). A request that already names an `id` is the page's own +/// `request("ui.command", {id})` and goes through unchanged. +fn command_params(params: Value) -> Value { + match params.get("command").and_then(Value::as_str) { + Some(command) if params.get("id").is_none() => json!({"id": command, "params": params.get("params").cloned().unwrap_or(Value::Null)}), + _ => params, + } +} + +#[cfg(target_arch = "wasm32")] +pub use web::install; + +#[cfg(target_arch = "wasm32")] +mod web { + use std::rc::Rc; + use std::sync::mpsc::{Receiver, TryRecvError}; + + use craft_web_api::methods::Methods; + use craft_web_api::page::{PageApi, Request}; + use pdfcraft_ui_egui::control::{ControlClient, Reply}; + use serde_json::{Value, json}; + use wasm_bindgen_futures::JsFuture; + + use super::{command_params, reply_object}; + + /// The app id in `window.`. + const APP: &str = "pdfcraft"; + + /// How often a waiting call looks for the app's reply, in milliseconds (about a frame). + const POLL_MS: i32 = 16; + + /// PdfCraft's control methods behind each function. + fn methods() -> Methods { + Methods { + commands: Some("ui.commands"), + execute: Some("ui.command"), + doc_inspect: Some("ui.state"), + ui_inspect: Some("ui.inspect"), + screenshot: Some("ui.screenshot"), + } + } + + /// Put `window.pdfcraft` on the page, sending its calls through `client`. An existing + /// `window.pdfcraft` keeps the functions it already has. + pub fn install(client: ControlClient) { + let request: Request = Rc::new(move |method, params| { + let params = if method == "ui.command" { command_params(params) } else { params }; + Box::pin(wait(client.send(&method, params))) + }); + if let Err(e) = Rc::new(PageApi::new(APP, methods(), request)).install() { + eframe::web_sys::console::warn_1(&format!("PdfCraft: couldn't install window.{APP}: {e:?}").into()); + } + } + + /// The app's reply, polled about once a frame. + async fn wait(reply: Receiver) -> Value { + loop { + match reply.try_recv() { + Ok(reply) => return reply_object(reply), + Err(TryRecvError::Empty) => sleep(POLL_MS).await, + Err(TryRecvError::Disconnected) => return json!({"ok": false, "error": "the app dropped the request"}), + } + } + } + + async fn sleep(ms: i32) { + let promise = js_sys::Promise::new(&mut |resolve, _| { + if let Some(window) = web_sys::window() { + let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms); + } + }); + let _ = JsFuture::from(promise).await; + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn execute_becomes_a_ui_command_by_id() { + assert_eq!(command_params(json!({"command": "file.word_count", "params": {"a": 1}})), json!({"id": "file.word_count", "params": {"a": 1}})); + assert_eq!(command_params(json!({"command": "file.save", "params": {}})), json!({"id": "file.save", "params": {}})); + } + + #[test] + fn a_request_that_names_an_id_goes_through_unchanged() { + let own = json!({"id": "file.save"}); + assert_eq!(command_params(own.clone()), own); + let both = json!({"id": "ui.panel", "command": "x", "params": {"reset": true}}); + assert_eq!(command_params(both.clone()), both, "an id wins over a command"); + assert_eq!(command_params(json!({})), json!({})); + } + + #[test] + fn replies_become_the_protocols_objects() { + assert_eq!(reply_object(Ok(json!({"n": 1}))), json!({"ok": true, "result": {"n": 1}})); + assert_eq!(reply_object(Err("boom".to_string())), json!({"ok": false, "error": "boom"})); + } +} diff --git a/apps/pdfcraft-web/src/main.rs b/apps/pdfcraft-web/src/main.rs index 73d14f4a6c..d23dd15a57 100644 --- a/apps/pdfcraft-web/src/main.rs +++ b/apps/pdfcraft-web/src/main.rs @@ -3,20 +3,16 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(any(target_arch = "wasm32", test))] +mod api; + #[cfg(target_arch = "wasm32")] fn main() { - use eframe::wasm_bindgen::JsCast; use pdfcraft_ui_egui::PdfCraftApp; let options = eframe::WebOptions { renderer: eframe::Renderer::Glow, ..Default::default() }; wasm_bindgen_futures::spawn_local(async move { - let Some(canvas) = web_sys::window() - .and_then(|w| w.document()) - .and_then(|d| d.get_element_by_id("pdfcraft")) - .and_then(|e| e.dyn_into::().ok()) - else { - return; - }; + let Some(canvas) = app_canvas() else { return }; let started = eframe::WebRunner::new() .start( canvas, @@ -54,11 +50,11 @@ fn main() { // eframe never raises a close request on a browser reload or tab close, so `guard_quit` never // prompts; ask the browser to confirm instead while a document has unsaved work (#812). warn_before_unload(app.unsaved_flag.clone()); - // `?host=parent`: a same-origin page embedding PdfCraft in an iframe (such - // as the Nextcloud app) passes files in and takes saves back (`host` below). - if query_param("host").as_deref() == Some("parent") - && let Some(bridge) = host::Bridge::new() - { + // Host modes (`host` below, over `craft-web-host`): `?host=parent`, a + // same-origin page embedding PdfCraft in an iframe (such as the Nextcloud + // app), or `data-host="page"` on the canvas, a host that runs the module in + // its own document. + if let Some(bridge) = app_canvas().as_ref().and_then(host::Bridge::connect) { // `?author=`: the host's name for the user signs their comments, // unless they chose a name in PdfCraft (the web default is "Guest"). if let Some(author) = query_param("author").map(|a| a.trim().to_string()).filter(|a| !a.is_empty()) @@ -75,15 +71,18 @@ fn main() { &"PdfCraft: ?host=parent needs a framing page on the same origin; saves download instead".into(), ); } + // `?control`: `window.pdfcraft`, the page's command API (`api`). Opt-in: the control + // channel keeps the accessibility tree of every frame. + if query_param("control").is_some() { + api::install(app.attach_control(&cc.egui_ctx)); + } Ok(Box::new(app)) }), ) .await; if let Err(e) = started { eframe::web_sys::console::error_1(&e); - if query_param("host").as_deref() == Some("parent") - && let Some(bridge) = host::Bridge::new() - { + if let Some(bridge) = app_canvas().as_ref().and_then(host::Bridge::connect) { bridge.post_failed(&e.as_string().unwrap_or_else(|| format!("{e:?}"))); } } @@ -123,136 +122,116 @@ async fn fetch_bytes(url: &str) -> Result, String> { Ok(js_sys::Uint8Array::new(&buf).to_vec()) } -/// The `postMessage` bridge to a same-origin parent page. Messages are plain objects with a -/// `type`: +/// The app's canvas: the one element `index.html` and every embedding page must provide, with +/// the id the app looks up. +#[cfg(target_arch = "wasm32")] +fn app_canvas() -> Option { + use eframe::wasm_bindgen::JsCast; + web_sys::window()?.document()?.get_element_by_id("pdfcraft").and_then(|e| e.dyn_into::().ok()) +} + +/// Host modes: how the page embedding PdfCraft passes files in and takes saves back. The messages +/// and the two transports are `craft-web-host`'s (storytold/craft-libs), the same contract as the +/// other Craft apps' bridges, with the `pdfcraft:` prefix. This module is PdfCraft's part of it. /// -/// - `pdfcraft:ready` (to the parent): the app is listening; send files now. -/// - `pdfcraft:open` (from the parent): `{ name, bytes }`, where `bytes` is an `ArrayBuffer` or -/// `Uint8Array` holding a PDF. It opens in a tab, like a `?file=` URL. -/// - `pdfcraft:save` (to the parent): `{ name, saveAs, bytes }` when the user saves; `bytes` is -/// a transferred `ArrayBuffer`. The parent stores the file (and reports its own errors). -/// - `pdfcraft:dirty` (to the parent): `{ dirty }` whenever unsaved work appears or goes away. -/// - `pdfcraft:failed` (to the parent): `{ error }` when the app couldn't start. -/// -/// Only the parent window on the app's own origin is heard or answered. With a parent on -/// another origin there is no bridge: the app runs as if unframed and saves download. +/// - `?host=parent` is the `postMessage` bridge for a same-origin page that embeds PdfCraft in an +/// iframe (such as the Nextcloud app). With a parent on another origin there is no bridge: the +/// app runs as if unframed and saves download (#2167). `data-host="page"` on the app's canvas +/// is for a host that loads the module into its own document, with no iframe: the same messages +/// travel as DOM `CustomEvent`s on that canvas. `?host=parent` wins when both are present. +/// - `pdfcraft:ready` once the app is listening; send files now. PdfCraft reads bytes, so it +/// says `file: false` and a host sends `bytes`. +/// - `pdfcraft:open` (from the host): `{ name, bytes }` holding a PDF. It opens in a tab, like a +/// `?file=` URL; a file over [`host::MAX_OPEN_BYTES`] is refused before it is copied. +/// - `pdfcraft:save` (to the host): `{ name, saveAs, bytes }` when the user saves. The host stores +/// the file (and reports its own errors). +/// - `pdfcraft:dirty` whenever unsaved work appears or goes away, `pdfcraft:failed` when the app +/// couldn't start. #[cfg(target_arch = "wasm32")] mod host { - use eframe::wasm_bindgen::{JsCast, JsValue, closure::Closure}; + use std::rc::Rc; + + use craft_web_host::web::{FileData, Host, Message}; + + /// The app id in the host messages (`pdfcraft:ready`). + const APP: &str = "pdfcraft"; + + /// The name a file gets when the host sent none (the crate says `document` for a blank name). + const DEFAULT_NAME: &str = "document.pdf"; + + /// The largest file the host may hand over to open, checked before it is copied (the app's own readers have their own limits). + const MAX_OPEN_BYTES: u64 = 512 << 20; #[derive(Clone)] pub struct Bridge { - window: web_sys::Window, - parent: web_sys::Window, - origin: String, + host: Rc, } impl Bridge { - /// `None` when the page isn't framed (there is no parent to talk to), or when the parent - /// is on another origin: messages only go to this origin, so the browser would drop - /// every save without an error while the document showed as saved (#2167). - pub fn new() -> Option { - let window = web_sys::window()?; - let parent = window.parent().ok().flatten()?; - if js_sys::Object::is(&parent, &window) { - return None; - } - let origin = window.location().origin().ok()?; - // Reading another origin's location throws, which is `None` too. - if parent.location().origin().ok()? != origin { - return None; - } - Some(Self { window, parent, origin }) - } - - fn post(&self, message: &js_sys::Object, transfer: Option<&JsValue>) -> Result<(), String> { - let sent = match transfer { - Some(t) => self.parent.post_message_with_transfer(message, &self.origin, &js_sys::Array::of1(t)), - None => self.parent.post_message(message, &self.origin), - }; - sent.map_err(|e| e.as_string().unwrap_or_else(|| format!("{e:?}"))) + /// The bridge for this page, or `None` when PdfCraft isn't hosted: no `?host=parent` and + /// no `data-host="page"` on the canvas, an unframed `?host=parent` page, or a parent on + /// another origin (messages only go to this origin, so the browser would drop every save + /// without an error while the document showed as saved, #2167). + pub fn connect(canvas: &web_sys::HtmlCanvasElement) -> Option { + Some(Self { host: Rc::new(Host::connect(APP, canvas)?) }) } pub fn post_ready(&self) { - if let Some(m) = message(&[("type", "pdfcraft:ready".into()), ("version", env!("CARGO_PKG_VERSION").into())]) { - let _ = self.post(&m, None); - } + let _ = self.host.post_ready(env!("CARGO_PKG_VERSION"), false); } - /// The app couldn't start (no WebGL, for example): let the parent say so. + /// The app couldn't start (no WebGL, for example): let the host say so. pub fn post_failed(&self, error: &str) { - if let Some(m) = message(&[("type", "pdfcraft:failed".into()), ("error", error.into())]) { - let _ = self.post(&m, None); - } + let _ = self.host.post_failed(error, None); } - /// Hear `pdfcraft:open` from the parent and queue the file like a `?file=` download. + /// Hear `pdfcraft:open` and queue the file like a `?file=` download. pub fn listen(&self, inbox: pdfcraft_ui_egui::Inbox, ctx: egui::Context) { - let parent = self.parent.clone(); - let origin = self.origin.clone(); - let on_message = Closure::::new(move |event: web_sys::MessageEvent| { - if event.origin() != origin || !event.source().is_some_and(|s| js_sys::Object::is(&s, &parent)) { - return; + self.host.listen(move |message| { + if let Message::Open { name, data } = message { + queue_open(&name, &data, &inbox, &ctx); } - let data = event.data(); - if string(&data, "type").as_deref() != Some("pdfcraft:open") { - return; - } - let name = string(&data, "name").filter(|n| !n.trim().is_empty()).unwrap_or_else(|| "document.pdf".to_string()); - let Ok(bytes) = js_sys::Reflect::get(&data, &JsValue::from_str("bytes")) else { return }; - let bytes = if let Some(array) = bytes.dyn_ref::() { - array.to_vec() - } else if bytes.is_instance_of::() { - js_sys::Uint8Array::new(&bytes).to_vec() - } else { - return; - }; - if let Ok(mut q) = inbox.lock() { - q.push((name, bytes)); - } - ctx.request_repaint(); }); - if self.window.add_event_listener_with_callback("message", on_message.as_ref().unchecked_ref()).is_ok() { - // The listener lives as long as the page. - on_message.forget(); - } } pub fn save_fn(&self) -> pdfcraft_ui_egui::HostSaveFn { - let bridge = self.clone(); - Box::new(move |name: &str, bytes: &[u8], save_as: bool| { - let buffer = js_sys::Uint8Array::from(bytes).buffer(); - let m = - message(&[("type", "pdfcraft:save".into()), ("name", name.into()), ("saveAs", save_as.into()), ("bytes", buffer.clone().into())]) - .ok_or_else(|| "couldn't build the message".to_string())?; - bridge.post(&m, Some(&buffer)) - }) + let host = self.host.clone(); + Box::new(move |name: &str, bytes: &[u8], save_as: bool| host.post_save(name, bytes, None, save_as)) } pub fn dirty_fn(&self) -> pdfcraft_ui_egui::HostDirtyFn { - let bridge = self.clone(); + let host = self.host.clone(); + // A host starts out clean: the first report is a change only when it is dirty. let mut last = false; Box::new(move |dirty: bool| { if dirty != last { last = dirty; - if let Some(m) = message(&[("type", "pdfcraft:dirty".into()), ("dirty", dirty.into())]) { - let _ = bridge.post(&m, None); - } + host.post_dirty(dirty); } }) } } - fn message(fields: &[(&str, JsValue)]) -> Option { - let m = js_sys::Object::new(); - for (key, value) in fields { - js_sys::Reflect::set(&m, &JsValue::from_str(key), value).ok()?; + /// The `pdfcraft:open` payload, `{ name, bytes }` where `bytes` holds a PDF: queue it like a + /// `?file=` download and wake the app. The crate has cleaned the name (and cut it to 255 + /// characters); a blank one comes as `document` and becomes `document.pdf`. + fn queue_open(name: &str, data: &FileData, inbox: &pdfcraft_ui_egui::Inbox, ctx: &egui::Context) { + let name = if name == "document" { DEFAULT_NAME } else { name }.to_string(); + let FileData::Bytes(view) = data else { + eframe::web_sys::console::warn_1(&format!("PdfCraft: host file {name} came as a File; PdfCraft reads bytes only; not opened").into()); + return; + }; + // A file from the host is untrusted input: refuse an oversized one before copying it into the app's memory. + if data.size() > MAX_OPEN_BYTES as f64 { + eframe::web_sys::console::warn_1( + &format!("PdfCraft: host file is {} bytes, over the {} MiB host open limit; not opened", data.size(), MAX_OPEN_BYTES >> 20).into(), + ); + return; } - Some(m) - } - - fn string(data: &JsValue, key: &str) -> Option { - js_sys::Reflect::get(data, &JsValue::from_str(key)).ok()?.as_string() + if let Ok(mut q) = inbox.lock() { + q.push((name, view.to_vec())); + } + ctx.request_repaint(); } } diff --git a/apps/pdfcraft/src/main.rs b/apps/pdfcraft/src/main.rs index 5bc5c5c372..be2290d2f9 100644 --- a/apps/pdfcraft/src/main.rs +++ b/apps/pdfcraft/src/main.rs @@ -439,6 +439,8 @@ fn app_creator<'a>( if let Some(dir) = pdfcraft_ui_egui::RecoveryStore::default_dir() { app.enable_recovery(pdfcraft_ui_egui::RecoveryStore::new(dir)); } + // The OCR Verify screen stores its per-language dictionary beside recovery data. + app.ocr_verify.dictionary_dir = pdfcraft_ui_egui::default_dictionary_dir(); if let Some(state) = &cc.wgpu_render_state { let adapter = state.adapter.get_info(); let max_texture_dimension_2d = state.device.limits().max_texture_dimension_2d; diff --git a/assets/icons/a-large-small.svg b/assets/icons/a-large-small.svg new file mode 100644 index 0000000000..7716db3cf2 --- /dev/null +++ b/assets/icons/a-large-small.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/contributors/people.toml b/contributors/people.toml index fecfb68c90..8ab196abc9 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -183,3 +183,6 @@ sync_github_name = true [people.Combiler01] sync_github_name = true + +[people.helenkwok] +real_name = "Helen Kwok" diff --git a/crates/annot/src/appearance.rs b/crates/annot/src/appearance.rs index 909250ef26..7148e9fc2e 100644 --- a/crates/annot/src/appearance.rs +++ b/crates/annot/src/appearance.rs @@ -232,6 +232,31 @@ fn rg_stroke(c: Rgb) -> String { format!("{} {} {} RG\n", n(c[0]), n(c[1]), n(c[2])) } +/// A FreeText's border colour where it has its own: the stroking colour in `/DA` (`RG`, or `G` +/// for grey). `None`: the border takes the text colour. +pub fn da_border(d: &Dict) -> Option { + let da = d.get(b"DA").and_then(|o| o.as_string()).map(|s| String::from_utf8_lossy(&s.bytes).into_owned())?; + let toks: Vec<&str> = da.split_whitespace().collect(); + let num = |i: usize| toks.get(i).and_then(|t| t.parse::().ok()).filter(|x| x.is_finite()); + let mut col = None; + for (i, t) in toks.iter().enumerate() { + match *t { + "RG" if i >= 3 => { + if let (Some(r), Some(g), Some(b)) = (num(i - 3), num(i - 2), num(i - 1)) { + col = Some([r, g, b].map(|x| x.clamp(0.0, 1.0))); + } + } + "G" if i >= 1 => { + if let Some(g) = num(i - 1) { + col = Some([g.clamp(0.0, 1.0); 3]); + } + } + _ => {} + } + } + col +} + /// `/DA` of a text box: (text colour, font size). Defaults to black 12 pt. pub fn parse_da(d: &Dict) -> (Rgb, f64) { let da = d.get(b"DA").and_then(|o| o.as_string()).map(|s| String::from_utf8_lossy(&s.bytes).into_owned()).unwrap_or_default(); @@ -262,23 +287,51 @@ pub fn parse_da(d: &Dict) -> (Rgb, f64) { (col.map(|x| x.clamp(0.0, 1.0)), size.min(400.0)) } -fn ext_gstate(opacity: f64, multiply: bool) -> Option { - if opacity >= 1.0 && !multiply { +/// How opaque a comment's parts are drawn, 0–1: its line or border (`/CA`); its fill or +/// background (`/ca`, which PDF 2.0 gives the non-stroking operations; without it, as the line); +/// and a text box's text (`/PCTextOpacity`, PdfCraft's own; without it, as the fill, since text +/// is filled too). A comment with only `/CA` is drawn at that opacity throughout, as before. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PartOpacity { + pub line: f64, + pub fill: f64, + pub text: f64, +} + +/// The parts' opacities of the comment `d` (see [`PartOpacity`]). +pub fn part_opacity(d: &Dict) -> PartOpacity { + let get = |key: &[u8]| d.get(key).and_then(|o| o.as_f64()).filter(|x| x.is_finite()).map(|x| x.clamp(0.0, 1.0)); + let line = get(b"CA").unwrap_or(1.0); + let fill = get(b"ca").unwrap_or(line); + let text = get(b"PCTextOpacity").unwrap_or(fill); + PartOpacity { line, fill, text } +} + +/// `/GS0`: the line's opacity for strokes and the fill's for everything filled; `/GS1`, where the +/// text's differs from the fill's: the text's for its glyphs. +fn ext_gstate(parts: PartOpacity, multiply: bool) -> Option { + if parts.line >= 1.0 && parts.fill >= 1.0 && parts.text >= 1.0 && !multiply { return None; } - let mut gs = Dict::new(); - gs.set(b"Type".to_vec(), Object::name("ExtGState")); - if multiply { - gs.set(b"BM".to_vec(), Object::name("Multiply")); - } - gs.set(b"CA".to_vec(), Object::Real(opacity)); - gs.set(b"ca".to_vec(), Object::Real(opacity)); + let gs = |stroke: f64, fill: f64| { + let mut gs = Dict::new(); + gs.set(b"Type".to_vec(), Object::name("ExtGState")); + if multiply { + gs.set(b"BM".to_vec(), Object::name("Multiply")); + } + gs.set(b"CA".to_vec(), Object::Real(stroke)); + gs.set(b"ca".to_vec(), Object::Real(fill)); + Object::Dict(gs) + }; let mut res = Dict::new(); - res.set(b"GS0".to_vec(), Object::Dict(gs)); + res.set(b"GS0".to_vec(), gs(parts.line, parts.fill)); + if parts.text != parts.fill { + res.set(b"GS1".to_vec(), gs(parts.line, parts.text)); + } Some(res) } -fn form(bbox: [f64; 4], content: &[u8], resources: Dict) -> Stream { +pub(crate) fn form(bbox: [f64; 4], content: &[u8], resources: Dict) -> Stream { let mut d = Dict::new(); d.set(b"Type".to_vec(), Object::name("XObject")); d.set(b"Subtype".to_vec(), Object::name("Form")); @@ -288,6 +341,18 @@ fn form(bbox: [f64; 4], content: &[u8], resources: Dict) -> Stream { Stream::flate(d, content) } +/// The `cm` that draws a picture into `at`: an image (its unit square) or a form XObject whose +/// `/Matrix` maps it to `size` points (`None`, or a side that is not positive, draws nothing). +pub(crate) fn picture_place(at: [f64; 4], image: bool, size: Option<(f64, f64)>) -> Option { + let [x0, y0, x1, y1] = at; + let (w, h) = (x1 - x0, y1 - y0); + if image { + return Some(format!("{} 0 0 {} {} {} cm", n(w), n(h), n(x0), n(y0))); + } + let (sw, sh) = size.filter(|(sw, sh)| *sw > 0.0 && *sh > 0.0)?; + Some(format!("{} 0 0 {} {} {} cm", n(w / sw), n(h / sh), n(x0), n(y0))) +} + /// The page `/Rotate` an image signature's picture is turned back by (`/PCPictureRotate`, written /// when the signature is added to a turned page): 0, 90, 180 or 270. pub(crate) fn picture_turn(d: &Dict) -> i64 { @@ -301,10 +366,18 @@ fn stamp_turn(d: &Dict) -> i64 { /// The page `/Rotate` a text box's or Fill & Sign text's lines are turned back by /// (`/PCTextRotate`, written when it is added to a turned page). -fn text_turn(d: &Dict) -> i64 { +pub(crate) fn text_turn(d: &Dict) -> i64 { d.get(b"PCTextRotate").and_then(Object::as_int).unwrap_or(0).rem_euclid(360) / 90 * 90 } +/// A FreeText's `/RD`: how far its text box lies inside `/Rect` from the left, bottom, right and +/// top. A distance a hair below 0 (rounding) reads as 0; `None` when one is negative or they can't +/// be read. +pub(crate) fn inset(d: &Dict) -> Option<[f64; 4]> { + let rd: [f64; 4] = nums(d, b"RD")?.try_into().ok()?; + rd.iter().all(|x| x.is_finite() && *x > -0.001).then(|| rd.map(|x| x.max(0.0))) +} + /// The first text [`build`] would draw in the standard fonts (Helvetica, WinAnsiEncoding) that /// they can't, which would come out as `?`: what it is (for the message) and the character. The /// text of a text box, callout or Fill & Sign text, a line's caption, a measurement's label, a @@ -325,7 +398,7 @@ pub fn undrawable_text(d: &Dict) -> Option<(&'static str, char)> { } /// `/Font` resources naming `/Helv`: standard Helvetica with WinAnsiEncoding. -fn helvetica_resource() -> Object { +pub(crate) fn helvetica_resource() -> Object { let mut font = Dict::new(); font.set(b"Type".to_vec(), Object::name("Font")); font.set(b"Subtype".to_vec(), Object::name("Type1")); @@ -379,7 +452,9 @@ struct AppearanceContext<'a> { subtype: &'a [u8], rect: [f64; 4], cloudy: bool, + /// The line's opacity (`parts.line`). opacity: f64, + parts: PartOpacity, stroke: Option, width: f64, } @@ -396,7 +471,7 @@ impl AppearanceDrawing { fn new(context: &AppearanceContext<'_>) -> Self { let mut drawing = Self::default(); if let Some(gs) = ext_gstate( - context.opacity, + context.parts, context.subtype == b"Highlight" || (context.subtype == b"Ink" && context.dict.name(b"IT") == Some(b"FreehandHighlight")), ) { drawing.resources.set(b"ExtGState".to_vec(), Object::Dict(gs)); @@ -443,7 +518,8 @@ fn appearance_context(d: &Dict) -> Option> { subtype, rect, cloudy, - opacity: d.get(b"CA").and_then(|o| o.as_f64()).unwrap_or(1.0).clamp(0.0, 1.0), + opacity: part_opacity(d).line, + parts: part_opacity(d), stroke: color(d, b"C")?, width: border_width(d), }) @@ -786,15 +862,12 @@ fn draw_stamp(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawing) - // the standard counter-rotation as its form `/Matrix`: viewers fit the turned // bounding box to `/Rect` (ISO 32000-2 §12.5.5), so it reads upright as displayed. if let Some(pic) = d.get(b"PCPicture").and_then(Object::as_ref) { - let [x0, y0, x1, y1] = rect; - let (w, h) = (x1 - x0, y1 - y0); let image = matches!(d.get(b"PCPictureImage"), Some(Object::Bool(true))); - let place = if image { - format!("{} 0 0 {} {} {} cm", n(w), n(h), n(x0), n(y0)) - } else { - let size = nums(d, b"PCPictureSize").filter(|s| s.len() == 2 && s[0] > 0.0 && s[1] > 0.0)?; - format!("{} 0 0 {} {} {} cm", n(w / size[0]), n(h / size[1]), n(x0), n(y0)) - }; + let size = nums(d, b"PCPictureSize").and_then(|s| match s.as_slice() { + [w, h] if *w > 0.0 && *h > 0.0 => Some((*w, *h)), + _ => None, + }); + let place = picture_place(rect, image, size)?; c.push_str(&format!("q {place} /Pic Do Q\n")); let mut xo = Dict::new(); xo.set(b"Pic".to_vec(), Object::Ref(pic)); @@ -836,46 +909,25 @@ fn draw_stamp(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawing) - let [x0, y0, x1, y1] = draw; let (w, h) = (x1 - x0, y1 - y0); let lw = w.min(h) * 0.12; + // The strokes come from `FillMark::strokes`, which a copy's outline draws too. + let path = |mark: crate::FillMark| { + let ops: Vec = mark + .strokes(draw) + .iter() + .flat_map(|s| s.iter().enumerate().map(|(i, p)| format!("{} {} {}", n(p[0]), n(p[1]), if i == 0 { "m" } else { "l" }))) + .collect(); + format!("{} S\n", ops.join(" ")) + }; match name { - b"PCCheck" => c.push_str(&format!( - "{}{} w 1 J 1 j\n{} {} m {} {} l {} {} l S\n", - rg_stroke(col), - n(lw), - n(x0 + w * 0.15), - n(y0 + h * 0.5), - n(x0 + w * 0.4), - n(y0 + h * 0.2), - n(x0 + w * 0.88), - n(y0 + h * 0.85) - )), - b"PCCross" => c.push_str(&format!( - "{}{} w 1 J\n{} {} m {} {} l {} {} m {} {} l S\n", - rg_stroke(col), - n(lw), - n(x0 + w * 0.18), - n(y0 + h * 0.18), - n(x1 - w * 0.18), - n(y1 - h * 0.18), - n(x0 + w * 0.18), - n(y1 - h * 0.18), - n(x1 - w * 0.18), - n(y0 + h * 0.18) - )), + b"PCCheck" => c.push_str(&format!("{}{} w 1 J 1 j\n{}", rg_stroke(col), n(lw), path(crate::FillMark::Check))), + b"PCCross" => c.push_str(&format!("{}{} w 1 J\n{}", rg_stroke(col), n(lw), path(crate::FillMark::Cross))), b"PCDot" => { let r = w.min(h) * 0.3; c.push_str(&rg(col)); c.push_str(&ellipse(x0 + w / 2.0 - r, y0 + h / 2.0 - r, x0 + w / 2.0 + r, y0 + h / 2.0 + r)); c.push_str("f\n"); } - b"PCLine" => c.push_str(&format!( - "{}{} w 1 J\n{} {} m {} {} l S\n", - rg_stroke(col), - n(h.clamp(0.5, 2.0)), - n(x0), - n(y0 + h / 2.0), - n(x1), - n(y0 + h / 2.0) - )), + b"PCLine" => c.push_str(&format!("{}{} w 1 J\n{}", rg_stroke(col), n(h.clamp(0.5, 2.0)), path(crate::FillMark::Line))), other => { // Only stamps PdfCraft made: others keep their own artwork. if !matches!(d.get(b"PCStamp"), Some(Object::Bool(true))) { @@ -893,14 +945,59 @@ fn draw_stamp(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawing) - Some(form(rect, c.as_bytes(), res)) } +/// The oval or circle (`shape`) around a text box whose whole box is `full`: its background +/// and border go into `c`; returns the text box inside it. +#[allow(clippy::too_many_arguments)] +fn shaped_border( + d: &Dict, + shape: crate::BorderShape, + full: [f64; 4], + turn: i64, + bw: f64, + bg: Option, + line_color: Rgb, + c: &mut String, +) -> Option<[f64; 4]> { + // Laid out as displayed: a quarter turn swaps the sides' differences. + let rect = match inset(d) { + Some(rd) => { + let (dx, dy) = if matches!(turn, 90 | 270) { (rd[1], rd[0]) } else { (rd[0], rd[1]) }; + [full[0] + dx, full[1] + dy, full[2] - dx, full[3] - dy] + } + None => shape.inside(full, bw, full), + }; + if rect[2] - rect[0] < 1.0 || rect[3] - rect[1] < 1.0 { + return None; + } + let h = bw / 2.0; + let [x0, y0, x1, y1] = [full[0] + h, full[1] + h, full[2] - h, full[3] - h]; + let path = match shape { + crate::BorderShape::Circle => { + let (cx, cy, r) = ((x0 + x1) / 2.0, (y0 + y1) / 2.0, (x1 - x0).min(y1 - y0) / 2.0); + ellipse(cx - r, cy - r, cx + r, cy + r) + } + _ => ellipse(x0, y0, x1, y1), + }; + if let Some(bg) = bg { + c.push_str(&format!("{}{path}f\n", rg(bg))); + } + if bw > 0.0 { + c.push_str(&format!("{}{} w\n{}{path}S\n[] 0 d\n", rg_stroke(line_color), n(bw), dash(d))); + } + Some(rect) +} + fn draw_free_text(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawing) -> Option { let d = context.dict; + let parts = context.parts; let rect = context.rect; let stroke = context.stroke; let mut c = std::mem::take(&mut drawing.content); let mut res = std::mem::take(&mut drawing.resources); let _tail = std::mem::take(&mut drawing.tail); let (text_color, size) = parse_da(d); + // The box's border and a callout's leader line: their own colour, else the text's. + let line_color = da_border(d).unwrap_or(text_color); let bg = stroke; let bw = if d.contains(b"BS") || d.contains(b"Border") { border_width(d) } else { 0.0 }; // A callout: the leader line (arrowhead at its first point), and the text box inside @@ -939,7 +1036,7 @@ fn draw_free_text(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawin if pts.len() < 2 || pts.iter().any(|p| !p.0.is_finite() || !p.1.is_finite()) { return None; } - c.push_str(&format!("{}{}{} w 1 J 1 j\n", rg_stroke(text_color), rg(bg.unwrap_or([1.0; 3])), n(lw))); + c.push_str(&format!("{}{}{} w 1 J 1 j\n", rg_stroke(line_color), rg(bg.unwrap_or([1.0; 3])), n(lw))); for (i, p) in pts.iter().enumerate() { c.push_str(&format!("{} {} {}\n", n(p.0), n(p.1), if i == 0 { "m" } else { "l" })); } @@ -948,26 +1045,40 @@ fn draw_free_text(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawin return None; } } - if let Some(bg) = bg { - c.push_str(&format!("{}{} {} {} {} re f\n", rg(bg), n(rect[0]), n(rect[1]), n(rect[2] - rect[0]), n(rect[3] - rect[1]))); - } - if bw > 0.0 { - let h = bw / 2.0; - c.push_str(&format!( - "{}{} w\n{}{} {} {} {} re S\n[] 0 d\n", - rg_stroke(text_color), - n(bw), - dash(d), - n(rect[0] + h), - n(rect[1] + h), - n(rect[2] - rect[0] - bw), - n(rect[3] - rect[1] - bw) - )); + // An oval or a circle (`/PCBorderShape`) around the text box, which is `/Rect` inset by + // `/RD` and keeps its place: background and border take the shape, the text is laid out + // in the text box as in a rectangle. + let shape = crate::BorderShape::of(d); + if shape != crate::BorderShape::Rectangle { + rect = shaped_border(d, shape, full, turn, bw, bg, line_color, &mut c)?; + } else { + if let Some(bg) = bg { + c.push_str(&format!("{}{} {} {} {} re f\n", rg(bg), n(rect[0]), n(rect[1]), n(rect[2] - rect[0]), n(rect[3] - rect[1]))); + } + if bw > 0.0 { + let h = bw / 2.0; + c.push_str(&format!( + "{}{} w\n{}{} {} {} {} re S\n[] 0 d\n", + rg_stroke(line_color), + n(bw), + dash(d), + n(rect[0] + h), + n(rect[1] + h), + n(rect[2] - rect[0] - bw), + n(rect[3] - rect[1] - bw) + )); + } } let text = d.get(b"Contents").and_then(|o| o.as_string()).map(PdfString::to_text).unwrap_or_default(); let pad = 2.0 + bw; let width = (rect[2] - rect[0] - 2.0 * pad).max(1.0); - let q = d.int(b"Q").unwrap_or(0); + // In an oval or a circle the lines are centred in the text box, up and down too. + let round = shape != crate::BorderShape::Rectangle; + let q = if round { 1 } else { d.int(b"Q").unwrap_or(0) }; + // The text at its own opacity, where it has one. + if parts.text != parts.fill { + c.push_str("/GS1 gs\n"); + } c.push_str(&format!( "{} {} {} {} re W n\nBT\n/Helv {} Tf\n{}", n(rect[0]), @@ -978,8 +1089,12 @@ fn draw_free_text(context: &AppearanceContext<'_>, mut drawing: AppearanceDrawin rg(text_color) )); let mut out = c.into_bytes(); + let lines = wrap(&text, size, width); let mut y = rect[3] - pad - size * 0.9; - for line in wrap(&text, size, width) { + if round { + y -= ((rect[3] - rect[1] - 2.0 * pad - lines.len() as f64 * size * 1.2) / 2.0).max(0.0); + } + for line in lines { if y < rect[1] - size { break; } diff --git a/crates/annot/src/appearance/symbols.rs b/crates/annot/src/appearance/symbols.rs index 3b33662524..3178f78e94 100644 --- a/crates/annot/src/appearance/symbols.rs +++ b/crates/annot/src/appearance/symbols.rs @@ -1,4 +1,4 @@ -use super::{Dict, Object, PdfString, Rgb, Stream, color, ext_gstate, form, literal, n, nums, rg, rg_stroke, text_width, win_ansi}; +use super::{Dict, Object, PartOpacity, PdfString, Rgb, Stream, color, ext_gstate, form, literal, n, nums, rg, rg_stroke, text_width, win_ansi}; /// The label and colour of a standard rubber stamp name (ISO 32000-2 Table 184), for a stamp /// another program wrote without an appearance. Names are matched byte for byte, as PDF names @@ -51,7 +51,7 @@ pub(crate) fn display_stamp(d: &Dict) -> Option { // A malformed `/C` falls back to the stamp's own colour rather than hiding the stamp. let stroke = color(d, b"C").flatten(); let mut res = Dict::new(); - if let Some(gs) = ext_gstate(opacity, false) { + if let Some(gs) = ext_gstate(PartOpacity { line: opacity, fill: opacity, text: opacity }, false) { res.set(b"ExtGState".to_vec(), Object::Dict(gs)); } // A stamp whose intent (`/IT`, PDF 2.0) is an image or a snapshot has no `/Name` to default. diff --git a/crates/annot/src/create.rs b/crates/annot/src/create.rs index 1904745e95..6376655ce9 100644 --- a/crates/annot/src/create.rs +++ b/crates/annot/src/create.rs @@ -1,6 +1,6 @@ use super::draw::draw_appearance; use super::lookup::{annot_dict, annot_ref, annots, fill_sign_shape, page_ref, set_annots}; -use super::modify::{n, rich_text}; +use super::modify::{free_text_da, n, rich_text, set_part_keys, shape_text_box}; use super::types::{ FLAG_NO_ROTATE, FLAG_NO_ZOOM, FLAG_PRINT, LineEnding, Markup, Meta, NOTE_SIZE, NewAnnotation, OverlayLook, Rgb, Shape, StampGroup, Style, }; @@ -214,8 +214,13 @@ pub fn add_annotation(doc: &mut Document, new: &NewAnnotation, meta: &Meta) -> R let rect = rect_for(&new.shape, style)?; let mut d = base_dict(new.shape.subtype(), rect, page, &new.contents, &new.author, meta); d.set(b"Subj".to_vec(), PdfString::text(subject(&new.shape))); - let opacity = style.opacity.clamp(0.0, 1.0); - set_non_opaque_style(&mut d, opacity); + if style.fill_opacity.into_iter().chain(style.text_opacity).any(|o| !o.is_finite()) { + return Err(AnnotError::Invalid("invalid opacity".into())); + } + let line = style.opacity.clamp(0.0, 1.0); + let fill = style.fill_opacity.map_or(line, |o| o.clamp(0.0, 1.0)); + let text = style.text_opacity.map_or(fill, |o| o.clamp(0.0, 1.0)); + set_part_keys(&mut d, appearance::PartOpacity { line, fill, text }); let border = |d: &mut Dict| { let mut bs = Dict::new(); bs.set(b"W".to_vec(), Object::Real(style.width.max(0.0))); @@ -380,20 +385,22 @@ pub fn add_annotation(doc: &mut Document, new: &NewAnnotation, meta: &Meta) -> R d.set(b"IT".to_vec(), Object::name("FreeTextTypeWriter")); } let size = if font_size.is_finite() && *font_size > 0.0 { font_size.clamp(1.0, 400.0) } else { 12.0 }; - let [r, g, b] = style.color.map(|x| x.clamp(0.0, 1.0)); - d.set(b"DA".to_vec(), PdfString::literal(format!("{} {} {} rg /Helv {} Tf", n(r), n(g), n(b), n(size)).into_bytes())); + d.set(b"DA".to_vec(), free_text_da(style.color, style.border_color, size)); d.set(b"Q".to_vec(), Object::Int(0)); if let Some(f) = style.fill { d.set(b"C".to_vec(), rgb(f)); } border(&mut d); rich_text(&mut d); - // The text is laid out for the page as displayed, like a stamp's label. if !matches!(new.shape, Shape::Callout { .. }) { + // The text is laid out for the page as displayed, like a stamp's label. let turn = pdfcraft_model::pages(doc).get(new.page).map_or(0, |p| p.rotation(doc)); if matches!(turn, 90 | 180 | 270) { d.set(b"PCTextRotate".to_vec(), Object::Int(turn)); } + // An oval or a circle goes around the text box, its text wrapped as displayed: + // after `/PCTextRotate`, which swaps the box's sides on a quarter-turned page. + shape_text_box(&mut d, rect, style.border_shape, style.width); } } _ => {} @@ -421,7 +428,7 @@ pub fn add_annotation(doc: &mut Document, new: &NewAnnotation, meta: &Meta) -> R } fn validate_style(style: &Style) -> Result<(), AnnotError> { - if !finite(&style.color) || !style.opacity.is_finite() || !style.width.is_finite() { + if !finite(&style.color) || style.border_color.is_some_and(|c| !finite(&c)) || !style.opacity.is_finite() || !style.width.is_finite() { return Err(AnnotError::Invalid("invalid style".into())); } Ok(()) @@ -443,12 +450,6 @@ fn set_redaction_properties(dict: &mut Dict, style: &Style, quads: &[[f64; 8]], } } -fn set_non_opaque_style(dict: &mut Dict, opacity: f64) { - if opacity < 1.0 { - dict.set(b"CA".to_vec(), Object::Real(opacity)); - } -} - fn add_popup(doc: &mut Document, parent: ObjRef, rect: Option<[f64; 4]>, annots: &mut Vec) -> Result<(), AnnotError> { let Some(rect) = rect else { return Ok(()) }; let mut popup = Dict::new(); diff --git a/crates/annot/src/info.rs b/crates/annot/src/info.rs index 1a54dbc077..ba181d7504 100644 --- a/crates/annot/src/info.rs +++ b/crates/annot/src/info.rs @@ -2,7 +2,7 @@ use super::create::finite; use super::draw::set_appearance; use super::lookup::{annot_dict, annot_ref, annotation_flags, annots, drawn_upright, is_fill_sign, page_ref, page_refs}; use super::modify::{fillable, touch, unlocked}; -use super::types::{FLAG_LOCKED, LineEnding, Meta, NoteIcon, Rgb}; +use super::types::{BorderShape, FLAG_LOCKED, LineEnding, Meta, NoteIcon, Rgb}; use super::{AnnotError, Dict, Document, ObjRef, Object, PdfString, appearance}; // ── reading ───────────────────────────────────────────────────────────────────────────────── @@ -217,6 +217,15 @@ pub struct Props { pub fill: Option, /// Text boxes, typewriter text and callouts: the `/DA` font size in points. pub font_size: Option, + /// Text boxes, typewriter text and callouts: the border's own colour (`None`: the text's). + pub border_color: Option, + /// Text boxes and typewriter text: the border's shape (callouts have none to choose). + pub border_shape: Option, + /// How opaque the fill is drawn, for a comment with a fill control (`opacity` is then its + /// line's or border's). + pub fill_opacity: Option, + /// How opaque a text box's, typewriter text's or callout's text is drawn. + pub text_opacity: Option, } /// The current properties of the comment at `(page, index)`. @@ -239,7 +248,7 @@ pub fn props(doc: &Document, page: usize, index: usize) -> Option { author: text_value(doc, d, b"T").unwrap_or_default(), subject: text_value(doc, d, b"Subj").unwrap_or_default(), color, - opacity: d.get(b"CA").and_then(|o| o.as_f64()).unwrap_or(1.0), + opacity: appearance::part_opacity(d).line, width, icon: (subtype == "Text").then(|| d.name(b"Name").and_then(|n| NoteIcon::from_name(&String::from_utf8_lossy(n))).unwrap_or(NoteIcon::Note)), modified: text_value(doc, d, b"M"), @@ -248,6 +257,10 @@ pub fn props(doc: &Document, page: usize, index: usize) -> Option { fillable: fillable(&subtype), fill: fill_of(d), font_size: (subtype == "FreeText").then(|| appearance::parse_da(d).1), + border_color: (subtype == "FreeText").then(|| appearance::da_border(d)).flatten(), + border_shape: (subtype == "FreeText" && !d.contains(b"CL")).then(|| BorderShape::of(d)), + fill_opacity: fillable(&subtype).then(|| appearance::part_opacity(d).fill), + text_opacity: (subtype == "FreeText").then(|| appearance::part_opacity(d).text), subtype, endings: line_endings_of(d), }) diff --git a/crates/annot/src/lib.rs b/crates/annot/src/lib.rs index 7bd0894c66..52c0aad74f 100644 --- a/crates/annot/src/lib.rs +++ b/crates/annot/src/lib.rs @@ -22,7 +22,9 @@ mod create; mod draw; mod info; mod lookup; +mod model3d; mod modify; +mod threed; mod types; #[cfg(test)] @@ -36,14 +38,17 @@ use lookup::{annots, page_ref}; pub use lookup::{ has_visible_fill_sign, is_fill_sign, isolate_annotation, isolate_annotations, page_refs, picture_rotation, set_annots, signature_image, }; +pub use model3d::{Model3d, Model3dKind, model_3d, model_3d_annotations}; pub(crate) use modify::n; pub use modify::{ - add_reply, add_text_replacement, deletable, delete_annotation, erase_ink, fillable, goes_with, lift_annotation, move_annotation, set_contents, - set_fill, set_font_size, set_locked, set_marked, set_points, set_rect, set_review_state, set_style, with_replies_and_popups, + add_reply, add_text_replacement, deletable, delete_annotation, erase_ink, fillable, goes_with, lift_annotation, move_annotation, + set_border_color, set_border_shape, set_contents, set_fill, set_font_size, set_locked, set_marked, set_part_opacity, set_points, set_rect, + set_review_state, set_style, with_replies_and_popups, }; +pub use threed::{Activation as Activation3d, MAX_MODEL_BYTES, ModelFormat, NewModel3d, PosterPicture, add_3d_model, check_model}; pub use types::{ - AttachIcon, FillMark, LineEnding, Markup, Meta, NOTE_SIZE, NewAnnotation, NoteIcon, OverlayFont, OverlayLook, ReviewState, Rgb, Shape, - StampGroup, StampKind, Style, StyleChange, TextReplacement, rect_quad, + AttachIcon, BorderShape, FillMark, LineEnding, Markup, Meta, NOTE_SIZE, NewAnnotation, NoteIcon, OverlayFont, OverlayLook, ReviewState, Rgb, + Shape, StampGroup, StampKind, Style, StyleChange, TextReplacement, rect_quad, }; pub mod appearance; @@ -70,6 +75,12 @@ pub enum AnnotError { /// with a font that has those characters. The message names the character. #[error("{0}")] Undrawable(String), + /// A 3D model file that is neither U3D (starts with `U3D` and a zero byte) nor PRC (starts `PRC`). + #[error("not a 3D model: the file must be U3D (ECMA-363) or PRC (ISO 14739-1)")] + NotAModel, + /// A 3D model larger than `MAX_MODEL_BYTES`. + #[error("the 3D model is {size} bytes; the largest accepted is {max}")] + ModelTooLarge { size: usize, max: usize }, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), } diff --git a/crates/annot/src/model3d.rs b/crates/annot/src/model3d.rs new file mode 100644 index 0000000000..b50342898e --- /dev/null +++ b/crates/annot/src/model3d.rs @@ -0,0 +1,77 @@ +//! Reading the model a 3D annotation shows (ISO 32000-2 §13.6.3 and §13.6.4): its `/3DD` is a +//! 3D stream, or a 3D reference dictionary naming the stream another annotation shares. + +use super::lookup::{annots, page_ref}; +use super::threed::MAX_MODEL_BYTES; +use super::{AnnotError, Dict, Document, Object}; + +/// The format of a 3D stream, from its `/Subtype`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Model3dKind { + /// Product Representation Compact (ISO 14739-1). + Prc, + /// Universal 3D (ECMA-363). + U3d, +} + +/// A 3D annotation's model: its format and the decoded stream. +#[derive(Debug, Clone, PartialEq)] +pub struct Model3d { + pub kind: Model3dKind, + pub data: Vec, +} + +/// The `/Annots` positions of the 3D annotations on 0-based `page`, in order. +/// +/// # Errors +/// [`AnnotError::NoPageTree`] or [`AnnotError::NoSuchPage`]. +pub fn model_3d_annotations(doc: &Document, page: usize) -> Result, AnnotError> { + let list = annots(doc, page_ref(doc, page)?); + Ok(list.iter().enumerate().filter(|(_, a)| doc.dict(a).is_some_and(|d| d.name(b"Subtype") == Some(b"3D"))).map(|(i, _)| i).collect()) +} + +/// The model of the 3D annotation at `/Annots` position `index` on 0-based `page`, decoded. +/// +/// # Errors +/// [`AnnotError::NoSuchAnnotation`] when there is no annotation there; [`AnnotError::Invalid`] +/// when it is not a 3D annotation, has no readable 3D stream, names a format other than PRC or +/// U3D, or decodes past [`MAX_MODEL_BYTES`]. +pub fn model_3d(doc: &Document, page: usize, index: usize) -> Result { + let list = annots(doc, page_ref(doc, page)?); + let entry = list.get(index).ok_or(AnnotError::NoSuchAnnotation { page, index })?; + let annot = doc.dict(entry).ok_or(AnnotError::NoSuchAnnotation { page, index })?; + let at = || format!("annotation {} on page {}", index.saturating_add(1), page.saturating_add(1)); + if annot.name(b"Subtype") != Some(b"3D") { + return Err(AnnotError::Invalid(format!("{} is not a 3D annotation", at()))); + } + let stream = stream_of(doc, &annot).ok_or_else(|| AnnotError::Invalid(format!("{} has no 3D stream", at())))?; + let Object::Stream(s) = stream.as_ref() else { + return Err(AnnotError::Invalid(format!("{} has no 3D stream", at()))); + }; + let kind = match s.dict.name(b"Subtype") { + Some(b"PRC") => Model3dKind::Prc, + Some(b"U3D") => Model3dKind::U3d, + other => { + let name = other.map_or_else(|| "none".to_owned(), |n| String::from_utf8_lossy(n).into_owned()); + return Err(AnnotError::Invalid(format!("{}'s 3D stream is of an unknown format ({name})", at()))); + } + }; + let data = doc + .with_direct_parms(s) + .decoded_within(MAX_MODEL_BYTES) + .map_err(|e| AnnotError::Invalid(format!("{}'s 3D stream could not be decoded: {e}", at())))?; + Ok(Model3d { kind, data }) +} + +/// What an annotation's `/3DD` names: the 3D stream itself, or the `/3D` entry of a 3D reference +/// dictionary. A reference is followed once; the caller refuses anything that is not a stream. +fn stream_of(doc: &Document, annot: &Dict) -> Option> { + let obj = doc.resolve(annot.get(b"3DD")?); + match obj.as_ref() { + Object::Dict(d) if d.name(b"Type") == Some(b"3DRef") => Some(doc.resolve(d.get(b"3D")?)), + _ => Some(obj), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/annot/src/model3d/tests.rs b/crates/annot/src/model3d/tests.rs new file mode 100644 index 0000000000..90addb1bb6 --- /dev/null +++ b/crates/annot/src/model3d/tests.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use super::*; + +/// A one-page document whose page has `annots` as its `/Annots`, plus `extra` objects from 4 on. +fn doc(annots: &str, extra: &[Vec]) -> Document { + let mut objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /Annots {annots} >>").into_bytes(), + ]; + objs.extend(extra.iter().cloned()); + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + Document::open(Arc::new(out)).expect("opens") +} + +fn stream(dict: &str, body: &[u8]) -> Vec { + let mut s = format!("<< {dict} /Length {} >>\nstream\n", body.len()).into_bytes(); + s.extend_from_slice(body); + s.extend_from_slice(b"\nendstream"); + s +} + +fn annot(dd: &str) -> Vec { + format!("<< /Type /Annot /Subtype /3D /Rect [0 0 100 100] /3DD {dd} >>").into_bytes() +} + +#[test] +fn a_plain_3d_stream_is_read_through_its_filters() { + let d = doc("[4 0 R]", &[annot("5 0 R"), stream("/Type /3D /Subtype /PRC /Filter /ASCIIHexDecode", b"505243>")]); + assert_eq!(model_3d(&d, 0, 0), Ok(Model3d { kind: Model3dKind::Prc, data: b"PRC".to_vec() })); +} + +#[test] +fn a_3dref_leads_to_the_stream_it_names() { + let d = doc("[4 0 R]", &[annot("<< /Type /3DRef /3D 5 0 R >>"), stream("/Type /3D /Subtype /PRC", b"PRC")]); + assert_eq!(model_3d(&d, 0, 0).map(|m| m.data), Ok(b"PRC".to_vec())); +} + +#[test] +fn a_u3d_stream_is_read_and_says_so() { + let d = doc("[4 0 R]", &[annot("5 0 R"), stream("/Type /3D /Subtype /U3D", b"U3D")]); + assert_eq!(model_3d(&d, 0, 0).map(|m| m.kind), Ok(Model3dKind::U3d)); +} + +#[test] +fn a_3d_annotation_without_a_stream_is_invalid() { + let d = doc("[<< /Type /Annot /Subtype /3D /Rect [0 0 1 1] >>]", &[]); + assert!(matches!(model_3d(&d, 0, 0), Err(AnnotError::Invalid(m)) if m.contains("no 3D stream"))); +} + +#[test] +fn an_unknown_format_is_invalid() { + let d = doc("[4 0 R]", &[annot("5 0 R"), stream("/Type /3D /Subtype /STL", b"x")]); + assert!(matches!(model_3d(&d, 0, 0), Err(AnnotError::Invalid(m)) if m.contains("unknown format (STL)"))); +} + +#[test] +fn a_3dref_is_followed_once_and_only_to_a_stream() { + // A reference naming another reference, or one that names itself, is damage: no stream. + let d = doc("[4 0 R]", &[annot("5 0 R"), b"<< /Type /3DRef /3D 6 0 R >>".to_vec(), b"<< /Type /3DRef /3D 5 0 R >>".to_vec()]); + assert!(matches!(model_3d(&d, 0, 0), Err(AnnotError::Invalid(m)) if m.contains("no 3D stream"))); + // The stream is under `/3D`; a `/3DD` inside the reference is not the standard's key. + let d = doc("[4 0 R]", &[annot("<< /Type /3DRef /3DD 5 0 R >>"), stream("/Type /3D /Subtype /PRC", b"PRC")]); + assert!(matches!(model_3d(&d, 0, 0), Err(AnnotError::Invalid(m)) if m.contains("no 3D stream"))); +} + +#[test] +fn another_annotation_is_not_a_3d_one() { + let d = doc("[<< /Type /Annot /Subtype /Square /Rect [0 0 1 1] >> 4 0 R]", &[annot("5 0 R"), stream("/Subtype /PRC", b"PRC")]); + assert!(matches!(model_3d(&d, 0, 0), Err(AnnotError::Invalid(m)) if m.contains("not a 3D annotation"))); + assert_eq!(model_3d(&d, 0, 2), Err(AnnotError::NoSuchAnnotation { page: 0, index: 2 })); + assert_eq!(model_3d_annotations(&d, 0), Ok(vec![1])); + assert_eq!(model_3d_annotations(&d, 1), Err(AnnotError::NoSuchPage(1))); +} diff --git a/crates/annot/src/modify.rs b/crates/annot/src/modify.rs index a615b8e013..c7d87be090 100644 --- a/crates/annot/src/modify.rs +++ b/crates/annot/src/modify.rs @@ -5,8 +5,8 @@ use super::draw::{draw_appearance, redraw_or_restore, set_appearance}; use super::info::{line_endings_of, text_value}; use super::lookup::{annot_dict, annot_ref, annotation_flags, annots, page_ref, set_annots}; use super::types::{ - FLAG_HIDDEN, FLAG_LOCKED, FLAG_NO_ROTATE, FLAG_NO_ZOOM, FLAG_PRINT, LineEnding, Markup, Meta, NewAnnotation, ReviewState, Rgb, Shape, Style, - StyleChange, TextReplacement, + BorderShape, FLAG_HIDDEN, FLAG_LOCKED, FLAG_NO_ROTATE, FLAG_NO_ZOOM, FLAG_PRINT, LineEnding, Markup, Meta, NewAnnotation, ReviewState, Rgb, + Shape, Style, StyleChange, TextReplacement, }; use super::{AnnotError, Dict, Document, HashMap, HashSet, ObjRef, Object, PdfString, appearance}; @@ -159,10 +159,13 @@ pub fn with_replies_and_popups(doc: &Document, list: &[Object], doomed: impl Int } /// Change a comment's text. Text boxes are redrawn to show it, and their rectangle follows the -/// new text: the wrap width and top edge stay, the height fits the wrapped lines. +/// new text: the wrap width and top edge stay, the height fits the wrapped lines. A circle snug +/// around its text stays snug around the new text, centred where it was. pub fn set_contents(doc: &mut Document, page: usize, index: usize, text: &str, meta: &Meta) -> Result<(), AnnotError> { let (_, r) = annot_ref(doc, page, index)?; - let free_text = annot_dict(doc, r).name(b"Subtype") == Some(b"FreeText"); + let d = annot_dict(doc, r); + let free_text = d.name(b"Subtype") == Some(b"FreeText"); + let snug = free_text && is_snug(&d); doc.update_dict(r, |d| { d.set(b"Contents".to_vec(), PdfString::text(text)); // The rich-text version follows the new plain text (text boxes), or would contradict it. @@ -177,8 +180,16 @@ pub fn set_contents(doc: &mut Document, page: usize, index: usize, text: &str, m // The box grows and shrinks with the text instead of clipping it. A locked box keeps its // text editable (as in Acrobat), so the re-fit ignores the lock. New text is drawn even // where the standard fonts can't draw some of it (marked and reported, see - // `draw_appearance`). - match fitted_box(doc, r, text) { + // `draw_appearance`). A circle snug around its text stays snug around the new text; one + // made larger by hand keeps its size, only growing for more lines. + let fit = if snug { + snug_refit(doc, r) + } else { + let d = annot_dict(doc, r); + let circle = text_box_of(&d).filter(|_| BorderShape::of(&d) == BorderShape::Circle); + fitted_box(doc, r, text).map(|fit| circle.map_or(fit, |old| [fit[0], fit[1].min(old[1]), fit[2], fit[3]])) + }; + match fit { Some(rect) => apply_text_box(doc, r, rect, meta, true)?, None => draw_appearance(doc, r)?, } @@ -197,11 +208,12 @@ fn fitted_box(doc: &Document, r: ObjRef, text: &str) -> Option<[f64; 4]> { let rect = nums(b"Rect").filter(|v| v.len() == 4 && v.iter().all(|x| x.is_finite()))?; let rect = [rect[0], rect[1], rect[2], rect[3]]; // A callout re-fits its text box (`/Rect` inset by `/RD`); the leader line keeps its place. + // So does a text box inside an oval or a circle. let tb = if d.contains(b"CL") { let rd = nums(b"RD").filter(|v| v.len() == 4 && v.iter().all(|x| *x >= 0.0))?; [rect[0] + rd[0], rect[1] + rd[1], rect[2] - rd[2], rect[3] - rd[3]] } else { - rect + text_box_of(&d).unwrap_or(rect) }; let (x0, top) = (tb[0].min(tb[2]), tb[1].max(tb[3])); let w = tb[2] - tb[0]; @@ -388,9 +400,134 @@ pub fn set_rect(doc: &mut Document, page: usize, index: usize, rect: [f64; 4], m if !matches!(subtype.as_str(), "Square" | "Circle" | "FreeText") { return Err(AnnotError::Invalid(format!("{subtype} comments can't be resized"))); } + // An oval's or a circle's frame is resized: its text box follows inside. + let shape = if subtype == "FreeText" { BorderShape::of(&d) } else { BorderShape::Rectangle }; + let rect = match (shape, text_box_of(&d)) { + (BorderShape::Rectangle, _) | (_, None) => rect, + // A circle's text box keeps its proportions and reaches the circle with its longer sides; + // the circle then goes around it again, never closer than its text allows. + (BorderShape::Circle, Some(was)) => { + let (o, w) = (normalize(rect), normalize(was)); + let (cx, cy, r) = ((o[0] + o[2]) / 2.0, (o[1] + o[3]) / 2.0, (o[2] - o[0]).min(o[3] - o[1]) / 2.0); + let (wx, wy) = ((w[2] - w[0]) / 2.0, (w[3] - w[1]) / 2.0); + let k = r / wx.max(wy).max(1e-6); + let (ix, iy) = ((wx * k).max(0.5), (wy * k).max(0.5)); + [cx - ix, cy - iy, cx + ix, cy + iy] + } + (shape, Some(was)) => shape.inside(rect, border_width_of(&d), was), + }; apply_text_box(doc, r, rect, meta, false) } +/// A FreeText's text box: `/Rect` inset by `/RD` for a callout or a text box inside an oval or a +/// circle, else `/Rect`. `None` when they can't be read. +pub(crate) fn text_box_of(d: &Dict) -> Option<[f64; 4]> { + let nums = |key: &[u8]| -> Option> { d.get(key)?.as_array()?.iter().map(|o| o.as_f64()).collect() }; + let rect = normalize(four(&nums(b"Rect")?)?); + if !d.contains(b"CL") && BorderShape::of(d) == BorderShape::Rectangle { + return Some(rect); + } + let rd = appearance::inset(d)?; + let tb = [rect[0] + rd[0], rect[1] + rd[1], rect[2] - rd[2], rect[3] - rd[3]]; + (tb[2] - tb[0] >= 1.0 && tb[3] - tb[1] >= 1.0).then_some(tb) +} + +/// Give text box or typewriter text `d`, whose text box is `tb`, the border `shape` (`width` +/// wide): `/Rect` around it, `/RD` and `/PCBorderShape`, or for a rectangle `/Rect` the text +/// box itself again. +pub(crate) fn shape_text_box(d: &mut Dict, tb: [f64; 4], shape: BorderShape, width: f64) { + // A circle goes around the text as the box lays it out. + let text = (shape == BorderShape::Circle).then(|| text_extent(d, displayed_sides(d, tb).0 - 2.0 * (2.0 + width.max(0.0)))); + let outer = shape.around(tb, width, text); + d.set(b"Rect".to_vec(), num_array(&outer)); + if shape == BorderShape::Rectangle { + d.remove(b"PCBorderShape"); + d.remove(b"RD"); + } else { + d.set(b"PCBorderShape".to_vec(), Object::name(shape.name())); + // A circle touching the text box's sides would put them a rounding error outside it. + let tb = normalize(tb); + let rd = [tb[0] - outer[0], tb[1] - outer[1], outer[2] - tb[2], outer[3] - tb[3]].map(|x| x.max(0.0)); + d.set(b"RD".to_vec(), num_array(&rd)); + } +} + +/// The width of the widest line and the height of the lines of FreeText `d`'s text, as its +/// appearance lays the text out `wrap` wide. +fn text_extent(d: &Dict, wrap: f64) -> [f64; 2] { + let text = d.get(b"Contents").and_then(|o| o.as_string()).map(PdfString::to_text).unwrap_or_default(); + let (_, size) = appearance::parse_da(d); + let lines = appearance::wrap(&text, size, wrap.max(1.0)); + let widest = lines.iter().map(|l| appearance::text_width(l, size)).fold(0.0, f64::max); + [widest, lines.len().max(1) as f64 * size * 1.2] +} + +/// The width and height of FreeText `d`'s text box `tb` as displayed: swapped on a page turned by +/// a quarter, where its text is laid out upright. +fn displayed_sides(d: &Dict, tb: [f64; 4]) -> (f64, f64) { + let tb = normalize(tb); + let (w, h) = (tb[2] - tb[0], tb[3] - tb[1]); + if matches!(appearance::text_turn(d), 90 | 270) { (h, w) } else { (w, h) } +} + +/// FreeText `d`'s text box `tb`, with a border `width` wide, made as large as its text wrapped +/// `wrap` wide needs (its widest line and its lines, inside the padding a box sized to its text +/// has) and centred where it was: a circle goes around this, not around a box wider than its text. +fn snug_text_box(d: &Dict, tb: [f64; 4], width: f64, wrap: f64) -> [f64; 4] { + let tb = normalize(tb); + let pad = 2.0 + width.max(0.0); + let [w, h] = text_extent(d, wrap); + let (hx, hy) = (w / 2.0 + pad + 2.0, h / 2.0 + pad + 1.0); + let (hx, hy) = if matches!(appearance::text_turn(d), 90 | 270) { (hy, hx) } else { (hx, hy) }; + let (cx, cy) = ((tb[0] + tb[2]) / 2.0, (tb[1] + tb[3]) / 2.0); + [cx - hx, cy - hy, cx + hx, cy + hy] +} + +/// Whether FreeText `d` is a circle whose text box is snug around its text ([`snug_text_box`]): +/// it then follows its text as that changes, where one made larger by hand keeps its size. +fn is_snug(d: &Dict) -> bool { + let Some(tb) = text_box_of(d).filter(|_| BorderShape::of(d) == BorderShape::Circle) else { return false }; + let width = border_width_of(d); + let s = snug_text_box(d, tb, width, displayed_sides(d, tb).0 - 2.0 * (2.0 + width)); + let tb = normalize(tb); + ((s[2] - s[0]) - (tb[2] - tb[0])).abs() < 0.5 && ((s[3] - s[1]) - (tb[3] - tb[1])).abs() < 0.5 +} + +/// The text box of a circle snug around its text for the text and size it has now: its lines up to +/// 300 pt wide, centred where it was. +fn snug_refit(doc: &Document, r: ObjRef) -> Option<[f64; 4]> { + let d = annot_dict(doc, r); + let width = border_width_of(&d); + Some(snug_text_box(&d, text_box_of(&d)?, width, 300.0 - 2.0 * (2.0 + width))) +} + +/// Give a text box or typewriter text an oval, a circle or a rectangle for its border, and redraw +/// it. Its text box keeps its place; the oval or circle goes around it. For a circle the text box +/// first shrinks to its text, its lines wrapped as they are. +pub fn set_border_shape(doc: &mut Document, page: usize, index: usize, shape: BorderShape, meta: &Meta) -> Result<(), AnnotError> { + let (_, r) = annot_ref(doc, page, index)?; + unlocked(doc, r)?; + let d = annot_dict(doc, r); + if d.name(b"Subtype") != Some(b"FreeText") || d.contains(b"CL") { + return Err(AnnotError::Invalid("only text boxes and typewriter text have a border shape".into())); + } + let tb = text_box_of(&d).ok_or_else(|| AnnotError::Invalid("the text box can't be read".into()))?; + let tb = match shape { + BorderShape::Circle => { + let width = border_width_of(&d); + snug_text_box(&d, tb, width, displayed_sides(&d, tb).0 - 2.0 * (2.0 + width)) + } + _ => tb, + }; + refuse_foreign_text(doc, &d)?; + let before = d; + doc.update_dict(r, |d| { + shape_text_box(d, tb, shape, border_width_of(d)); + touch(d, meta); + })?; + redraw_or_restore(doc, r, before) +} + /// At most this many points reshape a comment (a drawn polygon has a handful). const MAX_POINTS: usize = 100_000; @@ -503,6 +640,8 @@ fn apply_text_box(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta, ne } None => (rect, None), }; + let (shape, width) = (BorderShape::of(&d), border_width_of(&d)); + let free_text = d.name(b"Subtype") == Some(b"FreeText"); doc.update_dict(r, |d| { d.set(b"Rect".to_vec(), num_array(&outer)); match &cl { @@ -510,6 +649,8 @@ fn apply_text_box(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta, ne d.set(b"CL".to_vec(), num_array(l)); d.set(b"RD".to_vec(), num_array(&[rect[0] - outer[0], rect[1] - outer[1], outer[2] - rect[2], outer[3] - rect[3]])); } + // An oval or a circle goes around the text box again. + None if free_text => shape_text_box(d, rect, shape, width), None => { d.remove(b"RD"); } @@ -528,6 +669,8 @@ fn apply_text_box(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta, ne } } +/// The border width as the appearance draws it: `/BS /W`, else the third number of `/Border` +/// (another app's), else 1. fn border_width_of(d: &Dict) -> f64 { appearance::border_width(d) } @@ -558,7 +701,8 @@ fn ink_rect_for_width(d: &Dict, old_w: f64, w: f64) -> Option<[f64; 4]> { Some([inner[0] - base[0] - new_half, inner[1] - base[1] - new_half, inner[2] + base[2] + new_half, inner[3] + base[3] + new_half]) } -/// Change a comment's colour, opacity, line width and/or line endings, and redraw it. +/// Change a comment's colour, opacity (of the whole comment), line width and/or line endings, and +/// redraw it. pub fn set_style(doc: &mut Document, page: usize, index: usize, change: StyleChange, meta: &Meta) -> Result<(), AnnotError> { let StyleChange { color, opacity, width, endings } = change; let (_, r) = annot_ref(doc, page, index)?; @@ -581,25 +725,34 @@ pub fn set_style(doc: &mut Document, page: usize, index: usize, change: StyleCha } refuse_foreign_text(doc, &d)?; let free_text = subtype == "FreeText"; + // Typewriter text has no box of its own: a border goes around its text, so the box grows + // (or shrinks) by the change on every side and the text keeps its place and its lines. + // So does a text box inside an oval or a circle, which goes around it again. + let shape = if free_text { BorderShape::of(&d) } else { BorderShape::Rectangle }; + let typewriter = free_text && (d.name(b"IT") == Some(b"FreeTextTypeWriter") || shape != BorderShape::Rectangle) && !d.contains(b"CL"); + let regrow = width.filter(|_| typewriter).and_then(|w| { + let by = w.max(0.0) - if d.contains(b"BS") || d.contains(b"Border") { border_width_of(&d) } else { 0.0 }; + let rect = grow(text_box_of(&d)?, by); + (by != 0.0 && finite(&rect) && rect[2] - rect[0] >= 1.0 && rect[3] - rect[1] >= 1.0).then_some((rect, w.max(0.0))) + }); let before = d; doc.update_dict(r, |d| { + if let Some((rect, w)) = regrow { + shape_text_box(d, rect, shape, w); + } if let Some(c) = color { if free_text { let (_, size) = appearance::parse_da(d); - let [r, g, b] = c.map(|x| x.clamp(0.0, 1.0)); - d.set(b"DA".to_vec(), PdfString::literal(format!("{} {} {} rg /Helv {} Tf", n(r), n(g), n(b), n(size)).into_bytes())); + d.set(b"DA".to_vec(), free_text_da(c, appearance::da_border(d), size)); rich_text(d); } else { d.set(b"C".to_vec(), rgb(c)); } } + // The whole comment: its fill and text drop opacities of their own. if let Some(o) = opacity { let o = o.clamp(0.0, 1.0); - if o < 1.0 { - d.set(b"CA".to_vec(), Object::Real(o)); - } else { - d.remove(b"CA"); - } + set_part_keys(d, appearance::PartOpacity { line: o, fill: o, text: o }); } if let Some(w) = width { let old_w = border_width_of(d); @@ -629,6 +782,37 @@ pub fn set_style(doc: &mut Document, page: usize, index: usize, change: StyleCha redraw_or_restore(doc, r, before) } +/// A FreeText's `/DA`: the text colour, the border's own colour when it has one (the stroking +/// colour, `RG`), Helvetica at `size`. +pub(crate) fn free_text_da(text: Rgb, border: Option, size: f64) -> PdfString { + let [r, g, b] = text.map(|x| x.clamp(0.0, 1.0)); + let stroke = border.map(|c| c.map(|x| x.clamp(0.0, 1.0))).map(|[r, g, b]| format!("{} {} {} RG ", n(r), n(g), n(b))).unwrap_or_default(); + PdfString::literal(format!("{} {} {} rg {stroke}/Helv {} Tf", n(r), n(g), n(b), n(size)).into_bytes()) +} + +/// Give a text box, typewriter text or callout a border colour of its own (`None`: the text's +/// again), and redraw it. +pub fn set_border_color(doc: &mut Document, page: usize, index: usize, color: Option, meta: &Meta) -> Result<(), AnnotError> { + let (_, r) = annot_ref(doc, page, index)?; + unlocked(doc, r)?; + let d = annot_dict(doc, r); + if d.name(b"Subtype") != Some(b"FreeText") { + return Err(AnnotError::Invalid("only text boxes, typewriter text and callouts have a border colour".into())); + } + if color.is_some_and(|c| !finite(&c)) { + return Err(AnnotError::Invalid("invalid colour".into())); + } + refuse_foreign_text(doc, &d)?; + let before = d; + doc.update_dict(r, |d| { + let (text, size) = appearance::parse_da(d); + d.set(b"DA".to_vec(), free_text_da(text, color, size)); + rich_text(d); + touch(d, meta); + })?; + redraw_or_restore(doc, r, before) +} + /// Change the font size of a text box, typewriter text or callout (`/DA` keeps its colour; the /// rich-text strings follow) and redraw it. The box grows to hold the wrapped text at the new /// size and never shrinks, so a box the user drew large stays large. Sizes are 1 to 400 pt. @@ -643,26 +827,27 @@ pub fn set_font_size(doc: &mut Document, page: usize, index: usize, size: f64, m return Err(AnnotError::Invalid("font size must be 1\u{2013}400 pt".into())); } refuse_foreign_text(doc, &d)?; + let snug = is_snug(&d); let before = d; doc.update_dict(r, |d| { let (col, _) = appearance::parse_da(d); - d.set(b"DA".to_vec(), PdfString::literal(format!("{} {} {} rg /Helv {} Tf", n(col[0]), n(col[1]), n(col[2]), n(size)).into_bytes())); + d.set(b"DA".to_vec(), free_text_da(col, appearance::da_border(d), size)); rich_text(d); touch(d, meta); })?; let text = text_value(doc, &annot_dict(doc, r), b"Contents").unwrap_or_default(); - let result = match fitted_box(doc, r, &text) { - Some(fit) => { + let result = match (snug, if snug { snug_refit(doc, r) } else { fitted_box(doc, r, &text) }) { + // A circle snug around its text stays snug around it at the new size. + (true, Some(fit)) => apply_text_box(doc, r, fit, meta, false), + (false, Some(fit)) => { let cur = annot_dict(doc, r); - let nums: Vec = - cur.get(b"Rect").and_then(|o| o.as_array()).map(|a| a.iter().filter_map(|x| x.as_f64()).collect()).unwrap_or_default(); // Keep the top and sides; only the bottom edge moves, and only downwards. - match four(&nums).map(normalize) { + match text_box_of(&cur) { Some(old) if !cur.contains(b"CL") => apply_text_box(doc, r, [fit[0], fit[1].min(old[1]), fit[2], fit[3]], meta, false), _ => apply_text_box(doc, r, fit, meta, false), } } - None => redraw_or_restore(doc, r, before.clone()), + (_, None) => redraw_or_restore(doc, r, before.clone()), }; if result.is_err() { doc.set(r, Object::Dict(before)); @@ -707,6 +892,63 @@ pub fn set_fill(doc: &mut Document, page: usize, index: usize, fill: Option redraw_or_restore(doc, r, before) } +/// Draw a rectangle's, oval's, polygon's or text box's fill, or a text box's text, at an +/// opacity of its own (0–1), its line or border keeping the comment's (`/CA`), and redraw it. +/// `None` leaves that part as it is drawn now. The fill goes in `/ca` (PDF 2.0's opacity for +/// what is filled), the text in `/PCTextOpacity`, each only where it differs from the one it +/// would otherwise follow. +pub fn set_part_opacity(doc: &mut Document, page: usize, index: usize, fill: Option, text: Option, meta: &Meta) -> Result<(), AnnotError> { + let (_, r) = annot_ref(doc, page, index)?; + unlocked(doc, r)?; + let d = annot_dict(doc, r); + let subtype = String::from_utf8_lossy(d.name(b"Subtype").unwrap_or_default()).into_owned(); + if fill.is_some() && !fillable(&subtype) { + return Err(AnnotError::Invalid("only rectangles, ovals, polygons and text boxes have a fill of their own".into())); + } + if text.is_some() && subtype != "FreeText" { + return Err(AnnotError::Invalid("only text boxes, typewriter text and callouts have text of their own".into())); + } + if fill.into_iter().chain(text).any(|o| !o.is_finite()) { + return Err(AnnotError::Invalid("invalid opacity".into())); + } + // Check before changing anything: a stale appearance would contradict the new opacity. + if appearance::build(&d).is_none() { + return Err(AnnotError::Unsupported(subtype)); + } + refuse_foreign_text(doc, &d)?; + let before = d; + doc.update_dict(r, |d| { + let mut parts = appearance::part_opacity(d); + if let Some(f) = fill { + parts.fill = f.clamp(0.0, 1.0); + } + if let Some(t) = text { + parts.text = t.clamp(0.0, 1.0); + } + set_part_keys(d, parts); + touch(d, meta); + })?; + redraw_or_restore(doc, r, before) +} + +/// Write a comment's parts' opacities (see [`appearance::PartOpacity`]): the line's as `/CA`, the +/// fill's as `/ca` and the text's as `/PCTextOpacity`, each only where it differs from the one +/// it would otherwise follow (the fill the line, the text the fill), so a comment drawn at one +/// opacity keeps just `/CA`. +pub(crate) fn set_part_keys(d: &mut Dict, parts: appearance::PartOpacity) { + let appearance::PartOpacity { line, fill, text } = parts; + let mut put = |key: &[u8], value: f64, unless: f64| { + if value == unless { + d.remove(key); + } else { + d.set(key.to_vec(), Object::Real(value)); + } + }; + put(b"CA", line, 1.0); + put(b"ca", fill, line); + put(b"PCTextOpacity", text, fill); +} + /// Grow a line or polyline's rectangle so a non-`None` ending is not clipped. Idempotent. fn ensure_ending_room(d: &mut Dict) { let subtype = d.name(b"Subtype").unwrap_or_default(); diff --git a/crates/annot/src/tests.rs b/crates/annot/src/tests.rs index 32d28fc6ec..ff7f8925f0 100644 --- a/crates/annot/src/tests.rs +++ b/crates/annot/src/tests.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use pdfcraft_cos::{Document, ObjRef, Object, SaveOptions, write_incremental}; +use super::modify::text_box_of; use super::*; /// Two pages. Page 1 has an inline Square annotation in a direct /Annots array; page 2 has an @@ -40,7 +41,7 @@ fn new(page: usize, shape: Shape) -> NewAnnotation { NewAnnotation { page, shape, - style: Style { color: [0.0, 0.4, 1.0], opacity: 1.0, width: 2.0, fill: None }, + style: Style { color: [0.0, 0.4, 1.0], opacity: 1.0, width: 2.0, fill: None, ..Style::default() }, contents: "hello".into(), author: "Ada".into(), } @@ -356,6 +357,296 @@ fn restyle_and_resize_redraw_the_appearance() { assert!(matches!(set_rect(&mut doc, 0, l, [0.0, 0.0, 5.0, 5.0], &meta("")), Err(AnnotError::Invalid(_)))); } +/// Typed text or a text box takes an oval or a circle for its border: the text box keeps its +/// place and the shape goes around it (`/Rect` grows, `/RD` holds the text box, `/PCBorderShape` +/// the shape); back to a rectangle, it is the text box it was. A resized frame takes its text box +/// along, a thicker border leaves it where it is. +#[test] +fn text_boxes_take_an_oval_or_a_circle_around_their_text() { + let close = |a: &[f64], b: &[f64]| a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (x - y).abs() < 0.01); + let nums = |d: &Dict, k: &[u8]| -> Vec { + d.get(k).and_then(|o| o.as_array()).map(|a| a.iter().filter_map(|x| x.as_f64()).collect()).unwrap_or_default() + }; + // Where the text starts: its first line's text matrix. + let text_at = |doc: &Document, d: &Dict| ap_content(doc, d).lines().find(|l| l.ends_with(" Tm") || l.contains(" Tm ")).map(str::to_owned); + let mut doc = fixture(); + let mut tw = new(0, Shape::Typewriter { rect: [100.0, 100.0, 200.0, 120.0], font_size: 10.0 }); + tw.style.width = 1.0; + let t = add_annotation(&mut doc, &tw, &meta("t")).unwrap(); + // In an oval or a circle the text is centred in its text box (left-aligned in a box): one + // line of "hello" at 10 pt in the 100 × 20 text box centred on (150, 110), 3 pt from its top. + let centred = format!("1 0 0 1 {} {} Tm (hello) Tj", n(150.0 - appearance::text_width("hello", 10.0) / 2.0), n(107.0)); + assert!(text_at(&doc, &list(&doc, 0)[t]).is_some_and(|l| l.starts_with("1 0 0 1 103 ")), "a box: left-aligned"); + set_border_shape(&mut doc, 0, t, BorderShape::Oval, &meta("")).unwrap(); + let doc2 = reopen(&doc); + let d = &list(&doc2, 0)[t]; + // Around the 100 × 20 text box, centred: √2 its size, two points more for the 1 pt border. + let (ax, ay) = (50.0 * std::f64::consts::SQRT_2 + 2.0, 10.0 * std::f64::consts::SQRT_2 + 2.0); + assert!(close(&rect(d), &[150.0 - ax, 110.0 - ay, 150.0 + ax, 110.0 + ay]), "{:?}", rect(d)); + assert!(close(&nums(d, b"RD"), &[ax - 50.0, ay - 10.0, ax - 50.0, ay - 10.0]), "{:?}", nums(d, b"RD")); + assert_eq!(d.name(b"PCBorderShape"), Some(&b"Oval"[..])); + let ap = ap_content(&doc2, d); + assert!(ap.contains(" c\n") && ap.contains("S\n") && !ap.contains("re S"), "an oval, not a box: {ap}"); + assert_eq!(text_at(&doc2, d).as_deref(), Some(centred.as_str()), "the text keeps its place"); + assert_eq!(props(&doc2, 0, t).unwrap().border_shape, Some(BorderShape::Oval)); + // A circle around the text: the text box shrinks to "hello" with its padding, centred where it + // was, and the circle touches its sides (the text a point clear of the border inside). + set_border_shape(&mut doc, 0, t, BorderShape::Circle, &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + let hw = appearance::text_width("hello", 10.0) / 2.0; + let r = (hw + 5.0).max(hw.hypot(6.0) + 2.0); + assert!(close(&rect(d), &[150.0 - r, 110.0 - r, 150.0 + r, 110.0 + r]), "{:?}", rect(d)); + assert!(close(&nums(d, b"RD"), &[r - hw - 5.0, r - 10.0, r - hw - 5.0, r - 10.0]), "{:?}", nums(d, b"RD")); + assert_eq!(text_at(&doc, d).as_deref(), Some(centred.as_str())); + // A thicker border: the typed text's box grows by the change on every side, so its text keeps + // its place inside the thicker line (as with a rectangle), and the circle goes around that. + set_style(&mut doc, 0, t, StyleChange::new(None, None, Some(3.0), None), &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + assert_eq!(text_at(&doc, d).as_deref(), Some(centred.as_str())); + let r3 = (hw + 7.0).max(hw.hypot(6.0) + 4.0); + assert!(close(&rect(d), &[150.0 - r3, 110.0 - r3, 150.0 + r3, 110.0 + r3]), "{:?}", rect(d)); + // Its frame resized: the text box follows inside, keeping its proportions, its longer sides + // reaching the circle. + set_rect(&mut doc, 0, t, [150.0 - 2.0 * r3, 110.0 - 2.0 * r3, 150.0 + 2.0 * r3, 110.0 + 2.0 * r3], &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + let rd = nums(d, b"RD"); + let tb = [rect(d)[0] + rd[0], rect(d)[1] + rd[1], rect(d)[2] - rd[2], rect(d)[3] - rd[3]]; + assert!(((tb[2] - tb[0]) / (tb[3] - tb[1]) - (hw + 7.0) / 12.0).abs() < 0.01, "the text box keeps its proportions: {tb:?}"); + assert!(close(&rect(d), &[150.0 - 2.0 * r3, 110.0 - 2.0 * r3, 150.0 + 2.0 * r3, 110.0 + 2.0 * r3]), "{:?}", rect(d)); + // A rectangle again: the text box it has, nothing around it. + set_border_shape(&mut doc, 0, t, BorderShape::Rectangle, &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + assert!(close(&rect(d), &tb), "{:?} vs {tb:?}", rect(d)); + assert!(!d.contains(b"RD") && !d.contains(b"PCBorderShape")); + assert!(ap_content(&doc, d).contains("re S")); + + // A filled text box made with an oval: the background takes the shape too. + let mut tb = new(0, Shape::TextBox { rect: [300.0, 500.0, 400.0, 540.0], font_size: 12.0 }); + (tb.style.fill, tb.style.width, tb.style.border_shape) = (Some([1.0, 1.0, 0.6]), 1.0, BorderShape::Oval); + let b = add_annotation(&mut doc, &tb, &meta("b")).unwrap(); + let d = &list(&doc, 0)[b]; + assert_eq!(d.name(b"PCBorderShape"), Some(&b"Oval"[..])); + let ap = ap_content(&doc, d); + assert!(ap.contains("1 1 0.6 rg") && ap.contains(" c\nh\nf\n") && !ap.contains(" re f"), "{ap}"); + // A callout's border stays a rectangle. + let c = add_annotation( + &mut doc, + &new( + 0, + Shape::Callout { + rect: [300.0, 300.0, 400.0, 340.0], + knee: [280.0, 320.0], + point: [250.0, 280.0], + ending: LineEnding::OpenArrow, + font_size: 10.0, + }, + ), + &meta("c"), + ) + .unwrap(); + assert!(matches!(set_border_shape(&mut doc, 0, c, BorderShape::Oval, &meta("")), Err(AnnotError::Invalid(_)))); + assert_eq!(props(&doc, 0, c).unwrap().border_shape, None); + assert_eq!(BorderShape::parse("CIRCLE"), Some(BorderShape::Circle)); +} + +/// A circle goes snug around short text, as a number on a drawing is circled: the text box is no +/// wider than its text needs and the circle touches its sides, where one through the box's corners +/// stood well clear. It follows the text and its size as they change; one made larger by hand +/// keeps its size. +#[test] +fn a_circle_goes_snug_around_its_text_and_follows_it() { + let close = |a: (f64, f64), b: (f64, f64)| (a.0 - b.0).abs() < 0.01 && (a.1 - b.1).abs() < 0.01; + let mut doc = fixture(); + // "3" at 10 pt with a 1 pt border, in the box Fill & Sign sizes to it: the digit with 2 pt of + // padding and 4 pt to spare across, a line and 6 pt down, the border around that. + let w3 = appearance::text_width("3", 10.0); + let mut typed = new(0, Shape::Typewriter { rect: [100.0, 680.0, 110.0 + w3, 700.0], font_size: 10.0 }); + (typed.contents, typed.style.width, typed.style.border_shape) = ("3".into(), 1.0, BorderShape::Circle); + let t = add_annotation(&mut doc, &typed, &meta("t")).unwrap(); + let frame = |doc: &Document| { + let r = rect(&list(doc, 0)[t]); + ((r[2] - r[0], r[3] - r[1]), ((r[0] + r[2]) / 2.0, (r[1] + r[3]) / 2.0)) + }; + let centre = (105.0 + w3 / 2.0, 690.0); + // As tall as the box, 20 pt across: through the corners it was 29. + assert!(close(frame(&doc).0, (20.0, 20.0)) && close(frame(&doc).1, centre), "{:?}", frame(&doc)); + // The text box Fill & Sign gives a box to type into, 40 pt wide: a circle goes snug around its + // text there too. + let mut wide = typed.clone(); + wide.shape = Shape::Typewriter { rect: [300.0, 680.0, 342.0, 700.0], font_size: 10.0 }; + wide.style.border_shape = BorderShape::Rectangle; + let b = add_annotation(&mut doc, &wide, &meta("b")).unwrap(); + set_border_shape(&mut doc, 0, b, BorderShape::Circle, &meta("")).unwrap(); + let r = rect(&list(&doc, 0)[b]); + assert!(close((r[2] - r[0], r[3] - r[1]), (20.0, 20.0)), "{r:?}"); + // More text: the circle grows with it, about the same centre; less, and it shrinks back. + set_contents(&mut doc, 0, t, "123", &meta("")).unwrap(); + let w = appearance::text_width("123", 10.0) / 2.0; + let d = 2.0 * (w + 5.0).max(10.0).max(w.hypot(6.0) + 2.0); + assert!(close(frame(&doc).0, (d, d)) && close(frame(&doc).1, centre), "{:?}", frame(&doc)); + set_contents(&mut doc, 0, t, "3", &meta("")).unwrap(); + assert!(close(frame(&doc).0, (20.0, 20.0)) && close(frame(&doc).1, centre), "{:?}", frame(&doc)); + // A larger font: a larger circle, still snug. + set_font_size(&mut doc, 0, t, 20.0, &meta("")).unwrap(); + let w = appearance::text_width("3", 20.0) / 2.0; + let d = 2.0 * (w + 5.0).max(16.0).max(w.hypot(12.0) + 2.0); + assert!(close(frame(&doc).0, (d, d)) && close(frame(&doc).1, centre), "{:?}", frame(&doc)); + set_font_size(&mut doc, 0, t, 10.0, &meta("")).unwrap(); + assert!(close(frame(&doc).0, (20.0, 20.0)), "{:?}", frame(&doc)); + // Made larger by hand, it keeps its size as its text changes. + set_rect(&mut doc, 0, t, [centre.0 - 20.0, centre.1 - 20.0, centre.0 + 20.0, centre.1 + 20.0], &meta("")).unwrap(); + assert!(close(frame(&doc).0, (40.0, 40.0)), "{:?}", frame(&doc)); + set_contents(&mut doc, 0, t, "4", &meta("")).unwrap(); + assert!(close(frame(&doc).0, (40.0, 40.0)) && close(frame(&doc).1, centre), "{:?}", frame(&doc)); +} + +/// A circle around a line wider than it is tall touches its text box's sides: `/RD` is 0 there, +/// not a rounding error below it, and a hair below 0 still reads. (A text box that couldn't be +/// read stopped following its text, and its text wrapped in a narrower one.) The circle keeps its +/// centre as its text changes. +#[test] +fn a_circle_touching_its_text_box_keeps_following_its_text() { + let mut doc = fixture(); + // Sized to its text: 2 pt of padding and half a point to spare across, a line down. + let (w, h) = (appearance::text_width("Testtext", 12.0) + 4.5, 12.0 * 1.11 + 4.0); + let mut b = new(0, Shape::TextBox { rect: [210.79, 700.0 - h, 210.79 + w, 700.0], font_size: 12.0 }); + (b.contents, b.style.width) = ("Testtext".into(), 0.0); + let t = add_annotation(&mut doc, &b, &meta("t")).unwrap(); + // Circle, then a 1 pt border, which the circle grows by. + set_border_shape(&mut doc, 0, t, BorderShape::Circle, &meta("")).unwrap(); + set_style(&mut doc, 0, t, StyleChange::new(None, None, Some(1.0), None), &meta("")).unwrap(); + let lines = |doc: &Document| ap_content(doc, &list(doc, 0)[t]).matches(" Tj").count(); + let frame = |doc: &Document| { + let r = rect(&list(doc, 0)[t]); + ((r[2] - r[0], r[3] - r[1]), ((r[0] + r[2]) / 2.0, (r[1] + r[3]) / 2.0)) + }; + let rd = |doc: &Document| -> Vec { list(doc, 0)[t].get(b"RD").unwrap().as_array().unwrap().iter().map(|o| o.as_f64().unwrap()).collect() }; + assert!(rd(&doc).iter().all(|x| *x >= 0.0) && rd(&doc)[0] < 0.01, "touching the sides: {:?}", rd(&doc)); + assert_eq!(lines(&doc), 1, "Testtext on one line"); + let (wide, centre) = frame(&doc); + set_contents(&mut doc, 0, t, "KRS", &meta("")).unwrap(); + let (small, at) = frame(&doc); + assert!(small.0 < wide.0 - 10.0 && (small.0 - small.1).abs() < 0.01, "a smaller circle: {small:?} from {wide:?}"); + assert!((at.0 - centre.0).abs() < 0.01 && (at.1 - centre.1).abs() < 0.01, "about its centre: {at:?} {centre:?}"); + assert_eq!(lines(&doc), 1); + // A hair below 0, as rounding left it: still read, so the circle follows its text again. + let (_, r) = annot_ref(&mut doc, 0, t).unwrap(); + let mut was = rd(&doc); + was[0] = -2.8e-14; + doc.update_dict(r, |d| d.set(b"RD".to_vec(), num_array(&was))).unwrap(); + assert!(text_box_of(&list(&doc, 0)[t]).is_some()); + set_contents(&mut doc, 0, t, "Testtext", &meta("")).unwrap(); + assert!((frame(&doc).0.0 - wide.0).abs() < 0.01 && lines(&doc) == 1, "{:?}", frame(&doc)); + assert!(rd(&doc).iter().all(|x| *x >= 0.0), "{:?}", rd(&doc)); +} + +/// On a page turned a quarter, typed text is laid out as the page is shown (`/PCTextRotate`), and a +/// circle made with it goes around the text as shown: as large as around the same text on an +/// upright page. (Made before the turn was set, its text wrapped at the box's displayed height and +/// the circle went around several lines.) +#[test] +fn a_circle_made_on_a_turned_page_is_sized_as_on_an_upright_one() { + let w = appearance::text_width("12345", 10.0); + let typed = |rect: [f64; 4]| { + let mut t = new(0, Shape::Typewriter { rect, font_size: 10.0 }); + (t.contents, t.style.width, t.style.border_shape) = ("12345".into(), 1.0, BorderShape::Circle); + t + }; + let size = |doc: &Document, i: usize| { + let r = rect(&list(doc, 0)[i]); + (r[2] - r[0], r[3] - r[1]) + }; + let mut upright = fixture(); + let u = add_annotation(&mut upright, &typed([100.0, 680.0, 110.0 + w, 700.0]), &meta("u")).unwrap(); + for turn in [90, 270] { + let mut doc = fixture(); + let page = page_refs(&doc).unwrap()[0]; + doc.update_dict(page, |d| d.set(b"Rotate".to_vec(), Object::Int(turn))).unwrap(); + // The same box as shown: its sides swapped in user space. + let t = add_annotation(&mut doc, &typed([100.0, 680.0, 120.0, 690.0 + w]), &meta("t")).unwrap(); + assert_eq!(list(&doc, 0)[t].get(b"PCTextRotate").and_then(Object::as_int), Some(turn)); + let (turned, straight) = (size(&doc, t), size(&upright, u)); + assert!((turned.0 - straight.0).abs() < 0.01 && (turned.1 - straight.1).abs() < 0.01, "{turn}°: {turned:?}, upright {straight:?}"); + } +} + +/// A rectangle's fill, and a text box's border, background and text, each drawn at an opacity of +/// its own: the line's in `/CA`, the fill's in `/ca` (PDF 2.0), the text's in `/PCTextOpacity`, and +/// in the appearance `/GS0` (strokes at the line's, fills at the fill's) and `/GS1` for the text. +#[test] +fn parts_are_drawn_at_opacities_of_their_own() { + let num = |d: &Dict, k: &[u8]| d.get(k).and_then(|o| o.as_f64()); + let gstate = |doc: &Document, d: &Dict, name: &[u8]| -> Option<(f64, f64)> { + let n = d.get(b"AP").map(|a| doc.resolve(a)).and_then(|a| a.as_dict().and_then(|a| a.reference(b"N")))?; + let obj = doc.get(n); + let Object::Stream(s) = &*obj else { return None }; + let res = s.dict.get(b"Resources").map(|r| doc.resolve(r))?; + let gs = res.as_dict()?.get(b"ExtGState").map(|g| doc.resolve(g))?; + let gs = gs.as_dict()?.get(name).map(|g| doc.resolve(g))?; + let gs = gs.as_dict()?; + Some((gs.get(b"CA")?.as_f64()?, gs.get(b"ca")?.as_f64()?)) + }; + let mut doc = fixture(); + let r = add_annotation(&mut doc, &new(0, Shape::Rectangle { rect: [10.0, 10.0, 110.0, 60.0] }), &meta("r")).unwrap(); + set_fill(&mut doc, 0, r, Some([1.0, 1.0, 0.0]), &meta("")).unwrap(); + set_part_opacity(&mut doc, 0, r, Some(0.3), None, &meta("")).unwrap(); + let doc2 = reopen(&doc); + let d = &list(&doc2, 0)[r]; + assert_eq!((num(d, b"CA"), num(d, b"ca")), (None, Some(0.3)), "an opaque line, a fill of its own"); + assert_eq!(gstate(&doc2, d, b"GS0"), Some((1.0, 0.3))); + let ap = ap_content(&doc2, d); + assert!(ap.starts_with("/GS0 gs") && ap.contains("\nB\n"), "filled and stroked in one go: {ap}"); + let p = props(&doc2, 0, r).unwrap(); + assert_eq!((p.opacity, p.fill_opacity, p.text_opacity), (1.0, Some(0.3), None)); + // The whole comment's opacity (StyleAnnotation, the quick bar): the fill goes along again. + set_style(&mut doc, 0, r, StyleChange::new(None, Some(0.6), None, None), &meta("")).unwrap(); + let d = &list(&doc, 0)[r]; + assert_eq!((num(d, b"CA"), num(d, b"ca")), (Some(0.6), None)); + assert_eq!(props(&doc, 0, r).unwrap().fill_opacity, Some(0.6)); + + // A text box with its border opaque, its background at 20 % and its text opaque, as made. + let mut tb = new(0, Shape::TextBox { rect: [200.0, 600.0, 400.0, 650.0], font_size: 12.0 }); + tb.style.fill = Some([1.0, 1.0, 0.6]); + tb.style.width = 1.0; + (tb.style.fill_opacity, tb.style.text_opacity) = (Some(0.2), Some(1.0)); + let t = add_annotation(&mut doc, &tb, &meta("t")).unwrap(); + let doc2 = reopen(&doc); + let d = &list(&doc2, 0)[t]; + assert_eq!((num(d, b"CA"), num(d, b"ca"), num(d, b"PCTextOpacity")), (None, Some(0.2), Some(1.0))); + assert_eq!((gstate(&doc2, d, b"GS0"), gstate(&doc2, d, b"GS1")), (Some((1.0, 0.2)), Some((1.0, 1.0)))); + let ap = ap_content(&doc2, d); + let (gs1, text) = (ap.find("/GS1 gs\n").expect("the text's own state, an operator of its own"), ap.find("BT").expect("text")); + assert!(ap.find(" re f").is_some_and(|fill| fill < gs1) && gs1 < text, "background at 20 %, then the text at its own: {ap}"); + // A new fill opacity leaves the text as it is drawn. + set_part_opacity(&mut doc, 0, t, Some(0.5), None, &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + assert_eq!((num(d, b"ca"), num(d, b"PCTextOpacity")), (Some(0.5), Some(1.0))); + let p = props(&doc, 0, t).unwrap(); + assert_eq!((p.opacity, p.fill_opacity, p.text_opacity), (1.0, Some(0.5), Some(1.0))); + // All at one opacity again: only /CA stays. + set_part_opacity(&mut doc, 0, t, Some(1.0), Some(1.0), &meta("")).unwrap(); + let d = &list(&doc, 0)[t]; + assert_eq!((num(d, b"CA"), num(d, b"ca"), num(d, b"PCTextOpacity")), (None, None, None)); + + // Only the parts a comment has, and finite values; nothing changes when refused. + let l = add_annotation( + &mut doc, + &new(0, Shape::Line { from: [0.0, 0.0], to: [10.0, 10.0], start: LineEnding::None, end: LineEnding::None }), + &meta("l"), + ) + .unwrap(); + assert!(matches!(set_part_opacity(&mut doc, 0, l, Some(0.5), None, &meta("")), Err(AnnotError::Invalid(_))), "a line has no fill"); + assert!(matches!(set_part_opacity(&mut doc, 0, r, None, Some(0.5), &meta("")), Err(AnnotError::Invalid(_))), "a rectangle has no text"); + let before = list(&doc, 0)[r].clone(); + assert!(matches!(set_part_opacity(&mut doc, 0, r, Some(f64::NAN), None, &meta("")), Err(AnnotError::Invalid(_)))); + assert_eq!(list(&doc, 0)[r], before); + // Another app's comment with /CA alone is drawn at it throughout, as before. + assert_eq!( + appearance::part_opacity(&Dict::from_iter([(b"CA".to_vec(), Object::Real(0.4))])), + appearance::PartOpacity { line: 0.4, fill: 0.4, text: 0.4 } + ); +} + #[test] fn shapes_can_be_filled_and_unfilled() { let mut doc = fixture(); @@ -591,6 +882,85 @@ fn fill_and_sign_items_are_drawn() { assert!(set_style(&mut doc, 1, 0, StyleChange::new(Some([0.0; 3]), None, None, None), &meta("")).is_err()); } +/// A border chosen for typewriter text goes around it in the text's colour. Changing it grows or +/// shrinks the box on every side, so the text stays where it was typed; a text box keeps its box. +#[test] +fn a_typewriter_border_goes_around_its_text_which_stays_put() { + let mut doc = fixture(); + let shape = Shape::Typewriter { rect: [100.0, 700.0, 200.0, 716.0], font_size: 11.0 }; + let style = Style { width: 1.0, ..Style::default_for(&shape) }; + let t = add_annotation(&mut doc, &NewAnnotation { page: 1, shape, style, contents: "Ada".into(), author: "Ada".into() }, &meta("x")).unwrap(); + // The text starts 2 pt inside the border: 100 + 1 + 2 across, 716 - 1 - 2 - 0.9 em down. + let at = "1 0 0 1 103 703.1 Tm"; + let d = &list(&doc, 1)[t]; + let ap = ap_content(&doc, d); + assert!(ap.contains("1 w\n") && ap.contains(" re S"), "a 1 pt border: {ap}"); + assert!(ap.contains(at), "{ap}"); + for (width, expected) in [(2.0, [99.0, 699.0, 201.0, 717.0]), (0.0, [101.0, 701.0, 199.0, 715.0])] { + set_style(&mut doc, 1, t, StyleChange::new(None, None, Some(width), None), &meta("")).unwrap(); + let d = &list(&doc, 1)[t]; + assert_eq!(rect(d), expected, "{width} pt border"); + let ap = ap_content(&doc, d); + assert!(ap.contains(at), "{width} pt: the text keeps its place: {ap}"); + assert_eq!(ap.contains(" re S"), width > 0.0, "{width} pt: {ap}"); + } + let shape = Shape::TextBox { rect: [100.0, 600.0, 200.0, 650.0], font_size: 11.0 }; + let b = add_annotation(&mut doc, &new(1, shape), &meta("b")).unwrap(); + set_style(&mut doc, 1, b, StyleChange::new(None, None, Some(3.0), None), &meta("")).unwrap(); + assert_eq!(rect(&list(&doc, 1)[b]), [100.0, 600.0, 200.0, 650.0], "a text box keeps the box it was drawn with"); + // Another app's typed text with its width in /Border (no /BS): 4 pt, so 2 pt shrinks the box + // by 2 on every side, as it is drawn. + let shape = Shape::Typewriter { rect: [100.0, 500.0, 200.0, 530.0], font_size: 11.0 }; + let o = add_annotation( + &mut doc, + &NewAnnotation { page: 1, style: Style::default_for(&shape), shape: shape.clone(), contents: "Bo".into(), author: "Ada".into() }, + &meta("o"), + ) + .unwrap(); + let (_, r) = annot_ref(&mut doc, 1, o).unwrap(); + doc.update_dict(r, |d| { + d.remove(b"BS"); + d.set(b"Border".to_vec(), num_array(&[0.0, 0.0, 4.0])); + }) + .unwrap(); + set_style(&mut doc, 1, o, StyleChange::new(None, None, Some(2.0), None), &meta("")).unwrap(); + assert_eq!(rect(&list(&doc, 1)[o]), [102.0, 502.0, 198.0, 528.0], "the imported 4 pt border is read from /Border"); + // A border colour that isn't a colour is refused before anything is written. + let mut bad = NewAnnotation { page: 1, style: Style::default_for(&shape), shape, contents: "x".into(), author: "Ada".into() }; + bad.style.border_color = Some([f64::NAN, 0.0, 0.0]); + let before = list(&doc, 1).len(); + assert!(matches!(add_annotation(&mut doc, &bad, &meta("n")), Err(AnnotError::Invalid(_)))); + assert_eq!(list(&doc, 1).len(), before); +} + +/// Text keeps its colour and the border gets its own: the stroking colour in `/DA`, which a new +/// text colour or font size keeps, and which `None` gives back to the text's. +#[test] +fn a_text_border_can_have_its_own_colour() { + let mut doc = fixture(); + let shape = Shape::Typewriter { rect: [100.0, 700.0, 200.0, 716.0], font_size: 11.0 }; + let style = Style { color: [0.0, 0.5, 0.0], width: 1.0, border_color: Some([1.0, 0.0, 0.0]), ..Style::default_for(&shape) }; + let t = add_annotation(&mut doc, &NewAnnotation { page: 1, shape, style, contents: "Ada".into(), author: "Ada".into() }, &meta("x")).unwrap(); + let da = |doc: &Document| text(&list(doc, 1)[t], b"DA"); + assert_eq!(da(&doc), "0 0.5 0 rg 1 0 0 RG /Helv 11 Tf"); + let ap = ap_content(&doc, &list(&doc, 1)[t]); + assert!(ap.contains("1 0 0 RG\n1 w"), "the border in red: {ap}"); + assert!(ap.contains("0 0.5 0 rg"), "the text in green: {ap}"); + assert_eq!(props(&doc, 1, t).unwrap().border_color, Some([1.0, 0.0, 0.0])); + set_style(&mut doc, 1, t, StyleChange::new(Some([0.0, 0.0, 1.0]), None, None, None), &meta("")).unwrap(); + assert_eq!(da(&doc), "0 0 1 rg 1 0 0 RG /Helv 11 Tf", "a new text colour keeps the border's"); + set_font_size(&mut doc, 1, t, 14.0, &meta("")).unwrap(); + assert_eq!(da(&doc), "0 0 1 rg 1 0 0 RG /Helv 14 Tf", "so does a new size"); + set_border_color(&mut doc, 1, t, Some([0.5, 0.5, 0.5]), &meta("")).unwrap(); + assert!(ap_content(&doc, &list(&doc, 1)[t]).contains("0.5 0.5 0.5 RG\n1 w")); + set_border_color(&mut doc, 1, t, None, &meta("")).unwrap(); + assert_eq!(da(&doc), "0 0 1 rg /Helv 14 Tf"); + assert!(ap_content(&doc, &list(&doc, 1)[t]).contains("0 0 1 RG\n1 w"), "the text's colour again"); + assert_eq!(props(&doc, 1, t).unwrap().border_color, None); + let m = add_annotation(&mut doc, &new(0, Shape::Rectangle { rect: [10.0, 10.0, 50.0, 50.0] }), &meta("m")).unwrap(); + assert!(set_border_color(&mut doc, 0, m, Some([1.0, 0.0, 0.0]), &meta("")).is_err(), "a rectangle's colour is its border"); +} + #[test] fn author_subject_and_note_icon_change() { let mut doc = fixture(); @@ -873,8 +1243,8 @@ fn cloud_ys(path: &str) -> Vec { fn replace_text_groups_a_strikeout_and_a_caret() { let mut doc = fixture(); let quads = [[100.0, 720.0, 200.0, 720.0, 100.0, 706.0, 200.0, 706.0]]; - let red = Style { color: [0.89, 0.13, 0.13], opacity: 1.0, width: 1.0, fill: None }; - let blue = Style { color: [0.0, 0.47, 0.84], opacity: 1.0, width: 1.0, fill: None }; + let red = Style { color: [0.89, 0.13, 0.13], opacity: 1.0, width: 1.0, fill: None, ..Style::default() }; + let blue = Style { color: [0.0, 0.47, 0.84], opacity: 1.0, width: 1.0, fill: None, ..Style::default() }; let i = add_text_replacement( &mut doc, TextReplacement { @@ -2251,6 +2621,141 @@ fn freehand_highlight_is_compatible_ink_with_marker_blending() { } } +/// A 3D annotation is PDF 1.6 and a PRC model 1.7 (ISO 32000-2 §13.6): an older document's +/// catalog says so, and a newer one is left as it is. +#[test] +fn a_3d_model_raises_an_older_document_to_its_version() { + let catalog_version = |doc: &Document| { + let root = doc.root().unwrap(); + doc.get(root).as_dict().unwrap().name(b"Version").map(<[u8]>::to_vec) + }; + let add = |doc: &mut Document, model: &[u8]| { + let new = NewModel3d { page: 0, rect: [100.0, 100.0, 300.0, 300.0], model, activation: Activation3d::default(), poster: None, contents: "" }; + add_3d_model(doc, &new, &meta("m")).unwrap(); + }; + let mut doc = fixture(); + add(&mut doc, b"U3D\0rest"); + assert_eq!(catalog_version(&reopen(&doc)), None, "a 1.7 file needs nothing for U3D"); + for (model, version) in [(&b"U3D\0rest"[..], &b"1.6"[..]), (b"PRC rest", b"1.7")] { + let mut doc = fixture(); + doc.set_version("1.4"); + add(&mut doc, model); + assert_eq!(catalog_version(&reopen(&doc)).as_deref(), Some(version)); + } + let mut doc = fixture(); + doc.set_version("2.0"); + add(&mut doc, b"PRC rest"); + assert_eq!(catalog_version(&reopen(&doc)), None, "a 2.0 file takes PRC as it is"); +} + +/// Before 2.0, PRC is an Adobe extension to PDF 1.7 (§7.12): a PRC model declares it, keeping +/// other extensions and a later ADBE level; U3D and a 2.0 file need nothing. +#[test] +fn a_prc_model_declares_the_adobe_extension_before_pdf_2() { + let adbe = |doc: &Document| { + let root = doc.root().unwrap(); + let ext = doc.get(root).as_dict().unwrap().get(b"Extensions").map(|e| doc.resolve(e).as_dict().unwrap().clone()); + ext.map(|e| { + let adbe = e.get(b"ADBE").map(|a| a.as_dict().unwrap().clone()); + (e.get(b"XYZ").is_some(), adbe.map(|a| (a.name(b"BaseVersion").map(<[u8]>::to_vec), a.get(b"ExtensionLevel").and_then(Object::as_int)))) + }) + }; + let add = |doc: &mut Document, model: &[u8]| { + let new = NewModel3d { page: 0, rect: [100.0, 100.0, 300.0, 300.0], model, activation: Activation3d::default(), poster: None, contents: "" }; + add_3d_model(doc, &new, &meta("m")).unwrap(); + }; + let mut doc = fixture(); + add(&mut doc, b"PRC rest"); + assert_eq!(adbe(&reopen(&doc)), Some((false, Some((Some(b"1.7".to_vec()), Some(1)))))); + let mut doc = fixture(); + add(&mut doc, b"U3Drest"); + assert_eq!(adbe(&reopen(&doc)), None, "U3D is in ISO 32000-1"); + let mut doc = fixture(); + doc.set_version("2.0"); + add(&mut doc, b"PRC rest"); + assert_eq!(adbe(&reopen(&doc)), None, "PRC is in ISO 32000-2"); + + let mut doc = fixture(); + let root = doc.root().unwrap(); + let mut level3 = Dict::new(); + level3.set(b"BaseVersion".to_vec(), Object::name("1.7")); + level3.set(b"ExtensionLevel".to_vec(), Object::Int(3)); + let mut ext = Dict::new(); + ext.set(b"ADBE".to_vec(), Object::Dict(level3)); + ext.set(b"XYZ".to_vec(), Object::Dict(Dict::new())); + doc.update_dict(root, |c| c.set(b"Extensions".to_vec(), Object::Dict(ext.clone()))).unwrap(); + add(&mut doc, b"PRC rest"); + assert_eq!(adbe(&reopen(&doc)), Some((true, Some((Some(b"1.7".to_vec()), Some(3))))), "a later level is kept"); + ext.remove(b"ADBE"); + let mut doc = fixture(); + doc.update_dict(root, |c| c.set(b"Extensions".to_vec(), Object::Dict(ext))).unwrap(); + add(&mut doc, b"PRC rest"); + assert_eq!(adbe(&reopen(&doc)), Some((true, Some((Some(b"1.7".to_vec()), Some(1))))), "other extensions are kept"); + + let mut doc = fixture(); + let mut ext = Dict::new(); + ext.set(b"ADBE".to_vec(), Object::Array(vec![Object::Dict(Dict::new())])); + let ext = doc.add(Object::Dict(ext)); + doc.update_dict(root, |c| c.set(b"Extensions".to_vec(), Object::Ref(ext))).unwrap(); + add(&mut doc, b"PRC rest"); + let doc = reopen(&doc); + let ext = doc.resolve(doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"Extensions").unwrap()); + assert!(ext.as_dict().unwrap().get(b"ADBE").unwrap().as_array().is_some(), "an array of ADBE extensions is left alone"); +} + +#[test] +fn a_picture_is_placed_by_its_box_and_a_form_needs_a_positive_size() { + let at = [10.0, 20.0, 110.0, 70.0]; + assert_eq!(appearance::picture_place(at, true, None).as_deref(), Some("100 0 0 50 10 20 cm"), "an image fills its unit square"); + assert_eq!(appearance::picture_place(at, false, Some((200.0, 25.0))).as_deref(), Some("0.5 0 0 2 10 20 cm")); + for size in [None, Some((0.0, 25.0)), Some((200.0, -1.0))] { + assert_eq!(appearance::picture_place(at, false, size), None, "{size:?}"); + } +} + +/// The poster's box must lie in the rectangle, and a form poster needs a size to scale by. +#[test] +fn a_3d_poster_outside_its_rectangle_or_without_a_size_is_refused() { + let picture = |at: [f64; 4], size: (f64, f64)| PosterPicture { xobject: ObjRef { num: 6, generation: 0 }, image: false, size, at }; + for (poster, why) in [ + (picture([50.0, 100.0, 250.0, 300.0], (10.0, 10.0)), "starts left of the rectangle"), + (picture([100.0, 100.0, 300.0, 301.0], (10.0, 10.0)), "ends above it"), + (picture([100.0, 100.0, 300.0, 300.0], (0.0, 10.0)), "has no width"), + (picture([100.0 - 2e-7, 150.0, 100.0 - 1e-7, 200.0], (10.0, 10.0)), "lies outside within the rounding slack"), + ] { + let mut doc = fixture(); + let new = NewModel3d { + page: 0, + rect: [100.0, 100.0, 300.0, 300.0], + model: b"PRC rest", + activation: Activation3d::default(), + poster: Some(poster), + contents: "", + }; + assert!(matches!(add_3d_model(&mut doc, &new, &meta("m")), Err(AnnotError::Invalid(_))), "{why}"); + assert_eq!(list(&doc, 0).len(), 1, "{why}: nothing added"); + } +} + +/// A poster box fitted to the rectangle in floating point ends an ulp outside it: a 7×3 picture +/// centred in [0 100 3.6 110.8] spans -2.2e-16 to 3.6000000000000005. It is taken, inside the +/// rectangle. +#[test] +fn a_3d_poster_fitted_to_its_rectangle_with_rounding_is_taken() { + let mut doc = fixture(); + let at = [-2.220446049250313e-16, 104.62857142857143, 3.6000000000000005, 106.17142857142858]; + let poster = PosterPicture { xobject: ObjRef { num: 6, generation: 0 }, image: false, size: (7.0, 3.0), at }; + let new = NewModel3d { + page: 0, + rect: [0.0, 100.0, 3.6, 110.8], + model: b"PRC rest", + activation: Activation3d::default(), + poster: Some(poster), + contents: "", + }; + assert_eq!(add_3d_model(&mut doc, &new, &meta("m")), Ok(1)); +} + #[test] fn indirect_typewriter_names_keep_outer_whitespace_in_summaries() { let mut doc = fixture(); diff --git a/crates/annot/src/threed.rs b/crates/annot/src/threed.rs new file mode 100644 index 0000000000..cf828388c0 --- /dev/null +++ b/crates/annot/src/threed.rs @@ -0,0 +1,292 @@ +//! 3D annotations (ISO 32000-2 §13.6.2): a U3D or PRC model shown in a rectangle on a page, +//! with a poster (`/AP /N`) for printing, for readers without 3D, and until it is activated. + +use super::create::{finite, normalize, num_array}; +use super::lookup::{annots, page_ref, set_annots}; +use super::types::{FLAG_PRINT, Meta}; +use super::{AnnotError, Dict, Document, ObjRef, Object, PdfString, Stream, appearance, n}; + +/// The largest model accepted, in bytes: well past real U3D/PRC files, and bounded because the +/// model is held and compressed in memory. +pub const MAX_MODEL_BYTES: usize = 256 << 20; + +/// What the poster says when there is no picture. +const PLACEHOLDER_LABEL: &str = "3D model"; + +/// A 3D stream's data format (§13.6.3): ECMA-363 U3D or ISO 14739-1 PRC. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ModelFormat { + U3d, + Prc, +} + +impl ModelFormat { + /// The format a model file's first bytes declare: U3D files start `U3D\0` (ECMA-363 file + /// header block type), PRC files start `PRC`. + /// + /// # Errors + /// + /// [`AnnotError::NotAModel`] for anything else. + pub fn detect(bytes: &[u8]) -> Result { + if bytes.starts_with(b"U3D\0") { + Ok(Self::U3d) + } else if bytes.starts_with(b"PRC") { + Ok(Self::Prc) + } else { + Err(AnnotError::NotAModel) + } + } + + /// The 3D stream's `/Subtype`. + pub fn subtype(self) -> &'static str { + match self { + Self::U3d => "U3D", + Self::Prc => "PRC", + } + } + + /// The PDF version a document holding it needs (§13.6): a 3D annotation is 1.6, a PRC stream + /// 1.7 under Adobe's extension, declared by [`add_3d_model`] (PDF 2.0 takes PRC as standard). + pub fn version(self) -> &'static str { + match self { + Self::U3d => "1.6", + Self::Prc => "1.7", + } + } +} + +/// When a reader turns the poster into the live model (§13.6.2, the activation dictionary's +/// `/A`). The standard's default is [`Self::Click`]. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Activation { + /// `/XA`: when the user clicks it; it prints as its poster. + #[default] + Click, + /// `/PO`: when the page opens. + PageOpen, + /// `/PV`: when the page becomes visible. + PageVisible, +} + +impl Activation { + /// Every activation, with the word a tool argument names it by. + pub const ALL: [(Self, &'static str); 3] = [(Self::Click, "click"), (Self::PageOpen, "page_open"), (Self::PageVisible, "page_visible")]; + + /// The `/A` name. + pub fn name(self) -> &'static str { + match self { + Self::Click => "XA", + Self::PageOpen => "PO", + Self::PageVisible => "PV", + } + } +} + +/// A picture already in the document (an image XObject, or a form XObject whose `/Matrix` maps it +/// to `size` points) drawn into `at`, a box inside the annotation's rectangle. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PosterPicture { + pub xobject: ObjRef, + pub image: bool, + pub size: (f64, f64), + pub at: [f64; 4], +} + +/// A 3D model to add on `page` (0-based) in `rect` (user space). +#[derive(Clone, Debug, PartialEq)] +pub struct NewModel3d<'a> { + pub page: usize, + pub rect: [f64; 4], + pub model: &'a [u8], + pub activation: Activation, + /// The poster's picture; `None` draws a labelled frame. + pub poster: Option, + /// `/Contents`: the annotation's alternate description (§12.5.2). + pub contents: &'a str, +} + +/// Check a model before anything is added: its size and its format. +/// +/// # Errors +/// +/// [`AnnotError::ModelTooLarge`] past [`MAX_MODEL_BYTES`], else as [`ModelFormat::detect`]. +pub fn check_model(model: &[u8]) -> Result { + check_model_size(model, MAX_MODEL_BYTES)?; + ModelFormat::detect(model) +} + +fn check_model_size(model: &[u8], max: usize) -> Result<(), AnnotError> { + if model.len() > max { + return Err(AnnotError::ModelTooLarge { size: model.len(), max }); + } + Ok(()) +} + +/// Add a 3D annotation: the model in a Flate-compressed 3D stream referenced from `/3DD` (so the +/// annotation has its own instance, §13.6.2), `/3DA` stating only `/A`, and the poster as `/AP /N`. +/// No view (`/3DV`, `/VA`) is written: readers open on the model's own default view. A document +/// older than [`ModelFormat::version`] is raised to it. Returns its index in the page's `/Annots`. +/// +/// # Errors +/// +/// As [`check_model`]; [`AnnotError::NoSuchPage`]; [`AnnotError::Invalid`] for a rectangle with no +/// area or a poster box outside it. +pub fn add_3d_model(doc: &mut Document, new: &NewModel3d<'_>, meta: &Meta) -> Result { + let format = check_model(new.model)?; + let page = page_ref(doc, new.page)?; + let rect = normalize(new.rect); + if !finite(&new.rect) || rect[2] - rect[0] < 1.0 || rect[3] - rect[1] < 1.0 { + return Err(AnnotError::Invalid("invalid rectangle for the 3D model (each side at least 1 pt)".into())); + } + let poster = match &new.poster { + Some(p) => picture_poster(rect, p)?, + None => placeholder_poster(rect), + }; + let mut stream = Dict::new(); + stream.set(b"Type".to_vec(), Object::name("3D")); + stream.set(b"Subtype".to_vec(), Object::name(format.subtype())); + let stream = doc.add(Object::Stream(Stream::flate(stream, new.model))); + let poster = doc.add(Object::Stream(poster)); + let d = annotation_dict(new, rect, page, stream, poster, meta); + let r = doc.add(Object::Dict(d)); + let mut list = annots(doc, page); + let index = list.len(); + list.push(Object::Ref(r)); + set_annots(doc, page, list)?; + at_least_version(doc, format.version())?; + if format == ModelFormat::Prc && !at_least(doc, "2.0") { + declare_prc_extension(doc)?; + } + Ok(index) +} + +/// Whether the header or the catalog `/Version` says `min` or later (§7.7.2). +fn at_least(doc: &Document, min: &str) -> bool { + let reached = |v: &[u8]| v.len() >= 3 && v >= min.as_bytes(); + let catalog = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|c| c.get(b"Version").cloned())); + let catalog = catalog.and_then(|v| doc.resolve(&v).as_name().map(<[u8]>::to_vec)); + reached(doc.version().as_bytes()) || catalog.as_deref().is_some_and(reached) +} + +/// PRC came before PDF 2.0 as an Adobe extension to 1.7: such a file declares +/// `/Extensions /ADBE` with ExtensionLevel 1 or later (§7.12). Other extensions and an ADBE entry +/// that already has a level are kept; one that is not a dictionary (an array of them) is left alone. +fn declare_prc_extension(doc: &mut Document) -> Result<(), AnnotError> { + let Some(root) = doc.root() else { return Ok(()) }; + let catalog = doc.get(root); + // `None` when present but not a dictionary; an empty one when absent. + let dict_or_new = |o: Option<&Object>| match o.map(|o| doc.resolve(o)) { + None => Some(Dict::new()), + Some(o) => o.as_dict().cloned(), + }; + let Some(mut extensions) = dict_or_new(catalog.as_dict().and_then(|c| c.get(b"Extensions"))) else { return Ok(()) }; + let Some(mut adbe) = dict_or_new(extensions.get(b"ADBE")) else { return Ok(()) }; + if adbe.get(b"ExtensionLevel").map(|l| doc.resolve(l)).and_then(|l| l.as_int()).is_some_and(|l| l >= 1) { + return Ok(()); + } + adbe.set(b"BaseVersion".to_vec(), Object::name("1.7")); + adbe.set(b"ExtensionLevel".to_vec(), Object::Int(1)); + extensions.set(b"ADBE".to_vec(), Object::Dict(adbe)); + doc.update_dict(root, |c| c.set(b"Extensions".to_vec(), Object::Dict(extensions)))?; + Ok(()) +} + +/// A document whose header and catalog `/Version` both say less than `min` gets `/Version min` in +/// its catalog (§7.7.2: the later of the two applies). +fn at_least_version(doc: &mut Document, min: &str) -> Result<(), AnnotError> { + let Some(root) = doc.root() else { return Ok(()) }; + if !at_least(doc, min) { + doc.update_dict(root, |c| c.set(b"Version".to_vec(), Object::name(min)))?; + } + Ok(()) +} + +fn annotation_dict(new: &NewModel3d<'_>, rect: [f64; 4], page: ObjRef, stream: ObjRef, poster: ObjRef, meta: &Meta) -> Dict { + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("Annot")); + d.set(b"Subtype".to_vec(), Object::name("3D")); + d.set(b"Rect".to_vec(), num_array(&rect)); + d.set(b"P".to_vec(), Object::Ref(page)); + d.set(b"F".to_vec(), Object::Int(FLAG_PRINT)); + if !new.contents.is_empty() { + d.set(b"Contents".to_vec(), PdfString::text(new.contents)); + } + if !meta.id.is_empty() { + d.set(b"NM".to_vec(), PdfString::text(&meta.id)); + } + if let Some(date) = &meta.date { + d.set(b"M".to_vec(), PdfString::literal(date.as_bytes().to_vec())); + } + d.set(b"3DD".to_vec(), Object::Ref(stream)); + let mut activation = Dict::new(); + activation.set(b"A".to_vec(), Object::name(new.activation.name())); + d.set(b"3DA".to_vec(), Object::Dict(activation)); + let mut ap = Dict::new(); + ap.set(b"N".to_vec(), Object::Ref(poster)); + d.set(b"AP".to_vec(), Object::Dict(ap)); + d +} + +/// The poster drawing `p` into its box, which must lie within `rect`. +fn picture_poster(rect: [f64; 4], p: &PosterPicture) -> Result { + let at = normalize(p.at); + // A box fitted to `rect` in floating point can end a few ulps outside it: allow that, then clamp. + let slack = rect.iter().fold(1.0_f64, |m, v| m.max(v.abs())) * 1e-9; + let near = finite(&p.at) && at[0] >= rect[0] - slack && at[1] >= rect[1] - slack && at[2] <= rect[2] + slack && at[3] <= rect[3] + slack; + let at = [at[0].max(rect[0]), at[1].max(rect[1]), at[2].min(rect[2]), at[3].min(rect[3])]; + if !(near && at[2] > at[0] && at[3] > at[1]) { + return Err(AnnotError::Invalid("the poster must fit inside the 3D model's rectangle".into())); + } + let place = appearance::picture_place(at, p.image, Some(p.size)).ok_or_else(|| AnnotError::Invalid("the poster picture has no size".into()))?; + let mut xobjects = Dict::new(); + xobjects.set(b"Pic".to_vec(), Object::Ref(p.xobject)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + Ok(appearance::form(rect, format!("q {place} /Pic Do Q\n").as_bytes(), resources)) +} + +/// A light frame with "3D model" centred in it, shrunk to fit (left out when even 2 pt won't). +fn placeholder_poster(rect: [f64; 4]) -> Stream { + let [x0, y0, x1, y1] = rect; + let (w, h) = (x1 - x0, y1 - y0); + let mut c = + format!("0.94 g {} {} {} {} re f\n0.45 G 1 w {} {} {} {} re S\n", n(x0), n(y0), n(w), n(h), n(x0 + 0.5), n(y0 + 0.5), n(w - 1.0), n(h - 1.0)); + let mut resources = Dict::new(); + let unit = appearance::text_width(PLACEHOLDER_LABEL, 1.0); + let size = if unit > 0.0 { (w * 0.8 / unit).min(h * 0.4).min(14.0) } else { 0.0 }; + if size >= 2.0 { + let (tx, ty) = (x0 + (w - unit * size) / 2.0, y0 + (h - size * 0.7) / 2.0); + c.push_str(&format!("BT 0.3 g /Helv {} Tf {} {} Td ({PLACEHOLDER_LABEL}) Tj ET\n", n(size), n(tx), n(ty))); + resources.set(b"Font".to_vec(), appearance::helvetica_resource()); + } + appearance::form(rect, c.as_bytes(), resources) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn model_size_is_capped() { + assert_eq!(check_model_size(&[0; 8], 8), Ok(())); + assert_eq!(check_model_size(&[0; 9], 8), Err(AnnotError::ModelTooLarge { size: 9, max: 8 })); + } + + #[test] + fn the_format_is_read_from_the_header() { + assert_eq!(ModelFormat::detect(b"U3D\0rest"), Ok(ModelFormat::U3d)); + assert_eq!(ModelFormat::detect(b"PRC8800"), Ok(ModelFormat::Prc)); + for junk in [&b""[..], b"U3D", b"U3DX", b"PR", b"%PDF-1.7", b"prc"] { + assert_eq!(ModelFormat::detect(junk), Err(AnnotError::NotAModel), "{junk:?}"); + } + } + + #[test] + fn the_placeholder_label_shrinks_to_fit_and_is_dropped_when_it_cannot() { + let wide = String::from_utf8(placeholder_poster([0.0, 0.0, 300.0, 200.0]).decoded().unwrap().to_vec()).unwrap(); + assert!(wide.contains("/Helv 14 Tf") && wide.contains("(3D model) Tj"), "{wide}"); + let tiny = placeholder_poster([0.0, 0.0, 4.0, 4.0]); + assert!(!String::from_utf8(tiny.decoded().unwrap().to_vec()).unwrap().contains("Tj")); + assert!(tiny.dict.get(b"Resources").and_then(Object::as_dict).is_none_or(|r| r.get(b"Font").is_none())); + } +} diff --git a/crates/annot/src/types.rs b/crates/annot/src/types.rs index 522995b9f9..f6bcc68a7e 100644 --- a/crates/annot/src/types.rs +++ b/crates/annot/src/types.rs @@ -1,5 +1,5 @@ use super::create::normalize; -use super::{ObjRef, appearance}; +use super::{Dict, ObjRef, appearance}; pub type Rgb = [f64; 3]; @@ -124,6 +124,28 @@ impl FillMark { let (w, h) = if matches!(rotation.rem_euclid(360), 90 | 270) { (h, w) } else { (w, h) }; [at[0] - w / 2.0, at[1] - h / 2.0, at[0] + w / 2.0, at[1] + h / 2.0] } + + /// The mark whose `/Name` this is. + pub fn from_name(n: &[u8]) -> Option { + [FillMark::Check, FillMark::Cross, FillMark::Dot, FillMark::Line].into_iter().find(|m| m.name().as_bytes() == n) + } + + /// The pen strokes the mark is drawn with in `rect` (user space): what its appearance strokes, + /// and what a copy of it shows while it is being placed. The dot is a filled circle (radius + /// 0.3 × the shorter side, centred) and has none. + pub fn strokes(self, rect: [f64; 4]) -> Vec> { + let [x0, y0, x1, y1] = rect; + let (w, h) = (x1 - x0, y1 - y0); + match self { + FillMark::Check => vec![vec![[x0 + w * 0.15, y0 + h * 0.5], [x0 + w * 0.4, y0 + h * 0.2], [x0 + w * 0.88, y0 + h * 0.85]]], + FillMark::Cross => vec![ + vec![[x0 + w * 0.18, y0 + h * 0.18], [x1 - w * 0.18, y1 - h * 0.18]], + vec![[x0 + w * 0.18, y1 - h * 0.18], [x1 - w * 0.18, y0 + h * 0.18]], + ], + FillMark::Dot => Vec::new(), + FillMark::Line => vec![vec![[x0, y0 + h / 2.0], [x1, y0 + h / 2.0]]], + } + } } /// The stamps of Acrobat's stamp palette (drawn in PdfCraft's own style). @@ -566,6 +588,16 @@ pub struct Style { pub width: f64, /// Interior fill for rectangles and ovals (`/IC`), background for text boxes. pub fill: Option, + /// A text box's, typewriter text's or callout's border colour where it differs from the + /// text's (`None`: the text colour), the stroking colour in its `/DA`. + pub border_color: Option, + /// A text box's or typewriter text's border shape (a callout's is always a rectangle). + pub border_shape: BorderShape, + /// The fill's own opacity, 0–1 (`/ca`), where it differs from `opacity` (which is then the + /// line's or border's); `None`: as `opacity`. + pub fill_opacity: Option, + /// A text box's text's own opacity, 0–1 (`/PCTextOpacity`); `None`: as the fill's. + pub text_opacity: Option, } /// The style properties changed by one annotation edit. `None` leaves a property unchanged. @@ -597,7 +629,102 @@ pub struct TextReplacement<'a> { impl Default for Style { fn default() -> Self { - Self { color: [1.0, 0.82, 0.0], opacity: 1.0, width: 1.0, fill: None } + Self { + color: [1.0, 0.82, 0.0], + opacity: 1.0, + width: 1.0, + fill: None, + fill_opacity: None, + text_opacity: None, + border_color: None, + border_shape: BorderShape::Rectangle, + } + } +} + +/// The shape of a text box's or typewriter text's border (PdfCraft's `/PCBorderShape`, absent for +/// a rectangle). An oval or a circle goes around the text box, which keeps its place: `/Rect` +/// grows to hold it, centred on the text box, and `/RD` says where the text box is inside +/// (§12.5.6.6), as for a callout. Other apps that redraw the comment draw a rectangle. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum BorderShape { + #[default] + Rectangle, + Oval, + Circle, +} + +impl BorderShape { + pub const ALL: [BorderShape; 3] = [BorderShape::Rectangle, BorderShape::Oval, BorderShape::Circle]; + + pub fn name(self) -> &'static str { + match self { + BorderShape::Rectangle => "Rectangle", + BorderShape::Oval => "Oval", + BorderShape::Circle => "Circle", + } + } + + /// `rectangle`, `oval` or `circle`, in any case. + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|b| b.name().eq_ignore_ascii_case(s)) + } + + /// The border shape of FreeText `d`: always a rectangle for a callout. + pub fn of(d: &Dict) -> Self { + if d.contains(b"CL") { + return BorderShape::Rectangle; + } + match d.name(b"PCBorderShape") { + Some(b"Oval") => BorderShape::Oval, + Some(b"Circle") => BorderShape::Circle, + _ => BorderShape::Rectangle, + } + } + + /// The rectangle this shape, with a border `width` wide, takes around text box `tb`, centred on + /// it: an oval through the text box's corners (`√2` its size), a point clear of the border; a + /// circle around the text's lines (`text`: the widest line's width and the lines' height, when + /// known), a point clear of the border and touching the text box's longer sides at least + /// (its corners, padding only, may lie outside), else through the box's corners; the text box + /// itself for a rectangle. + pub fn around(self, tb: [f64; 4], width: f64, text: Option<[f64; 2]>) -> [f64; 4] { + let tb = normalize(tb); + let (cx, cy, hx, hy) = ((tb[0] + tb[2]) / 2.0, (tb[1] + tb[3]) / 2.0, (tb[2] - tb[0]) / 2.0, (tb[3] - tb[1]) / 2.0); + let gap = width.max(0.0) + 1.0; + let (ax, ay) = match self { + BorderShape::Rectangle => return tb, + BorderShape::Oval => (hx * std::f64::consts::SQRT_2 + gap, hy * std::f64::consts::SQRT_2 + gap), + BorderShape::Circle => { + let r = match text { + Some([w, h]) => ((w / 2.0).hypot(h / 2.0) + gap).max(hx).max(hy), + None => hx.hypot(hy) + gap, + }; + (r, r) + } + }; + [cx - ax, cy - ay, cx + ax, cy + ay] + } + + /// The text box inside `outer` for this shape when none is recorded: the inverse of + /// [`Self::around`] for an oval, a box through a circle's corners for a circle, keeping the + /// proportions of `was` (the one it had). + pub(crate) fn inside(self, outer: [f64; 4], width: f64, was: [f64; 4]) -> [f64; 4] { + let o = normalize(outer); + let (cx, cy, hx, hy) = ((o[0] + o[2]) / 2.0, (o[1] + o[3]) / 2.0, (o[2] - o[0]) / 2.0, (o[3] - o[1]) / 2.0); + let gap = width.max(0.0) + 1.0; + let (ix, iy) = match self { + BorderShape::Rectangle => return o, + BorderShape::Oval => ((hx - gap) / std::f64::consts::SQRT_2, (hy - gap) / std::f64::consts::SQRT_2), + BorderShape::Circle => { + let w = normalize(was); + let (wx, wy) = ((w[2] - w[0]) / 2.0, (w[3] - w[1]) / 2.0); + let k = (hx.min(hy) - gap) / wx.hypot(wy).max(1e-6); + (wx * k, wy * k) + } + }; + let (ix, iy) = (ix.max(0.5), iy.max(0.5)); + [cx - ix, cy - iy, cx + ix, cy + iy] } } @@ -615,7 +742,7 @@ impl Style { } Shape::Callout { .. } => ([0.0, 0.0, 0.0], 1.0), Shape::Caret { .. } | Shape::Attachment { .. } => ([0.0, 0.47, 0.84], 1.0), - Shape::FreehandHighlight { .. } => return Self { color: [1.0, 0.94, 0.0], opacity: 0.35, width: 14.0, fill: None }, + Shape::FreehandHighlight { .. } => return Self { color: [1.0, 0.94, 0.0], opacity: 0.35, width: 14.0, ..Self::default() }, Shape::Ink { .. } => ([0.0, 0.4, 0.87], 2.0), Shape::TextBox { .. } | Shape::Typewriter { .. } => ([0.0, 0.0, 0.0], 0.0), Shape::Mark { .. } => ([0.0, 0.0, 0.0], 1.5), @@ -624,9 +751,9 @@ impl Style { Shape::CustomStamp { .. } => ([0.0, 0.0, 0.0], 0.0), Shape::TypedSignature { .. } => ([0.0, 0.0, 0.0], 0.0), // Red outline while marked; a black box once applied. - Shape::Redact { .. } => return Self { color: [0.89, 0.13, 0.13], opacity: 1.0, width: 1.0, fill: Some([0.0, 0.0, 0.0]) }, + Shape::Redact { .. } => return Self { color: [0.89, 0.13, 0.13], fill: Some([0.0, 0.0, 0.0]), ..Self::default() }, }; - Self { color, opacity: 1.0, width, fill: None } + Self { color, width, ..Self::default() } } } diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index a6e43de1f0..a825b1fe52 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -14,6 +14,8 @@ pdfcraft-cos = { workspace = true } pdfcraft-render = { workspace = true } pdfcraft-organize = { workspace = true } pdfcraft-platform = { workspace = true } +pdfcraft-prc = { workspace = true } +pdfcraft-prc-render = { workspace = true } serde_json = { workspace = true } sha2 = "0.11" tempfile = "3.27" diff --git a/crates/automation/README.md b/crates/automation/README.md index 8ffa2d0b3e..2fbbff8ccb 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -26,6 +26,11 @@ its undo/redo history. `page_add_text`, `content_update` and `text_edit` take per-line `aligns`. An `"inherit"` entry follows the box-level `align`; legacy `null` entries remain accepted with the same meaning. +`page_add_text` and `content_update` take one of `color` (`#RRGGBB` or a name, painted in +DeviceRGB), `gray` (0 black to 1 white, DeviceGray `g`) and `cmyk` (`[c, m, y, k]`, each 0 to 1, +DeviceCMYK `k`), so text added to a print file can be 100 K black. `content_list` reports `gray` or +`cmyk` for such text. + `Content` is `Json(Value)` or `Png { data, width, height }`. Errors are `UnknownTool`, `InvalidArgs` (the call didn't match the schema) or `Failed` (a readable message for the agent). ## Conventions @@ -250,6 +255,15 @@ choices and values over 4096 UTF-8 bytes return an error. Example: `form_set_props doc=1 field=agree export_value=Accepted widget=1`. +## Rendering a 3D model + +`model_render` draws a page's 3D model (a 3D annotation's PRC stream, directly or through a 3D +reference) to a PNG, without the viewer. `model` picks the page's Nth 3D model (default 1); +`width` and `height` are 1 to 4096 (default 512); `direction` is the view direction from the eye +(default `[-1, 1, -1]`, with Z up unless the view looks along Z); `perspective` defaults to true. +The model is framed to fill the image and shaded by a headlight on white. U3D models are refused +with an error, and the annotation's own views, lighting and render modes are not applied yet. + ## Viewer navigation history A running PdfCraft instance launched with `--control FILE` exposes its per-tab history through @@ -272,3 +286,26 @@ navigation-history tool; use the opt-in UI control channel to navigate the displ (`#RRGGBB`, default black) for `text` and `date`. The colour is saved in the FreeText appearance and survives reopening. Other mark/signature types reject `color`; their appearance controls are separate. + +## Reviewing OCR headlessly + +`ocr_words {doc, retain_review: true}` reads words without editing the document and returns +`review`, plus pages and word boxes/confidences. Page and word positions are 1-based. +`region` can restrict recognition to one named page, using displayed-page points. Without +`retain_review`, word inspection remains stateless. + +Apply reviewed text with `ocr_review_apply {doc, review, corrections: [{page: 1, word: 1, +text: "Corrected"}]}`. Omitted words keep their text; empty text removes a word. Geometry +comes from the retained recognition, never caller input. The captured document guard rejects +stale/foreign results. Successful application consumes the handle and is one undoable step. +Use `output_mode: "editable", out: "reviewed.pdf"` to write a separate editable document, +or `discard: true` to release the result without applying it. Failed validation preserves the +review for correction or discard. A new retained recognition replaces that document's prior +handle; closing the document releases it. At most eight documents may retain reviews, with +65,536-word/4-MiB input bounds and a 65,536-page cap. + +`ocr_dictionary {path: "ocr-dictionary-en.txt", add: ["Quartz"], remove: ["old"]}` atomically +updates the same one-word-per-UTF-8-line format OCR Verify uses. Omit add/remove to read it. +The explicit path honors `--root`, including symlink confinement; choose the appropriate +per-language file used by the UI. Entries are case-insensitive and duplicate-free. Invalid +or oversized inputs fail before writing (65,536 words, 4 MiB, at most 1,024 bytes per word). diff --git a/crates/automation/src/a11y/ocr.rs b/crates/automation/src/a11y/ocr.rs index 82e521ddd5..2846aa44a6 100644 --- a/crates/automation/src/a11y/ocr.rs +++ b/crates/automation/src/a11y/ocr.rs @@ -1,98 +1,534 @@ use super::{Args, Automation, Result, ToolError, Value, child, failed, json, write_atomic}; impl Automation { - /// Recognition settings from the arguments: `dpi` (clamped to 72–600), `language` (refused - /// unless the models read it) and `skip_text_pages`, each left at its default when absent. + /// The settings every OCR tool shares: dpi, language, the engine/strategy choice, the four + /// preprocessing flags, the skip flags and the output mode — each named exactly as its tool + /// schema says. An unknown engine, strategy or mode is an honest error, never a silent + /// fallback: a tool run must not quietly do something other than what it was asked. pub(crate) fn ocr_settings(&self, a: &Args) -> Result { + use pdfcraft_engine::ocr::{EngineChoice, MergeStrategy, OutputMode}; let mut settings = pdfcraft_engine::ocr::OcrSettings::default(); if let Some(d) = a.opt_num("dpi")? { settings.dpi = d.clamp(72.0, 600.0) as f32; } if let Some(l) = a.opt_str("language")? { if !pdfcraft_engine::ocr::LANGUAGES.iter().any(|x| x.0 == l) { - return Err(ToolError::InvalidArgs(format!("unsupported language {l:?}"))); + return Err(ToolError::InvalidArgs(format!( + "unsupported language {l:?} (supported: {})", + pdfcraft_engine::ocr::LANGUAGES.iter().map(|(c, _)| *c).collect::>().join(", ") + ))); } settings.language = l.into(); } - if let Some(s) = a.opt_bool("skip_text_pages")? { - settings.skip_text_pages = s; + if let Some(e) = a.opt_str("engine")? { + settings.engine = match e.trim().to_ascii_lowercase().as_str() { + "ocrs" => EngineChoice::Ocrs, + "tesseract" => EngineChoice::Tesseract, + "auto" | "automatic" | "ensemble" => EngineChoice::Auto, + other => return Err(ToolError::InvalidArgs(format!("unsupported engine {other:?} (ocrs, tesseract or auto)"))), + }; + } + settings.strategy = match a.opt_str("strategy")? { + Some(s) => match s.trim().to_ascii_lowercase().as_str() { + "primary" | "primary-only" | "primary_only" => MergeStrategy::PrimaryOnly, + "confidence" | "confidence-weighted" | "confidence_weighted" => MergeStrategy::ConfidenceWeighted, + "rover" | "rover-vote" | "rover_vote" | "vote" => MergeStrategy::RoverVote, + other => return Err(ToolError::InvalidArgs(format!("unsupported strategy {other:?} (primary, confidence or rover)"))), + }, + // Unset: the engine choice decides (auto brings in the second engine when the + // first reading is weak; a named engine reads alone). + None => settings.engine.default_strategy(), + }; + if let Some(m) = a.opt_str("output_mode")? { + settings.output_mode = match m.trim().to_ascii_lowercase().as_str() { + "searchable" | "searchable-image" | "searchable_image" => OutputMode::Searchable, + "editable" | "editable-text" | "editable_text" => OutputMode::EditableText, + other => return Err(ToolError::InvalidArgs(format!("unsupported output mode {other:?} (searchable or editable)"))), + }; + } + for (key, flag) in [ + ("auto_rotate", &mut settings.auto_rotate), + ("deskew", &mut settings.deskew), + ("denoise", &mut settings.denoise), + ("binarize", &mut settings.binarize), + ] { + if let Some(on) = a.opt_bool(key)? { + *flag = on; + } + } + for (key, flag) in [ + ("skip_text_pages", &mut settings.skip_text_pages), + ("skip_text_files", &mut settings.skip_text_files), + ("force_ocr", &mut settings.force_ocr), + ] { + if let Some(on) = a.opt_bool(key)? { + *flag = on; + } } Ok(settings) } - /// Recognizes text in each of `paths` and writes the result under the same file name in - /// `folder`, reporting per file its output, word count and skipped pages, or its error. A - /// path outside the root refuses the whole call before any work starts. + /// The optional `region` argument ([x0, y0, x1, y1] in points from the top-left of the + /// displayed page) as the job's pixel crop, at the resolution that page is really read at. + /// A region reads exactly one page. + fn ocr_region( + &self, + a: &Args, + id: pdfcraft_engine::DocId, + page: usize, + settings: &pdfcraft_engine::ocr::OcrSettings, + ) -> Result> { + const BAD: &str = "region must be [x0, y0, x1, y1] in points"; + let Some(v) = a.get("region") else { return Ok(None) }; + let arr = v.as_array().ok_or_else(|| ToolError::InvalidArgs(BAD.into()))?; + if arr.len() != 4 { + return Err(ToolError::InvalidArgs(BAD.into())); + } + let mut pts = [0.0f64; 4]; + for (i, x) in arr.iter().enumerate() { + pts[i] = x.as_f64().filter(|f| f.is_finite()).ok_or_else(|| ToolError::InvalidArgs(BAD.into()))?; + } + let scale = f64::from( + self.session.ocr_job(id, &[page], settings.clone()).ok_or_else(|| failed("the document could not be read"))?.render_scale(page), + ); + let px: Vec = pts.iter().map(|p| scale * p).map(|p| if p.is_finite() { p } else { 0.0 }).map(|p| p as f32).collect(); + Ok(Some([px[0], px[1], px[2], px[3]])) + } + + /// The per-page result entries the OCR tools return: recognised text or why the page was + /// passed over. The text itself is only echoed when the document allows copying; the + /// counts and metadata always are. + fn ocr_pages_json(found: &[pdfcraft_engine::ocr::OcrPage], include_text: bool) -> Vec { + found + .iter() + .map(|p| match &p.skipped { + Some(why) => json!({ "page": p.page + 1, "skipped": why }), + None => { + let mut page = json!({ "page": p.page + 1, "words": p.words.len(), "already_on_page": p.already_on_page }); + // The editable output's substitution warnings (characters no font could + // show) are part of the result: reported per page, never dropped. + if !p.notes.is_empty() { + page["notes"] = json!(p.notes); + } + if include_text { + page["text"] = json!(p.text()); + } + page + } + }) + .collect() + } + pub(crate) fn ocr_recognize_files(&mut self, a: &Args) -> Result { let settings = self.ocr_settings(a)?; let folder = self.resolve(a.str("folder")?, true)?; - // A path that leaves the root refuses the whole call before any work starts, like the - // other tools (the write check passes missing files, which stay per-file errors below). + // A path that leaves the read root refuses the whole call before any work starts; + // a file that is merely missing stays a per-file error below. let paths = a.strs("paths")?; for p in &paths { self.resolve_for(p, crate::file_access::Access::Read, true)?; } - // Two inputs that would write the same output file overwrite each other while both report - // success, so the whole batch is refused here: before the folder is created, anything is - // read, or the OCR engines load. + // Two inputs of one file name (from different folders) would write the same output: + // the second would replace the first while both report success. The whole batch is + // refused up front — before the folder is created, anything is read or the engines + // load — naming the file, the way the default filesystem compares names. let targets: Vec = paths.iter().map(|p| child(&folder, &input_file_name(p))).collect(); let shared = shared_file_names(&targets); if !shared.is_empty() { return Err(ToolError::InvalidArgs(format!( - "more than one input would write to the same output file, so each would replace the one before it. Colliding file names: {}. Give each input a different file name, or run them in separate batches with different folders.", + "several paths share a file name, so their outputs in folder would overwrite each other (rename or run them separately): {}", shared.join(", ") ))); } std::fs::create_dir_all(&folder).map_err(|e| failed(e.to_string()))?; - let ocr = pdfcraft_engine::ocr::engine().map_err(failed)?; + let names: Vec = paths.iter().map(|p| input_file_name(p)).collect(); + // Collision pre-flight: a searchable copy never silently overwrites. Every target is + // checked before the first page is read; a collision aborts the whole batch with the + // names in the error. + let mut collisions = std::collections::BTreeSet::new(); + for name in names { + if child(&folder, &name).exists() { + collisions.insert(name); + } + } + if !collisions.is_empty() { + let mut list: Vec = collisions.iter().take(5).cloned().collect(); + if collisions.len() > list.len() { + list.push(format!("and {} more", collisions.len() - list.len())); + } + return Err(failed(format!( + "the output folder already holds {} (delete or move them, or choose another folder): {}", + if collisions.len() == 1 { "a file of that name" } else { "files of those names" }, + list.join(", ") + ))); + } + // The engines are built (and named if they cannot be) before the first file is read. + let recognizers = pdfcraft_engine::ocr::recognizers(&settings).map_err(failed)?; let mut out = Vec::new(); + let (mut ok, mut skipped_files, mut words) = (0usize, 0usize, 0usize); for p in paths { let name = input_file_name(p); let result = self.resolve(p, false).map_err(|e| e.to_string()).and_then(|src| crate::read_input_bytes(&src).map_err(|e| e.to_string())).and_then( - |bytes| pdfcraft_engine::ocr::recognize_file(&name, std::sync::Arc::new(bytes), None, settings.clone(), &ocr, |_, _| true), + |bytes| { + pdfcraft_engine::ocr::recognize_file(&name, std::sync::Arc::new(bytes), None, settings.clone(), &recognizers, |_, _| true) + }, ); - out.push(match result { + match result { Ok(r) => { + let skipped_pages: Vec = r.pages.iter().filter(|p| p.skipped.is_some()).map(|p| p.page + 1).collect(); + if r.skipped() { + // A file passed over by the skip-files option is its own bucket: its + // original is left alone and nothing is written. + skipped_files += 1; + out.push(json!({ "path": p, "skipped": r.file_skipped, "skipped_pages": skipped_pages })); + continue; + } let target = child(&folder, &name); write_atomic(&target, &r.bytes)?; - let skipped: Vec = r.pages.iter().filter(|p| p.skipped.is_some()).map(|p| p.page + 1).collect(); - json!({ "path": p, "output": target.to_string_lossy(), "words": r.words(), "skipped_pages": skipped }) + ok += 1; + words += r.words(); + let mut entry = json!({ "path": p, "output": target.to_string_lossy(), "words": r.words(), "skipped_pages": skipped_pages }); + // The batch's low-confidence note: at or under the settings' threshold (60 + // by default), on 1-based sorted pages. + let threshold = settings.low_confidence_threshold; + if let Some((count, pages)) = pdfcraft_engine::ocr::low_confidence_pages(&r.pages, threshold) { + entry["low_confidence"] = json!({ "count": count, "pages": pages, "threshold": threshold }); + } + out.push(entry); } - Err(e) => json!({ "path": p, "error": e }), - }); + Err(e) => out.push(json!({ "path": p, "error": e })), + } } - Ok(json!({ "files": out })) + Ok(json!({ "files": out, "recognized": ok, "words": words, "skipped_files": skipped_files })) } - /// Recognizes text on the open document's `pages` (every page when absent) and adds it, - /// reporting the total word count and, per page, why it was skipped or what it found. pub(crate) fn ocr_recognize(&mut self, a: &Args) -> Result { - let settings = self.ocr_settings(a)?; + use pdfcraft_engine::ocr::OutputMode; + let mut settings = self.ocr_settings(a)?; let pages = if a.opt_ints("pages")?.is_some() { self.pages(a, "pages")? } else { Vec::new() }; - let doc = self.doc(a)?; // Recognizing needs only permission to change the document; the words it found are // returned only when copying is allowed too. - let (id, copy) = (doc.id, doc.allows_copying()); - let found = self.session.recognize_text(id, &pages, settings).map_err(failed)?; + let (id, copy) = { + let doc = self.doc(a)?; + (doc.id, doc.allows_copying()) + }; + // The optional region reads exactly one page. + if a.get("region").is_some() { + if pages.len() != 1 { + return Err(ToolError::InvalidArgs("region reads one page: name it in pages".into())); + } + settings.region = self.ocr_region(a, id, pages[0], &settings)?; + } + // The editable output's destination is checked before anything runs: a NEW file, never + // the open source and never an existing file replaced silently. + let out = match settings.output_mode { + OutputMode::Searchable => { + // `out` belongs to the editable output; a searchable run writes into the open + // document, so an `out` here would be silently ignored (and never checked + // against the root). + if a.get("out").is_some() { + return Err(ToolError::InvalidArgs("out is only used with output_mode editable; remove it, or choose editable".into())); + } + None + } + OutputMode::EditableText => { + // The editable output writes the recognized text out of the document: a + // document that forbids copying refuses here, before anything runs. + self.doc_to_copy(a)?; + // A write: the file does not exist yet, and write_atomic makes its folder. + let out = self.resolve(a.str("out")?, true)?; + if let Some(source) = self.session.get(id).and_then(|d| d.path.clone()) + && let (Ok(a), Ok(b)) = (std::fs::canonicalize(&out), std::fs::canonicalize(&source)) + && a == b + { + return Err(failed("out is the source document; the editable output is a new file, so choose another path")); + } + if out.exists() { + return Err(failed(format!( + "out already exists ({}); the editable output is a new file, so choose a path that does not exist yet", + out.to_string_lossy() + ))); + } + Some(out) + } + }; + // Honest up front: the engines are built (and named if they cannot be) before anything + // is read or written. + let recognizers = pdfcraft_engine::ocr::recognizers(&settings).map_err(failed)?; + match out { + None => { + let found = self.session.recognize_text(id, &pages, settings).map_err(failed)?; + let words = found.iter().map(|p| p.words.len()).sum::(); + Ok(json!({ "words": words, "pages": Self::ocr_pages_json(&found, copy) })) + } + Some(out) => { + let job = self.session.ocr_job(id, &pages, settings).ok_or_else(|| failed("the document could not be read"))?; + let found = job.run(&recognizers, |_, _| true); + let bytes = self.session.editable_ocr_bytes(id, &found).map_err(failed)?; + // recognition can take minutes: a file created meanwhile is refused, not replaced + if let Some(folder) = out.parent() { + std::fs::create_dir_all(folder).map_err(failed)?; + } + crate::bates::publish_new(&out, &bytes)?; + Ok(json!({ + "out": out.to_string_lossy(), + "words": found.iter().map(|p| p.words.len()).sum::(), + "pages": Self::ocr_pages_json(&found, copy), + })) + } + } + } + + /// The words of a recognition, for reviewing without the GUI: text, box (points from the + /// top-left of the displayed page), confidence, band, and which words are suspects. + /// Changes nothing; needs the engines (ocr_status). + pub(crate) fn ocr_words(&mut self, a: &Args) -> Result { + let mut settings = self.ocr_settings(a)?; + let pages = if a.opt_ints("pages")?.is_some() { self.pages(a, "pages")? } else { Vec::new() }; + // The word list is the document's text, word by word: refused when the document + // forbids copying it out. + let id = self.doc_to_copy(a)?.id; + if a.get("region").is_some() { + if pages.len() != 1 { + return Err(ToolError::InvalidArgs("region reads one page: name it in pages".into())); + } + settings.region = self.ocr_region(a, id, pages[0], &settings)?; + } + let retain = a.opt_bool("retain_review")?.unwrap_or(false); + if retain && !self.ocr_reviews.contains_key(&id) && self.ocr_reviews.len() >= 8 { + return Err(failed("at most 8 OCR reviews may be retained; discard one first")); + } + let recognizers = pdfcraft_engine::ocr::recognizers(&settings).map_err(failed)?; + let job = self.session.ocr_job(id, &pages, settings.clone()).ok_or_else(|| failed("the document could not be read"))?; + let infos: std::collections::HashMap = + self.session.get(id).map(|d| d.info.pages.clone()).unwrap_or_default().into_iter().enumerate().collect(); + let guard = job.guard.clone(); + let found = job.run(&recognizers, |_, _| true); + if found.len() > 65536 { + return Err(failed("at most 65536 OCR review pages are allowed")); + } + pdfcraft_engine::ocr::validate_ocr_words(found.iter().flat_map(|p| &p.words)).map_err(failed)?; + let band_name = |band: pdfcraft_engine::ocr::ConfidenceBand| match band { + pdfcraft_engine::ocr::ConfidenceBand::High => "high", + pdfcraft_engine::ocr::ConfidenceBand::Medium => "medium", + pdfcraft_engine::ocr::ConfidenceBand::Low => "low", + }; + let mut suspects = 0usize; let list: Vec = found .iter() - .map(|p| match &p.skipped { - Some(why) => json!({ "page": p.page + 1, "skipped": why }), - None if copy => json!({ "page": p.page + 1, "words": p.words.len(), "already_on_page": p.already_on_page, "text": p.text() }), - None => json!({ "page": p.page + 1, "words": p.words.len(), "already_on_page": p.already_on_page }), + .map(|p| { + let info = infos.get(&p.page); + let words: Vec = p + .words + .iter() + .map(|w| { + let suspect = pdfcraft_engine::ocr::is_suspect(w.confidence); + if suspect { + suspects += 1; + } + // The word's corners in view space, as the box [x, y, w, h] in points + // from the top-left of the displayed page. + let view = |pt: [f64; 2]| info.map(|i| i.user_to_view(pt[0] as f32, pt[1] as f32)); + let corners = [ + view(w.origin), + view([w.origin[0] + w.across[0], w.origin[1] + w.across[1]]), + view([w.origin[0] + w.up[0], w.origin[1] + w.up[1]]), + view([w.origin[0] + w.across[0] + w.up[0], w.origin[1] + w.across[1] + w.up[1]]), + ]; + let xs: Vec = corners.iter().flatten().map(|v| v[0]).collect(); + let ys: Vec = corners.iter().flatten().map(|v| v[1]).collect(); + let (min_x, min_y) = (xs.iter().cloned().fold(f32::INFINITY, f32::min), ys.iter().cloned().fold(f32::INFINITY, f32::min)); + let (w_pt, h_pt) = ( + xs.iter().cloned().fold(f32::NEG_INFINITY, f32::max) - min_x, + ys.iter().cloned().fold(f32::NEG_INFINITY, f32::max) - min_y, + ); + json!({ + "text": w.text, + "box": [min_x, min_y, w_pt, h_pt], + "confidence": w.confidence.map(|c| (c * 10.0).round() / 10.0), + "band": w.confidence.map(|c| band_name(pdfcraft_engine::ocr::band_for(c))), + "suspect": suspect, + "source": w.source, + }) + }) + .collect(); + let mut page = + json!({ "page": p.page + 1, "already_on_page": p.already_on_page, "words": words, "mean_confidence": p.mean_confidence().map(|c| (c * 10.0).round() / 10.0) }); + if let Some(why) = &p.skipped { + page["skipped"] = json!(why); + } + page }) .collect(); - Ok(json!({ "words": found.iter().map(|p| p.words.len()).sum::(), "pages": list })) + let mut out = json!({ "words": found.iter().map(|p| p.words.len()).sum::(), "suspects": suspects, "pages": list }); + if retain { + self.next_ocr_review = + self.next_ocr_review.checked_add(1).filter(|n| *n <= i64::MAX as u64).ok_or_else(|| failed("OCR review handles exhausted"))?; + out["review"] = json!(self.next_ocr_review); + self.ocr_reviews.insert(id, crate::OcrReview { handle: self.next_ocr_review, guard, pages: found }); + } + Ok(out) + } + + /// Apply only caller corrections to retained original boxes, through the stored guard. + pub(crate) fn ocr_review_apply(&mut self, a: &Args) -> Result { + let id = self.doc(a)?.id; + let handle = a.int("review")?; + let review = self + .ocr_reviews + .get(&id) + .filter(|r| i64::try_from(r.handle).ok() == Some(handle)) + .ok_or_else(|| failed("unknown OCR review for this document; call ocr_words with retain_review"))?; + if a.opt_bool("discard")?.unwrap_or(false) { + if a.get("corrections").is_some() { + return Err(ToolError::InvalidArgs("discard cannot include corrections".into())); + } + self.ocr_reviews.remove(&id); + return Ok(json!({ "discarded": handle })); + } + if let Some(why) = self.session.stale_reason(&review.guard) { + return Err(failed(why)); + } + let corrections = a.get("corrections").map(|v| v.as_array().ok_or_else(|| Args::wrong("corrections", "an array"))).transpose()?; + if corrections.is_some_and(|c| c.len() > 65536) { + return Err(failed("at most 65536 OCR corrections are allowed")); + } + // Prevalidate and cap every replacement before cloning the retained result. + let mut edits = Vec::new(); + let mut bytes = 0usize; + let mut seen = std::collections::BTreeSet::new(); + let page_indices: std::collections::BTreeMap<_, _> = review.pages.iter().enumerate().map(|(i, p)| (p.page, i)).collect(); + for item in corrections.into_iter().flatten() { + if !item.is_object() || item.as_object().is_some_and(|o| o.keys().any(|k| !matches!(k.as_str(), "page" | "word" | "text"))) { + return Err(ToolError::InvalidArgs("corrections accept only page, word and text".into())); + } + let c = Args(item); + let page = + usize::try_from(c.int("page")?).ok().and_then(|p| p.checked_sub(1)).ok_or_else(|| Args::wrong("page", "a 1-based position"))?; + let word = + usize::try_from(c.int("word")?).ok().and_then(|w| w.checked_sub(1)).ok_or_else(|| Args::wrong("word", "a 1-based position"))?; + let text = c.str("text")?; + bytes = bytes.checked_add(text.len()).filter(|n| *n <= 4 * 1024 * 1024).ok_or_else(|| failed("OCR corrections exceed 4 MiB"))?; + if !seen.insert((page, word)) { + return Err(failed("duplicate OCR word correction")); + } + let pi = page_indices.get(&page).copied().ok_or_else(|| failed("page is not in this OCR review"))?; + if review.pages.get(pi).and_then(|p| p.words.get(word)).is_none() { + return Err(failed("word is not in this OCR review")); + } + edits.push((pi, word, text.to_owned())); + } + let mut pages = review.pages.clone(); + for (pi, wi, text) in edits { + if let Some(w) = pages.get_mut(pi).and_then(|p| p.words.get_mut(wi)) { + w.text = text; + } + } + pdfcraft_engine::ocr::validate_ocr_words(pages.iter().flat_map(|p| &p.words)).map_err(failed)?; + for page in &mut pages { + page.words.retain(|w| !w.text.trim().is_empty()); + } + if pages.iter().all(|p| p.words.is_empty()) { + return Err(failed("the OCR review has no words to apply")); + } + let out = match a.opt_str("output_mode")?.unwrap_or("searchable") { + "searchable" => { + if a.get("out").is_some() { + return Err(ToolError::InvalidArgs("out is only for editable output".into())); + } + let count = self.session.apply_ocr_checked(id, &pages, &review.guard).map_err(failed)?; + json!({ "applied": count, "review": handle }) + } + "editable" => { + // The editable output writes the review's words out of the document: a + // document that forbids copying refuses here (the review is kept). + self.doc_to_copy(a)?; + let target = self.resolve(a.str("out")?, true)?; + if self + .session + .get(id) + .and_then(|d| d.path.as_deref()) + .and_then(|p| std::fs::canonicalize(p).ok()) + .as_ref() + .is_some_and(|source| std::fs::canonicalize(&target).as_ref().ok() == Some(source)) + { + return Err(failed("out is the source document; choose a new file")); + } + // A NEW file: an existing one is never replaced silently (the review is kept). + if target.exists() { + return Err(failed(format!("out already exists ({}); choose a path that does not exist yet", target.to_string_lossy()))); + } + let bytes = self.session.editable_ocr_bytes(id, &pages).map_err(failed)?; + write_atomic(&target, &bytes)?; + json!({ "out": target.to_string_lossy(), "review": handle, "words": pages.iter().map(|p| p.words.len()).sum::() }) + } + _ => return Err(ToolError::InvalidArgs("output_mode must be searchable or editable".into())), + }; + self.ocr_reviews.remove(&id); + Ok(out) + } + + /// Read/update the same per-language dictionary file used by OCR Verify. + pub(crate) fn ocr_dictionary(&mut self, a: &Args) -> Result { + use pdfcraft_engine::ocr::dictionary; + use std::io::Read as _; + for key in ["add", "remove"] { + if let Some(value) = a.get(key) { + let values = value.as_array().ok_or_else(|| Args::wrong(key, "an array of strings"))?; + if values.len() > 65536 { + return Err(failed("at most 65536 dictionary words are allowed")); + } + } + } + let add = if a.get("add").is_some() { a.strs("add")? } else { Vec::new() }; + let remove = if a.get("remove").is_some() { a.strs("remove")? } else { Vec::new() }; + let writing = a.get("add").is_some() || a.get("remove").is_some(); + // Updating sorts, lower-cases and rewrites the file, so this tool names dictionary files only: + // pointed at `.env` or a README it would corrupt it. + let file = std::path::Path::new(a.str("path")?).file_name().and_then(|n| n.to_str()).unwrap_or(""); + let plain = file.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')); + if file.len() > 64 || !plain || !file.starts_with("ocr-dictionary") || !file.ends_with(".txt") { + return Err(failed("path must name a dictionary file: ocr-dictionary-.txt (letters, digits, '-', '_' and '.' only)")); + } + let path = self.resolve(a.str("path")?, writing)?; + let mut input = String::new(); + match std::fs::File::open(&path) { + Ok(file) => { + file.take((dictionary::MAX_BYTES + 1) as u64).read_to_string(&mut input).map_err(|e| failed(format!("OCR dictionary: {e}")))?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound && writing => {} + Err(e) => return Err(failed(format!("OCR dictionary: {e}"))), + } + let text = dictionary::update(&input, &add, &remove).map_err(failed)?; + if writing { + write_atomic(&path, text.as_bytes())?; + } + let words = dictionary::words(&text).map_err(failed)?; + Ok(json!({ "path": path.to_string_lossy(), "count": words.len(), "words": words })) } - /// Whether text recognition's models are installed, the directories searched for them (in - /// order), and the languages it reads, each as a code and a name. pub(crate) fn ocr_status(&self) -> Result { use pdfcraft_engine::ocr; let dirs: Vec = ocr::Models::search_dirs().iter().map(|d| d.to_string_lossy().into_owned()).collect(); let langs: Vec = ocr::LANGUAGES.iter().map(|(c, n)| json!({ "code": c, "name": n })).collect(); - Ok(json!({ "available": ocr::available(), "search_dirs": dirs, "languages": langs })) + let engines: Vec = ocr::engines_status() + .iter() + .map(|e| { + let mut v = json!({ "id": e.id, "available": e.available }); + if let Some(p) = &e.program { + v["path"] = json!(p); + } + if let Some(vv) = &e.version { + v["version"] = json!(vv); + } + if let Some(why) = &e.why_not { + v["unavailable"] = json!(why); + } + v + }) + .collect(); + Ok(json!({ "available": ocr::available(), "search_dirs": dirs, "languages": langs, "engines": engines })) } } diff --git a/crates/automation/src/batch.rs b/crates/automation/src/batch.rs index decb18d2b7..f650e565b7 100644 --- a/crates/automation/src/batch.rs +++ b/crates/automation/src/batch.rs @@ -84,8 +84,8 @@ impl<'a> Recipe<'a> { match *step { "convert" => bytes = session.convert_to_pdf(name, &bytes)?.1, "ocr" => { - let engine = ocr::engine().map_err(failed)?; - let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &engine, |_, _| true).map_err(failed)?; + let recognizers = ocr::recognizers(&self.ocr).map_err(failed)?; + let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &recognizers, |_, _| true).map_err(failed)?; log.push( json!({"step":"ocr","words":result.words(),"skipped_pages":result.pages.iter().filter(|p| p.skipped.is_some()).count()}), ); @@ -185,6 +185,7 @@ impl Automation { let recipe = Recipe::parse(self, args)?; let recursive = args.opt_bool("recursive")?.unwrap_or(false); let stop = args.opt_bool("stop_on_error")?.unwrap_or(false); + let dry_run = args.opt_bool("dry_run")?.unwrap_or(false); let paths = args.strs("paths")?; if paths.is_empty() || paths.len() > 1000 { return Err(Args::wrong("paths", "1 to 1000 files or folders")); @@ -211,11 +212,41 @@ impl Automation { let mut files = discovery.skipped; let (mut completed, mut failed_count) = (0, 0); let mut stopped = false; + let mut planned_outputs: HashSet = HashSet::new(); for (path, relative) in discovery.files { if stopped || !recipe.accepts(&path) { files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}})); continue; } + if dry_run { + let planned: Result = (|| { + let target = out.join(&relative).with_extension("pdf"); + let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?; + if !target.starts_with(&out) { + return Err(failed("output path leaves out_dir")); + } + let dir = target.parent().ok_or_else(|| failed("output has no parent"))?; + let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy().into_owned(); + for n in 1..=10_000 { + let candidate = if n == 1 { target.clone() } else { dir.join(format!("{stem} ({n}).pdf")) }; + if candidate.symlink_metadata().is_ok() || planned_outputs.contains(&candidate) { + continue; + } + planned_outputs.insert(candidate.clone()); + return Ok(candidate); + } + Err(failed("all 10000 output names are occupied")) + })(); + match planned { + Ok(target) => files.push(json!({"path":path.to_string_lossy(),"status":"planned","output":target.to_string_lossy().replace('\\', "/"),"steps":recipe.steps})), + Err(e) => { + failed_count += 1; + files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()})); + stopped = stop; + } + } + continue; + } // Recheck confinement and links before reading a discovered file. let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?; @@ -244,8 +275,8 @@ impl Automation { } } } - let skipped = files.len().saturating_sub(completed + failed_count); - Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) + let skipped = files.iter().filter(|file| file["status"] == "skipped").count(); + Ok(json!({"dry_run":dry_run,"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) } } @@ -279,4 +310,20 @@ mod tests { discovery.visited = 0; assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64")); } + + #[test] + fn doc_batch_dry_run_lists_outputs_without_writing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + let pdf = Session::new().create_from_text("Doc", "Hello").unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), &*pdf).unwrap(); + let auto = Automation::new().with_root(dir.path()).unwrap(); + let report = auto.doc_batch(&Args(&json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","dry_run":true}))).unwrap(); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["files"][0]["status"], "planned", "{report}"); + assert!(report["files"][0]["output"].as_str().unwrap().ends_with("out/a.pdf"), "{report}"); + assert!(!dir.path().join("out").exists()); + } } diff --git a/crates/automation/src/bates.rs b/crates/automation/src/bates.rs index 3cd1d7849e..5267716e85 100644 --- a/crates/automation/src/bates.rs +++ b/crates/automation/src/bates.rs @@ -199,7 +199,7 @@ impl Automation { } /// Publish a PDF atomically while refusing a target created by another process after preflight. -fn publish_new(path: &Path, bytes: &[u8]) -> Result<()> { +pub(super) fn publish_new(path: &Path, bytes: &[u8]) -> Result<()> { let parent = path.parent().ok_or_else(|| failed("output has no parent directory"))?; let mut temp = tempfile::NamedTempFile::new_in(parent).map_err(failed)?; temp.write_all(bytes).map_err(failed)?; diff --git a/crates/automation/src/bookmark_io.rs b/crates/automation/src/bookmark_io.rs new file mode 100644 index 0000000000..2a06d912f4 --- /dev/null +++ b/crates/automation/src/bookmark_io.rs @@ -0,0 +1,209 @@ +//! Bookmark trees in and out of documents: `bookmark_export`, `bookmark_import`, `bookmark_copy`. +//! +//! The portable form is JSON: `{"bookmarks": [node, …]}`, where a node is `title`, `page` +//! (1-based, or `null` for one that goes nowhere), `view` (`{"type", "values"}`: a destination +//! type and its values in PDF user space, as the file stores them), `color`, `bold`, `italic`, +//! `open` and `children`. What export writes, import takes back unchanged. A file also carries +//! `"truncated": true` when the export left out bookmarks nested past the listing limit. +use crate::{Args, Automation, Result, ToolError, failed}; +use pdfcraft_engine::{DocId, Edit, OutlineNode}; +use serde_json::{Value, json}; + +/// Deepest nesting a bookmark tree may have (the listing reaches 33 levels). +const MAX_DEPTH: usize = 33; + +/// The keys a node may have: the ones [`node_json`] writes. +const NODE_KEYS: [&str; 8] = ["title", "page", "view", "color", "bold", "italic", "open", "children"]; + +impl Automation { + pub(crate) fn bookmark_export(&self, a: &Args) -> Result { + let id = self.doc(a)?.id; + let path = a.opt_path("path")?.unwrap_or_default(); + let tree = self.outline_of(id, &path)?; + let mut out = json!({ "bookmarks": tree.nodes.iter().map(node_json).collect::>(), "truncated": tree.truncated }); + if let Some(file) = a.opt_str("out")? { + let target = self.resolve(file, true)?; + let mut saved = json!({ "bookmarks": out["bookmarks"] }); + if tree.truncated { + saved["truncated"] = json!(true); + } + let text = serde_json::to_vec_pretty(&saved).map_err(failed)?; + crate::write_atomic(&target, &text)?; + out["path"] = json!(target); + } + Ok(out) + } + + pub(crate) fn bookmark_import(&mut self, a: &Args) -> Result { + let (nodes, truncated) = match (a.get("bookmarks"), a.opt_str("file")?) { + (Some(list), None) => (nodes_of(list)?, false), + (None, Some(file)) => { + let path = self.resolve(file, false)?; + let value: Value = serde_json::from_slice(&crate::read_input(&path)?).map_err(|e| failed(format!("not a bookmark file: {e}")))?; + let list = value.get("bookmarks").ok_or_else(|| failed("not a bookmark file: it has no \"bookmarks\" list"))?; + (nodes_of(list)?, value.get("truncated") == Some(&Value::Bool(true))) + } + _ => return Err(ToolError::InvalidArgs("give either bookmarks or file".into())), + }; + let mut out = self.write_tree(a, nodes)?; + out["truncated"] = json!(truncated); + Ok(out) + } + + pub(crate) fn bookmark_copy(&mut self, a: &Args) -> Result { + let from = a.int("from")?; + let from = DocId(u64::try_from(from).map_err(|_| ToolError::InvalidArgs("from must be positive".into()))?); + let path = a.opt_path("path")?.unwrap_or_default(); + let tree = self.outline_of(from, &path)?; + let mut out = self.write_tree(a, tree.nodes)?; + out["truncated"] = json!(tree.truncated); + Ok(out) + } + + fn outline_of(&self, id: DocId, path: &[usize]) -> Result { + match self.session.outline_tree(id, path) { + // The engine names the path 0-based; the caller gave it 1-based. + Some(tree) => tree.map_err(|e| match e { + pdfcraft_organize::OutlineError::NoSuchBookmark(_) => failed(format!("there is no bookmark at {:?}", one_based(path))), + e => failed(e), + }), + None => Err(failed(format!("document {} is not open, or its bookmarks can't be read for editing", id.0))), + } + } + + /// Write `nodes` into `doc` under `parent` at `position` (or in place of the outline with + /// `replace`), and say which of them went nowhere because the document lacks their page. + fn write_tree(&mut self, a: &Args, bookmarks: Vec) -> Result { + let parent = a.opt_path("parent")?.unwrap_or_default(); + let index = a.opt_int("position")?.map_or(usize::MAX, |p| usize::try_from(p.max(1) - 1).unwrap_or(usize::MAX)); + let replace = a.opt_bool("replace")?.unwrap_or(false); + let count = self.doc(a)?.info.pages.len(); + let unmapped = pdfcraft_engine::outline_unmapped(&bookmarks, count); + let mut out = self.apply(a, Edit::ImportBookmarks { parent, index, bookmarks, replace })?; + out["unmapped"] = json!(unmapped.iter().map(|p| one_based(p)).collect::>()); + Ok(out) + } +} + +fn one_based(path: &[usize]) -> Vec { + path.iter().map(|i| i.saturating_add(1)).collect() +} + +/// A node as JSON, leaving out what is at its default (no view, black, regular, no children). +fn node_json(n: &OutlineNode) -> Value { + let mut v = json!({ "title": n.title, "page": n.page.map(|p| p.saturating_add(1)), "open": n.open }); + if let Some((kind, values)) = &n.view { + v["view"] = json!({ "type": String::from_utf8_lossy(kind), "values": values }); + } + if let Some(c) = n.color { + v["color"] = json!(c); + } + if n.bold { + v["bold"] = json!(true); + } + if n.italic { + v["italic"] = json!(true); + } + if !n.children.is_empty() { + v["children"] = json!(n.children.iter().map(node_json).collect::>()); + } + v +} + +fn nodes_of(list: &Value) -> Result> { + let mut count = 0; + list_of(list, &[], &mut count) +} + +/// The nodes of one level; `at` is the 1-based path of their parent, for messages. +fn list_of(list: &Value, at: &[usize], count: &mut usize) -> Result> { + let items = list.as_array().ok_or_else(|| bad(at, "children must be a list of bookmarks"))?; + if at.len() >= MAX_DEPTH && !items.is_empty() { + return Err(bad(at, &format!("bookmarks may nest at most {MAX_DEPTH} levels deep"))); + } + *count = count.saturating_add(items.len()); + if *count > pdfcraft_engine::MAX_IMPORT_BOOKMARKS { + return Err(ToolError::InvalidArgs(format!("at most {} bookmarks can be imported at once", pdfcraft_engine::MAX_IMPORT_BOOKMARKS))); + } + let mut out = Vec::with_capacity(items.len()); + for (i, item) in items.iter().enumerate() { + let mut path = at.to_vec(); + path.push(i.saturating_add(1)); + out.push(node_of(item, &path, count)?); + } + Ok(out) +} + +/// One node and, through [`list_of`], its children (recursion bounded by `MAX_DEPTH`). +fn node_of(v: &Value, at: &[usize], count: &mut usize) -> Result { + let o = v.as_object().ok_or_else(|| bad(at, "must be an object"))?; + if let Some(k) = o.keys().find(|k| !NODE_KEYS.contains(&k.as_str())) { + return Err(bad(at, &format!("unknown key {k:?} (a bookmark has {})", NODE_KEYS.join(", ")))); + } + let field = |k: &str| o.get(k).filter(|x| !x.is_null()); + // An empty title is kept: files have them, and export writes them as "". + let title = field("title").and_then(Value::as_str).ok_or_else(|| bad(at, "needs a title"))?.to_string(); + let page = match field("page") { + None => None, + Some(p) => Some(p.as_u64().filter(|p| *p >= 1).and_then(|p| usize::try_from(p - 1).ok()).ok_or_else(|| bad(at, "page must be 1 or more"))?), + }; + let flag = |k: &str| field(k).map_or(Ok(None), |x| x.as_bool().map(Some).ok_or_else(|| bad(at, &format!("{k} must be true or false")))); + let color = match field("color") { + None => None, + Some(c) => { + let rgb: Option> = c.as_array().and_then(|c| c.iter().map(|x| x.as_f64().map(|x| x as f32)).collect()); + match rgb.as_deref() { + Some(&[r, g, b]) if [r, g, b].iter().all(|c| (0.0..=1.0).contains(c)) => Some([r, g, b]), + _ => return Err(bad(at, "color must be [r, g, b], each from 0 to 1")), + } + } + }; + let children = match field("children") { + None => Vec::new(), + Some(list) => list_of(list, at, count)?, + }; + Ok(OutlineNode { + title, + page, + view: field("view").map(|x| view_of(x, at)).transpose()?, + color, + italic: flag("italic")?.unwrap_or(false), + bold: flag("bold")?.unwrap_or(false), + open: flag("open")?.unwrap_or(true), + children, + }) +} + +/// `{"type": "XYZ", "values": [left, top, zoom]}`: a destination type with exactly its values. +fn view_of(v: &Value, at: &[usize]) -> Result<(Vec, Vec>)> { + let kind = v.get("type").and_then(Value::as_str).ok_or_else(|| bad(at, "view needs a type such as XYZ or Fit"))?; + let values = match v.get("values").filter(|x| !x.is_null()) { + None => Vec::new(), + Some(list) => { + let list = list.as_array().filter(|l| l.len() <= 4).ok_or_else(|| bad(at, "view values must be a list of at most 4 numbers or nulls"))?; + list.iter() + .map(|x| match x { + Value::Null => Ok(None), + x => x.as_f64().map(Some).ok_or_else(|| bad(at, "view values must be numbers or null")), + }) + .collect::>()? + } + }; + let arity = pdfcraft_engine::outline_view_arity(kind.as_bytes()).ok_or_else(|| bad(at, VIEW_TYPES))?; + if values.len() != arity { + return Err(bad(at, &format!("a {kind} view takes {arity} values (numbers, or null to keep the reader's)"))); + } + if kind == "FitR" && values.iter().any(Option::is_none) { + return Err(bad(at, "a FitR view needs all four values: left, bottom, right, top")); + } + Ok((kind.as_bytes().to_vec(), values)) +} + +const VIEW_TYPES: &str = "view type must be XYZ (3 values: left, top, zoom), Fit, FitB (none), FitH, FitV, FitBH, FitBV (1) or FitR (4)"; + +fn bad(at: &[usize], what: &str) -> ToolError { + if at.is_empty() { + return ToolError::InvalidArgs(format!("bookmarks: {what}")); + } + ToolError::InvalidArgs(format!("bookmark {at:?}: {what}")) +} diff --git a/crates/automation/src/comments.rs b/crates/automation/src/comments.rs index ae46be87d0..78f4cefbe0 100644 --- a/crates/automation/src/comments.rs +++ b/crates/automation/src/comments.rs @@ -5,7 +5,9 @@ use std::collections::HashMap; -use pdfcraft_engine::{Edit, LineEnding, Markup, NewAnnotation, NoteIcon, ReviewState, Rgb, Shape, StampGroup, StampKind, Style, SummarySort}; +use pdfcraft_engine::{ + BorderShape, Edit, LineEnding, Markup, NewAnnotation, NoteIcon, ReviewState, Rgb, Shape, StampGroup, StampKind, Style, SummarySort, +}; use pdfcraft_render::{Annotation, PageInfo}; use serde_json::{Value, json}; @@ -75,7 +77,7 @@ fn to_user(p: &PageInfo, x: f64, y: f64) -> [f64; 2] { [ux as f64, uy as f64] } -fn rect_to_user(p: &PageInfo, r: [f64; 4]) -> [f64; 4] { +pub(crate) fn rect_to_user(p: &PageInfo, r: [f64; 4]) -> [f64; 4] { let (a, b) = (to_user(p, r[0], r[1]), to_user(p, r[2], r[3])); [a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])] } @@ -190,6 +192,25 @@ impl Automation { "rect": crate::view_rect(&doc.info, c.page, c.shown_rect(doc.info.pages.get(c.page).map_or(0, |p| p.rotation))), "color": c.color.map(hex), }); + // A text box's or typed text's oval or circle (a rectangle is not reported). + if c.subtype == "FreeText" + && let Some(shape) = doc.comment_props(c.page, c.index).and_then(|p| p.border_shape).filter(|s| *s != BorderShape::Rectangle) + { + v["border_shape"] = json!(shape.name().to_ascii_lowercase()); + } + // How opaque it is drawn: as a whole, or its fill and text each at their own. + // Not for replies, which are drawn with the comment they answer. + if !c.is_reply() + && let Some(p) = doc.comment_props(c.page, c.index) + { + v["opacity"] = json!(p.opacity); + if let Some(f) = p.fill_opacity { + v["fill_opacity"] = json!(f); + } + if let Some(t) = p.text_opacity { + v["text_opacity"] = json!(t); + } + } // A line's ends, or the corners of connected lines, a polygon or a cloud, as // comment_edit's `points` takes them. // Unchanged on a page the viewer could not read, as `rect` is. @@ -253,6 +274,12 @@ impl Automation { if endings.is_some() && !matches!(kind, "line" | "arrow" | "polyline" | "callout") { return Err(ToolError::InvalidArgs("endings apply to a line, arrow, polyline or callout".into())); } + // Shift while drawing: each end or corner held level or plumb with the one before. + let ortho = a.opt_bool("ortho")?.unwrap_or(false); + if ortho && !matches!(kind, "line" | "arrow" | "polygon" | "cloud" | "polyline") { + return Err(ToolError::InvalidArgs("ortho applies to a line, arrow, polygon, cloud or polyline".into())); + } + let held = |last: [f64; 2], p: [f64; 2]| if ortho { pdfcraft_engine::measure::snap::ortho(last, p) } else { p }; let markup = match kind { "highlight" => Some(Markup::Highlight), "underline" => Some(Markup::Underline), @@ -325,7 +352,8 @@ impl Automation { Shape::TextBox { rect: rect_to_user(&info, a.need::<4>("rect", "a text box")?), font_size } } "line" | "arrow" => { - let (f, t) = (a.need::<2>("from", "a line")?, a.need::<2>("to", "a line")?); + let f = a.need::<2>("from", "a line")?; + let t = held(f, a.need::<2>("to", "a line")?); let [start, end] = ending_pair(kind, endings.as_deref())?; Shape::Line { from: to_user(&info, f[0], f[1]), to: to_user(&info, t[0], t[1]), start, end } } @@ -352,10 +380,15 @@ impl Automation { "polygon" | "cloud" | "polyline" => { let wrong = || ToolError::InvalidArgs(format!("{kind} needs `points`: an array of [x, y] points")); let pts = a.get("points").and_then(|p| p.as_array()).ok_or_else(wrong)?; + let mut last = None; let vertices = pts .iter() .map(|p| match p.as_array().map(|p| p.iter().map(|v| v.as_f64()).collect::>()).as_deref() { - Some([Some(x), Some(y)]) => Ok(to_user(&info, *x, *y)), + Some([Some(x), Some(y)]) => { + let p = last.map_or([*x, *y], |l| held(l, [*x, *y])); + last = Some(p); + Ok(to_user(&info, p[0], p[1])) + } _ => Err(wrong()), }) .collect::>>()?; @@ -425,9 +458,28 @@ impl Automation { if let Some(o) = a.opt_num("opacity")? { style.opacity = o.clamp(0.0, 1.0); } + // The fill's and a text box's text's own opacities (the line keeps `opacity`). + let (fill_opacity, text_opacity) = (a.opt_num("fill_opacity")?, a.opt_num("text_opacity")?); + let texts = matches!(shape, Shape::TextBox { .. } | Shape::Typewriter { .. } | Shape::Callout { .. }); + if fill_opacity.is_some() && !(texts || matches!(shape, Shape::Rectangle { .. } | Shape::Oval { .. } | Shape::Polygon { .. })) { + return Err(ToolError::InvalidArgs( + "fill_opacity applies to a rectangle, oval, polygon, cloud, text box, typewriter text or callout".into(), + )); + } + if text_opacity.is_some() && !texts { + return Err(ToolError::InvalidArgs("text_opacity applies to a text box, typewriter text or callout".into())); + } + style.fill_opacity = fill_opacity.map(|o| o.clamp(0.0, 1.0)); + style.text_opacity = text_opacity.map(|o| o.clamp(0.0, 1.0)); if let Some(w) = a.opt_num("width")? { style.width = w.max(0.0); } + if let Some(b) = border_shape(a)? { + if !matches!(shape, Shape::TextBox { .. } | Shape::Typewriter { .. }) { + return Err(ToolError::InvalidArgs("border_shape is for a text box or typewriter text".into())); + } + style.border_shape = b; + } let contents = a.opt_str("contents")?.unwrap_or_default().to_string(); let author = a.opt_str("author")?.unwrap_or(DEFAULT_AUTHOR).to_string(); // Replace Text: the struck-out text and a grouped caret holding `contents`. @@ -488,6 +540,7 @@ impl Automation { } pub(crate) fn fill_sign_add(&mut self, a: &Args) -> Result { + let border_shape = border_shape(a)?; use pdfcraft_engine::FillMark; let page = self.page(a)?; let info = self.doc(a)?.info.pages[page].clone(); @@ -515,6 +568,25 @@ impl Automation { if !(4.0..=300.0).contains(&size) { return Err(ToolError::InvalidArgs("font_size must be 4–300 pt".into())); } + let requested_border = a.opt_num("border")?; + if requested_border.is_some() && !matches!(kind, "text" | "date") { + return Err(ToolError::InvalidArgs("border is only for type text or date".into())); + } + let border = requested_border.unwrap_or(0.0); + if !(0.0..=12.0).contains(&border) { + return Err(ToolError::InvalidArgs("border must be 0–12 pt".into())); + } + let border_color = a.opt_str("border_color")?.map(parse_color).transpose()?; + if border_color.is_some() && border <= 0.0 { + return Err(ToolError::InvalidArgs("border_color needs a border".into())); + } + if border_shape.is_some() && !matches!(kind, "text" | "date") { + return Err(ToolError::InvalidArgs("border_shape is only for type text or date".into())); + } + // Without a border an oval or a circle would grow the box around nothing drawn. + if border_shape.is_some_and(|s| s != BorderShape::Rectangle) && border <= 0.0 { + return Err(ToolError::InvalidArgs("border_shape needs a border".into())); + } if let Some(path) = a.opt_str("path")? { if !matches!(kind, "signature" | "initials") || a.opt_str("text")?.is_some() { return Err(ToolError::InvalidArgs("path is only for an image signature or initials; pass either path or text".into())); @@ -538,13 +610,16 @@ impl Automation { let edit = image.edit(page, &info, at, kind == "initials", &author).ok_or_else(|| ToolError::InvalidArgs("at must be finite".into()))?; return self.apply(a, edit); } + // A circle goes around the text, so its box is no wider than the text needs. + let min_width = if border_shape == Some(BorderShape::Circle) { 0.0 } else { 20.0 }; let text_at = |t: &str| { let longest = t.lines().map(|line| pdfcraft_engine::annot_text::text_width(line, size)).fold(0.0, f64::max); - let w = (longest + 8.0).clamp(20.0, 600.0); + let w = (longest + 8.0).clamp(min_width, 600.0); let rows = pdfcraft_engine::annot_text::wrap(t, size, w - 4.0).len().max(1); let h = rows as f64 * size * 1.2 + 6.0; - let rect = pdfcraft_engine::text_box_at(i64::from(info.rotation), at, w, h).unwrap_or([at[0], at[1] - h, at[0] + w, at[1]]); - Shape::Typewriter { rect, font_size: size } + let [x0, y0, x1, y1] = pdfcraft_engine::text_box_at(i64::from(info.rotation), at, w, h).unwrap_or([at[0], at[1] - h, at[0] + w, at[1]]); + // A border goes around the box, so the text sits where it would without one. + Shape::Typewriter { rect: [x0 - border, y0 - border, x1 + border, y1 + border], font_size: size } }; let (shape, contents) = match kind { "text" => { @@ -590,6 +665,11 @@ impl Automation { if let Some(color) = requested_color { style.color = color; } + if matches!(shape, Shape::Typewriter { .. }) { + style.width = border; + style.border_color = border_color; + style.border_shape = border_shape.unwrap_or_default(); + } self.apply(a, Edit::AddAnnotation(NewAnnotation { page, shape, style, contents, author })) } @@ -656,6 +736,21 @@ impl Automation { let fill = if f.eq_ignore_ascii_case("none") { None } else { Some(parse_color(f)?) }; edits.push(Edit::FillAnnotation { page, index, fill }); } + if let Some(c) = a.opt_str("border_color")? { + let color = if c.eq_ignore_ascii_case("text") { None } else { Some(parse_color(c)?) }; + edits.push(Edit::BorderColorAnnotation { page, index, color }); + } + if let Some(shape) = border_shape(a)? { + edits.push(Edit::BorderShapeAnnotation { page, index, shape }); + } + // After `opacity`, which sets the whole comment's. + let (fill_opacity, text_opacity) = (a.opt_num("fill_opacity")?, a.opt_num("text_opacity")?); + if fill_opacity.into_iter().chain(text_opacity).any(|o| !(0.0..=1.0).contains(&o)) { + return Err(ToolError::InvalidArgs("fill_opacity and text_opacity are 0–1".into())); + } + if fill_opacity.is_some() || text_opacity.is_some() { + edits.push(Edit::PartOpacity { page, index, fill: fill_opacity, text: text_opacity }); + } if let Some(size) = a.opt_num("font_size")? { if !(1.0..=400.0).contains(&size) { return Err(ToolError::InvalidArgs("font_size must be 1–400 pt".into())); @@ -689,7 +784,7 @@ impl Automation { } if edits.is_empty() { return Err(ToolError::InvalidArgs( - "nothing to change: pass contents, color, opacity, width, fill, font_size, rect, move or points".into(), + "nothing to change: pass contents, color, opacity, fill_opacity, text_opacity, width, fill, border_color, border_shape, font_size, rect, move or points".into(), )); } let edit = if edits.len() == 1 { edits.remove(0) } else { Edit::Batch { label: "Edit comment".into(), edits } }; @@ -700,6 +795,49 @@ impl Automation { let (page, index) = self.comment_target(a)?; self.apply(a, Edit::DeleteAnnotation { page, index }) } + + /// The XFDF of the listed comments and their replies (`comments`: 1-based [page, index] pairs). + pub(crate) fn comments_copy(&self, a: &Args) -> Result { + let id = self.doc(a)?.id; + let all = self.comments(a)?; + let shape = || ToolError::InvalidArgs("comments: [[page, index], …] (1-based, as comment_list reports)".into()); + let list = a.get("comments").and_then(Value::as_array).ok_or_else(shape)?; + let mut which = Vec::with_capacity(list.len()); + for item in list { + let pair = item.as_array().filter(|p| p.len() == 2).and_then(|p| Some((p.first()?.as_u64()?, p.get(1)?.as_u64()?))); + let Some((page, index)) = pair.filter(|(p, i)| *p >= 1 && *i >= 1) else { return Err(shape()) }; + let (page, index) = (usize::try_from(page - 1).map_err(|_| shape())?, usize::try_from(index - 1).map_err(|_| shape())?); + if !all.iter().any(|c| c.page == page && c.index == index) { + return Err(failed(format!("there is no comment {} on page {} (see comment_list)", index + 1, page + 1))); + } + which.push((page, index)); + } + let xfdf = self.session.export_comments(id, &which).map_err(failed)?; + Ok(json!({ "xfdf": xfdf, "comments": which.len() })) + } + + /// Add the comments of `xfdf` (from `comments_copy`) to `page` as new comments. + pub(crate) fn comments_paste(&mut self, a: &Args) -> Result { + let page = self.page(a)?; + let xfdf = a.str("xfdf")?.to_owned(); + if !pdfcraft_engine::is_xfdf_comments(&xfdf) { + return Err(ToolError::InvalidArgs("xfdf: the text comments_copy returned (XFDF with )".into())); + } + let offset = match a.nums::<2>("at")? { + Some([x, y]) => { + let info = self.doc(a)?.info.pages.get(page).cloned().ok_or_else(|| failed(format!("no page {}", page + 1)))?; + let [cx, cy] = to_user(&info, x, y); + let b = + pdfcraft_engine::comments_bounds(&xfdf).ok_or_else(|| ToolError::InvalidArgs("xfdf: no comment with a rect to place".into()))?; + Some([cx - (b[0] + b[2]) / 2.0, cy - (b[1] + b[3]) / 2.0]) + } + None => None, + }; + let before = self.comments(a)?.len(); + let mut out = self.apply(a, Edit::PasteComments { xfdf, page, offset })?; + out["comments"] = json!(self.comments(a)?.len().saturating_sub(before)); + Ok(out) + } } impl Automation { @@ -744,6 +882,13 @@ impl Automation { } } +/// `border_shape`: `rectangle`, `oval` or `circle`. +fn border_shape(a: &Args) -> Result> { + a.opt_str("border_shape")? + .map(|s| BorderShape::parse(s).ok_or_else(|| ToolError::InvalidArgs(format!("unknown border_shape {s:?}: rectangle, oval or circle")))) + .transpose() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/automation/src/content.rs b/crates/automation/src/content.rs index ac9b61d651..b11114e5b3 100644 --- a/crates/automation/src/content.rs +++ b/crates/automation/src/content.rs @@ -2,7 +2,7 @@ //! move/resize/retype/restyle it, delete it. Geometry is in points from the top-left of the //! displayed page (y down), like every other tool. -use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign}; +use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign, TextInk}; use serde_json::{Value, json}; use crate::comments::parse_color; @@ -30,6 +30,27 @@ pub(crate) fn rect_arg(a: &Args, key: &str) -> Result> { .transpose() } +const INK_ARGS: &str = "give one of color (#RRGGBB or a name), gray (0 black to 1 white) and cmyk ([c, m, y, k], each 0 to 1)"; + +/// `gray` or `cmyk`: the text painted in DeviceGray or DeviceCMYK rather than RGB. At most one of +/// them and `color`. +fn ink_arg(a: &Args) -> Result> { + if ["color", "gray", "cmyk"].iter().filter(|k| a.get(k).is_some()).count() > 1 { + return Err(bad(INK_ARGS)); + } + let unit = |v: Option, key: &str| v.filter(|v| (0.0..=1.0).contains(v)).ok_or_else(|| bad(format!("{key} takes numbers from 0 to 1"))); + if let Some(g) = a.get("gray") { + return Ok(Some(TextInk::Gray(unit(g.as_f64(), "gray")?))); + } + let Some(c) = a.get("cmyk") else { return Ok(None) }; + let parts = c.as_array().filter(|p| p.len() == 4).ok_or_else(|| bad("cmyk must be [c, m, y, k]"))?; + let mut cmyk = [0.0; 4]; + for (out, v) in cmyk.iter_mut().zip(parts) { + *out = unit(v.as_f64(), "cmyk")?; + } + Ok(Some(TextInk::Cmyk(cmyk))) +} + /// Apply the style arguments to `t`. fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(s) = a.opt_num("size")? { @@ -49,8 +70,12 @@ fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(i) = a.opt_bool("italic")? { t.italic = i; } - if let Some(c) = a.opt_str("color")? { - t.color = parse_color(c)?; + if let Some(ink) = ink_arg(a)? { + t.set_ink(Some(ink)); + } else if let Some(c) = a.opt_str("color")? { + // An RGB colour paints in RGB, even one that is the ink's look. + t.set_ink(None); + t.set_color(parse_color(c)?); } if let Some(al) = a.opt_str("align")? { t.align = parse_align(al)?; @@ -124,6 +149,12 @@ impl Automation { v["bold"] = json!(t.bold); v["italic"] = json!(t.italic); v["char_spacing"] = json!(t.char_spacing); + // Text painted in a print colour space says so; RGB text has neither. + match t.painted_ink() { + Some(TextInk::Gray(g)) => v["gray"] = json!(g), + Some(TextInk::Cmyk(c)) => v["cmyk"] = json!(c), + None => {} + } } AddedContent::Image(_) => v["type"] = json!("image"), } @@ -205,8 +236,8 @@ impl Automation { if a.get("text").is_some() { return Err(bad("images have no text")); } - if ["font", "size", "bold", "italic", "color", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { - return Err(bad("font, size, bold, italic, color, align, aligns and char_spacing apply to text")); + if ["font", "size", "bold", "italic", "color", "gray", "cmyk", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { + return Err(bad("font, size, bold, italic, color, gray, cmyk, align, aligns and char_spacing apply to text")); } if let Some(deg) = a.opt_int("rotate")? { if deg.rem_euclid(90) != 0 { diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index f3c51d22e0..ebe2a93526 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -69,6 +69,7 @@ impl Automation { self.apply(&a, edit)? } "page_render" => return self.page_render(&a), + "model_render" => return self.model_render(&a), "comment_image_preview" => return self.comment_image_preview(&a), "text_extract" => self.text_extract(&a)?, "text_word_count" => self.text_word_count(&a)?, @@ -136,6 +137,7 @@ impl Automation { out["detected"] = json!(given.is_none()); out } + "page_enhance_scan" => self.page_enhance_scan(&a)?, "page_delete" => { let pages = self.pages(&a, "pages")?; self.apply(&a, Edit::DeletePages { pages })? @@ -196,6 +198,9 @@ impl Automation { out } "bookmark_list" => self.bookmark_listing(&a)?, + "bookmark_export" => self.bookmark_export(&a)?, + "bookmark_import" => self.bookmark_import(&a)?, + "bookmark_copy" => self.bookmark_copy(&a)?, "bookmark_add" => { let page = self.page(&a)?; let parent = a.opt_path("parent")?.unwrap_or_default(); @@ -769,6 +774,7 @@ impl Automation { "comment_list" => self.comment_list(&a)?, "comment_add" => self.comment_add(&a)?, "stamp_custom" => self.stamp_custom(&a)?, + "model_add" => self.model_add(&a)?, "comment_reply" => self.comment_reply(&a)?, "comment_set_status" => self.comment_set_status(&a)?, "sign_list" => self.sign_list(&a)?, @@ -806,6 +812,9 @@ impl Automation { "form_set_actions" => self.form_set_actions(&a)?, "ocr_status" => self.ocr_status()?, "ocr_recognize_files" => self.ocr_recognize_files(&a)?, + "ocr_words" => self.ocr_words(&a)?, + "ocr_review_apply" => self.ocr_review_apply(&a)?, + "ocr_dictionary" => self.ocr_dictionary(&a)?, "accessibility_report" => self.a11y_report(&a)?, "accessibility_review_record" => self.a11y_review_record(&a)?, "accessibility_review_save" => self.a11y_review_save(&a)?, @@ -861,6 +870,8 @@ impl Automation { "comments_summarize" => self.comments_summarize(&a)?, "comment_edit" => self.comment_edit(&a)?, "comment_delete" => self.comment_delete(&a)?, + "comments_copy" => self.comments_copy(&a)?, + "comments_paste" => self.comments_paste(&a)?, other => return Err(ToolError::UnknownTool(other.into())), }; Ok(vec![Content::Json(out)]) @@ -914,3 +925,35 @@ impl Automation { Ok(out) } } + +impl Automation { + /// `page_enhance_scan`: Enhance ▸ Scanned document on the chosen pages (all by default). + fn page_enhance_scan(&mut self, a: &Args) -> Result { + let pages = match a.opt_ints("pages")? { + Some(_) => self.pages(a, "pages")?, + None => Vec::new(), + }; + let level = match a.opt_int("level")? { + None => pdfcraft_engine::scan_clean::ScanCleanSettings::default().level, + Some(l @ 0..=100) => l as u8, + Some(l) => return Err(ToolError::InvalidArgs(format!("level must be between 0 and 100, not {l}"))), + }; + let mut settings = pdfcraft_engine::scan_clean::ScanCleanSettings::at(level); + if let Some(on) = a.opt_bool("edge_shadows")? { + settings.edge_shadows = on; + } + if let Some(on) = a.opt_bool("despeckle")? { + settings.despeckle = on; + } + let id = self.doc(a)?.id; + let report = self.session.enhance_scanned_pages(id, &pages, settings).map_err(failed)?; + let numbered = |p: &[usize]| p.iter().map(|p| p + 1).collect::>(); + let skipped: Vec = report.skipped.iter().map(|k| json!({ "pages": numbered(&k.pages), "name": k.name, "reason": k.reason })).collect(); + let mut out = summary(self.doc(a)?); + out["pages_cleaned"] = json!(numbered(&report.pages)); + out["images"] = json!(report.images); + out["not_scanned"] = json!(numbered(&report.not_scanned)); + out["skipped"] = json!(skipped); + Ok(out) + } +} diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index 979b041a73..f8c7988bbc 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -120,6 +120,7 @@ impl Automation { self.texts.remove(&id); self.listed.retain(|(d, _), _| *d != id); self.evicted.remove(&id); + self.ocr_reviews.remove(&id); Ok(json!({ "closed": id.0 })) } diff --git a/crates/automation/src/file_access.rs b/crates/automation/src/file_access.rs index 067a593888..dab2760f5d 100644 --- a/crates/automation/src/file_access.rs +++ b/crates/automation/src/file_access.rs @@ -394,8 +394,18 @@ mod tests { // Helpers that are only called with a resolved path. `doc_batch` (batch.rs) resolves its // inputs and out_dir, walks only inside them (skipping links) and resolves every // discovered file again before `process` reads it or `publish` writes it. - const RESOLVED_BY_CALLER: &[&str] = - &["write_atomic", "write_atomic_with", "read_input", "read_input_bytes", "read_review_sidecar", "process", "linked", "walk", "publish"]; + const RESOLVED_BY_CALLER: &[&str] = &[ + "write_atomic", + "write_atomic_with", + "read_input", + "read_input_bytes", + "read_model", + "read_review_sidecar", + "process", + "linked", + "walk", + "publish", + ]; let mut dirs = vec![Path::new(env!("CARGO_MANIFEST_DIR")).join("src")]; let mut files = vec![]; while let Some(dir) = dirs.pop() { diff --git a/crates/automation/src/json.rs b/crates/automation/src/json.rs index 9e93d4c347..c950e72f05 100644 --- a/crates/automation/src/json.rs +++ b/crates/automation/src/json.rs @@ -375,6 +375,9 @@ pub(crate) fn info(d: &Document) -> Value { })).collect::>(), "warnings": i.warnings, "repairs": d.repair_log(), + // Objects that could not be read (each reads as null): nothing was repaired, what they held is + // lost. Last, so it includes what reading the fields above failed on. + "unreadable_objects": d.unreadable_objects(), }) } diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 5bcb032fce..7e4443d244 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -27,6 +27,8 @@ mod links; #[cfg(feature = "mcp")] pub mod mcp; mod measure; +mod model3d; +mod model_render; mod printing; mod reading; mod redact; @@ -50,6 +52,7 @@ use file_access::foreign_share; use text_cache::TextCache; mod assemble; +mod bookmark_io; mod dispatch; mod documents; mod error; @@ -73,6 +76,12 @@ pub(crate) use paths::{read_input, read_input_bytes}; #[cfg(test)] use paths::{write_atomic_synced, write_atomic_with}; +struct OcrReview { + handle: u64, + guard: pdfcraft_engine::ocr::OcrGuard, + pages: Vec, +} + /// A headless PdfCraft session driven by tool calls. pub struct Automation { session: Session, @@ -89,6 +98,9 @@ pub struct Automation { /// Documents that lost a listing to the cap. A page of one with no listing is refused rather /// than read as "a number is a position", which the page may no longer match. evicted: HashSet, + /// Review sessions held for `ocr_words` (a guard and the pages read), by document. + ocr_reviews: HashMap, + next_ocr_review: u64, } impl Default for Automation { @@ -107,6 +119,8 @@ impl Automation { texts: TextCache::default(), listed: HashMap::new(), evicted: HashSet::new(), + ocr_reviews: HashMap::new(), + next_ocr_review: 0, } } diff --git a/crates/automation/src/mcp.rs b/crates/automation/src/mcp.rs index 84cc21b258..b033871867 100644 --- a/crates/automation/src/mcp.rs +++ b/crates/automation/src/mcp.rs @@ -557,7 +557,7 @@ fn tool_json(t: &crate::ToolDef) -> Value { "title": t.title, "description": t.description, "inputSchema": t.input_schema, - "annotations": { "title": t.title, "readOnlyHint": t.read_only, "destructiveHint": t.destructive, "idempotentHint": t.read_only || matches!(t.name, "doc_save" | "doc_optimize" | "doc_export_data" | "measure_export" | "doc_export_images" | "doc_export_office" | "doc_export_all_images" | "doc_export_text" | "image_save"), "openWorldHint": false }, + "annotations": { "title": t.title, "readOnlyHint": t.read_only, "destructiveHint": t.destructive, "idempotentHint": t.read_only || matches!(t.name, "doc_save" | "doc_optimize" | "doc_export_data" | "measure_export" | "doc_export_images" | "doc_export_office" | "doc_export_all_images" | "doc_export_text" | "image_save" | "bookmark_export"), "openWorldHint": false }, }) } diff --git a/crates/automation/src/model3d.rs b/crates/automation/src/model3d.rs new file mode 100644 index 0000000000..2d89e4ae60 --- /dev/null +++ b/crates/automation/src/model3d.rs @@ -0,0 +1,114 @@ +//! 3D model tools (Acrobat: Rich Media ▸ Add 3D, AGENTS.md §3). + +use std::path::Path; + +use pdfcraft_engine::{Activation3d, Edit, MAX_3D_MODEL_BYTES, MarkFile, Model3dFormat}; +use serde_json::Value; + +use crate::comments::rect_to_user; +use crate::{Args, Automation, Result, ToolError, failed}; + +/// The largest side of a 3D model's box, in points: the largest page PDF allows (ISO 32000-2 Annex C). +const MAX_MODEL_SIDE: f64 = 14_400.0; + +impl Automation { + /// A U3D or PRC model in `rect` on `page`, its poster the picture at `poster` (a PDF page or an + /// image) fitted inside the box, or a labelled frame. + pub(crate) fn model_add(&mut self, a: &Args) -> Result { + let page = self.page(a)?; + let info = self.doc(a)?.info.pages.get(page).cloned().ok_or_else(|| ToolError::InvalidArgs(format!("page {} does not exist", page + 1)))?; + let r = a.nums::<4>("rect")?.ok_or_else(|| ToolError::InvalidArgs("missing argument rect".into()))?; + let (w, h) = ((r[2] - r[0]).abs(), (r[3] - r[1]).abs()); + if !((1.0..=MAX_MODEL_SIDE).contains(&w) && (1.0..=MAX_MODEL_SIDE).contains(&h)) { + return Err(ToolError::InvalidArgs(format!( + "rect must be [x0, y0, x1, y1] with each side at least 1 pt and at most {MAX_MODEL_SIDE} pt" + ))); + } + let activation = match a.opt_str("activation")? { + None => Activation3d::default(), + Some(word) => Activation3d::ALL + .iter() + .find(|(_, w)| *w == word) + .map(|(act, _)| *act) + .ok_or_else(|| ToolError::InvalidArgs(format!("activation must be one of: {}", Activation3d::ALL.map(|(_, w)| w).join(", "))))?, + }; + let path = self.resolve(a.str("path")?, false)?; + let model = read_model(&path, MAX_3D_MODEL_BYTES)?; + let file = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let name = a.opt_str("name")?.map(str::to_owned).unwrap_or_else(|| file.rsplit_once('.').map_or(file.clone(), |(s, _)| s.to_owned())); + let poster = match a.opt_str("poster")? { + None => None, + Some(p) => { + let path = self.resolve(p, false)?; + let bytes = crate::read_input(&path)?; + let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let page = usize::try_from(a.opt_int("poster_page")?.unwrap_or(1).max(1)).unwrap_or(1).saturating_sub(1); + Some(MarkFile { name, bytes: std::sync::Arc::new(bytes), page }) + } + }; + let edit = Edit::AddModel3d { page, rect: rect_to_user(&info, r), model: std::sync::Arc::new(model), poster, activation, name }; + self.apply(a, edit) + } +} + +/// Read a resolved model file of at most `max` bytes. Its size (from the file system) and its first +/// bytes are checked before the rest is read, so a large file that is not a model is never loaded. +fn read_model(path: &Path, max: usize) -> Result> { + use std::io::Read as _; + let io = |e: std::io::Error| failed(format!("{}: {e}", path.display())); + let too_large = |size: u64| failed(format!("the 3D model is {size} bytes; the largest accepted is {max}")); + let cap = u64::try_from(max).unwrap_or(u64::MAX); + let mut file = std::fs::File::open(path).map_err(io)?; + if let Ok(meta) = file.metadata() + && meta.is_file() + && meta.len() > cap + { + return Err(too_large(meta.len())); + } + let mut bytes = Vec::new(); + (&mut file).take(4).read_to_end(&mut bytes).map_err(io)?; + Model3dFormat::detect(&bytes).map_err(failed)?; + let rest = cap.saturating_add(1).saturating_sub(bytes.len() as u64); + file.take(rest).read_to_end(&mut bytes).map_err(io)?; + if bytes.len() > max { + return Err(failed(format!("the 3D model is more than {max} bytes, the largest accepted"))); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(dir: &Path, name: &str, bytes: &[u8]) -> std::path::PathBuf { + let path = dir.join(name); + std::fs::write(&path, bytes).unwrap(); + path + } + + #[test] + fn a_model_is_read_whole_when_it_fits() { + let dir = tempfile::tempdir().unwrap(); + let mut prc = b"PRC".to_vec(); + prc.extend_from_slice(&[7; 40]); + assert_eq!(read_model(&file(dir.path(), "m.prc", &prc), 64).unwrap(), prc); + assert_eq!(read_model(&file(dir.path(), "m.u3d", b"U3D\0x"), 6).unwrap(), b"U3D\0x"); + } + + #[test] + fn a_model_past_the_cap_is_refused_by_its_size_before_it_is_read() { + let dir = tempfile::tempdir().unwrap(); + let path = file(dir.path(), "big.prc", &[b'P', b'R', b'C', 0, 0, 0, 0, 0, 0, 0]); + let err = read_model(&path, 4).unwrap_err().to_string(); + assert!(err.contains("is 10 bytes") && err.contains("largest accepted is 4"), "the file's own size, from the file system: {err}"); + } + + #[test] + fn a_file_that_is_not_a_model_is_refused_by_its_header() { + let dir = tempfile::tempdir().unwrap(); + for junk in [&b"%PDF-1.7 a page"[..], b"PR", b"", b"U3DX and more"] { + let err = read_model(&file(dir.path(), "junk.bin", junk), 64).unwrap_err().to_string(); + assert!(err.contains("not a 3D model"), "{junk:?}: {err}"); + } + } +} diff --git a/crates/automation/src/model_render.rs b/crates/automation/src/model_render.rs new file mode 100644 index 0000000000..dbfabc6708 --- /dev/null +++ b/crates/automation/src/model_render.rs @@ -0,0 +1,104 @@ +//! `model_render`: a page's 3D model (ISO 32000-2 §13.6) drawn to a PNG, without a viewer. + +use pdfcraft_engine::model3d::Model3dKind; +use pdfcraft_prc::PrcFile; +use pdfcraft_prc_render::{Camera, RenderOptions, assemble, render}; + +use super::Automation; +use super::dispatch::Content; +use super::error::{Result, ToolError, failed}; +use super::json::Args; +use super::pages::encode_png; + +/// Default and largest side of the image, in pixels. +pub(crate) const DEFAULT_SIDE: u32 = 512; +pub(crate) const MAX_SIDE: u32 = 4096; + +/// The default view: from the front, right and above, as an isometric view looks. +const DEFAULT_DIRECTION: [f64; 3] = [-1.0, 1.0, -1.0]; + +impl Automation { + pub(crate) fn model_render(&mut self, a: &Args) -> Result> { + let page = self.page(a)?; + let width = side(a, "width")?; + let height = side(a, "height")?; + let direction = direction(a)?; + let perspective = a.opt_bool("perspective")?.unwrap_or(true); + let doc = self.doc(a)?; + let found = doc.model_3d_annotations(page).map_err(failed)?; + let index = match a.opt_int("model")? { + None => *found.first().ok_or_else(|| failed(format!("page {} has no 3D model", page.saturating_add(1))))?, + Some(n) => { + let pos = usize::try_from(n).ok().and_then(|n| n.checked_sub(1)).ok_or_else(|| Args::wrong("model", "a positive integer"))?; + *found + .get(pos) + .ok_or_else(|| ToolError::InvalidArgs(format!("page {} has {} 3D models, not {n}", page.saturating_add(1), found.len())))? + } + }; + let model = doc.model_3d(page, index).map_err(failed)?; + if model.kind == Model3dKind::U3d { + return Err(failed("this 3D model is in U3D, which can't be drawn yet; only PRC models can")); + } + let prc = PrcFile::parse(&model.data).map_err(|e| failed(format!("the 3D model could not be read: {e}")))?; + let meshes = assemble(&prc).map_err(failed)?; + let aspect = f64::from(width) / f64::from(height); + let camera = Camera::fit_meshes(&meshes, direction, up_for(direction), perspective, aspect).map_err(failed)?; + let image = render(&meshes, &camera, &RenderOptions { width, height, ..RenderOptions::default() }).map_err(failed)?; + let data = encode_png(image.width, image.height, &premultiplied(image.rgba))?; + Ok(vec![Content::Png { data, width: image.width, height: image.height }]) + } +} + +fn side(a: &Args, key: &str) -> Result { + match a.opt_int(key)? { + None => Ok(DEFAULT_SIDE), + Some(n) => u32::try_from(n).ok().filter(|n| (1..=MAX_SIDE).contains(n)).ok_or_else(|| Args::wrong(key, &format!("between 1 and {MAX_SIDE}"))), + } +} + +/// `direction`: three finite numbers, not all zero, from the eye towards the model. +fn direction(a: &Args) -> Result<[f64; 3]> { + let Some(v) = a.get("direction") else { return Ok(DEFAULT_DIRECTION) }; + let wrong = || Args::wrong("direction", "three numbers [x, y, z], not all zero"); + let items = v.as_array().filter(|x| x.len() == 3).ok_or_else(wrong)?; + let mut out = [0.0; 3]; + for (o, x) in out.iter_mut().zip(items) { + *o = x.as_f64().filter(|f| f.is_finite()).ok_or_else(wrong)?; + } + if out.iter().all(|c| *c == 0.0) { + return Err(wrong()); + } + Ok(out) +} + +/// Z is up, as CAD models are drawn, unless the view looks along it. +fn up_for([x, y, z]: [f64; 3]) -> [f64; 3] { + if x.hypot(y) <= z.abs() * 1e-6 { [0.0, 1.0, 0.0] } else { [0.0, 0.0, 1.0] } +} + +/// Straight RGBA to the premultiplied RGBA `encode_png` takes. +fn premultiplied(mut rgba: Vec) -> Vec { + for [r, g, b, alpha] in rgba.as_chunks_mut::<4>().0 { + let a = u16::from(*alpha); + for c in [r, g, b] { + *c = u8::try_from((u16::from(*c) * a + 127) / 255).unwrap_or(u8::MAX); + } + } + rgba +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn premultiplying_scales_colour_by_alpha() { + assert_eq!(premultiplied(vec![200, 100, 0, 255, 200, 100, 0, 128, 9, 9, 9, 0]), vec![200, 100, 0, 255, 100, 50, 0, 128, 0, 0, 0, 0]); + } + + #[test] + fn a_view_along_z_keeps_y_up() { + assert_eq!(up_for([0.0, 0.0, -1.0]), [0.0, 1.0, 0.0]); + assert_eq!(up_for(DEFAULT_DIRECTION), [0.0, 0.0, 1.0]); + } +} diff --git a/crates/automation/src/pages.rs b/crates/automation/src/pages.rs index 8c793be153..6db58ab475 100644 --- a/crates/automation/src/pages.rs +++ b/crates/automation/src/pages.rs @@ -20,6 +20,10 @@ pub(crate) const FIND_MAX_WARNINGS: usize = 100; pub(crate) const FIND_MAX_WARNING_BYTES: usize = 1024; +/// HTML that one `text_extract` with `format: "html"` returns across all its pages; a page past +/// it gets none. `html_truncated` covers this and any page whose own copy hit its caps. +const MAX_EXTRACT_HTML_BYTES: usize = 32 << 20; + impl Automation { // ---- pages and text ---------------------------------------------------------------------- @@ -201,6 +205,16 @@ impl Automation { } pub(crate) fn text_extract(&mut self, a: &Args) -> Result { + self.text_extract_within(a, MAX_EXTRACT_HTML_BYTES) + } + + /// [`Self::text_extract`] with at most `html_budget` bytes of HTML across all pages. + pub(crate) fn text_extract_within(&mut self, a: &Args, html_budget: usize) -> Result { + let html = match a.opt_str("format")?.unwrap_or("text") { + "text" => false, + "html" => true, + other => return Err(ToolError::InvalidArgs(format!("format must be \"text\" or \"html\", not {other:?}"))), + }; let doc = self.doc_to_copy(a)?; let id = doc.id; let pages = match a.opt_ints("pages")? { @@ -214,8 +228,42 @@ impl Automation { Some(r) => t.column_text(&t.glyphs_in(r)), None => t.plain_text(), }; - let out: Vec = pages.iter().zip(texts).map(|(p, t)| json!({ "page": p + 1, "text": text(&t) })).collect(); - Ok(json!({ "pages": out })) + let doc = self.doc(a)?; + let mut out = Vec::with_capacity(pages.len()); + // `spent`: the budget ran out, so later pages get no HTML. `cut`: some page's HTML is + // missing part of its text, by the budget or by the copy's own caps. + let (mut html_left, mut spent, mut cut) = (html_budget, false, false); + for (&p, t) in pages.iter().zip(texts) { + let mut page = json!({ "page": p + 1, "text": text(&t) }); + if html { + // Copy with formatting: the same glyphs as an HTML fragment, within one budget for + // the whole call; a page past it gets none and the call says so. + let mut copy = pdfcraft_engine::RichCopy::default(); + if !spent { + let glyphs: Vec = match rect { + Some(r) => t.glyphs_in(r), + None => (0..t.glyphs.len()).collect(), + }; + copy = doc.rich_copy(p, &t, &glyphs).map_err(failed)?; + match html_left.checked_sub(copy.html.len()) { + Some(left) => html_left = left, + None => { + spent = true; + copy = pdfcraft_engine::RichCopy::default(); + } + } + } + cut |= spent || copy.truncated; + page["html"] = json!(copy.html); + page["styles_incomplete"] = json!(copy.styles_incomplete || spent); + } + out.push(page); + } + let mut result = json!({ "pages": out }); + if html { + result["html_truncated"] = json!(cut); + } + Ok(result) } /// Matches in document order, extracting page text in growing batches and stopping as soon as diff --git a/crates/automation/src/signing.rs b/crates/automation/src/signing.rs index 743a36ff56..df0fd643fb 100644 --- a/crates/automation/src/signing.rs +++ b/crates/automation/src/signing.rs @@ -119,6 +119,7 @@ impl Automation { "page": s.page.map(|p| p + 1), "rect": rect, "visible": s.visible, + "verdict_shown": s.verdict_shown, "revision": s.revision, "sub_filter": s.sub_filter, "algorithm": s.algorithm, diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index fd4efa79da..ad0aa039dc 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -550,3 +550,35 @@ fn tool_file_reads_refuse_inputs_exceeding_512_mib() { } let _ = std::fs::remove_dir_all(&dir); } + +#[test] +fn text_extract_html_has_one_budget_across_pages() { + let mut a = Automation::new(); + let id = search_document(&mut a, 3); + let args = json!({ "doc": id.0, "format": "html" }); + let full = a.text_extract(&Args(&args)).unwrap(); + assert_eq!(full["html_truncated"], false); + let sizes: Vec = full["pages"].as_array().unwrap().iter().map(|p| p["html"].as_str().unwrap().len()).collect(); + assert!(sizes.iter().all(|n| *n > 0), "{full}"); + // Room for two pages: the third gets no HTML, and the call says so. + let capped = a.text_extract_within(&Args(&args), sizes[0] + sizes[1] + sizes[2] - 1).unwrap(); + let pages = capped["pages"].as_array().unwrap(); + assert_eq!(capped["html_truncated"], true); + assert_eq!(pages[1]["html"], full["pages"][1]["html"]); + assert_eq!(pages[2]["html"], ""); + assert_eq!(pages[2]["styles_incomplete"], true); + assert_eq!(pages[2]["text"], "Page 3", "the plain text is not capped by the HTML budget"); + // Plain text alone says nothing about HTML. + assert!(a.text_extract(&Args(&json!({ "doc": id.0 }))).unwrap().get("html_truncated").is_none()); + // A smaller page after the one that overflowed still gets none: "Page 1" fits what "Page 10" + // did not. + let id = search_document(&mut a, 10); + let args = json!({ "doc": id.0, "pages": [10, 1], "format": "html" }); + let full = a.text_extract(&Args(&args)).unwrap(); + let (ten, one) = (full["pages"][0]["html"].as_str().unwrap().len(), full["pages"][1]["html"].as_str().unwrap().len()); + assert!(one < ten, "{full}"); + let capped = a.text_extract_within(&Args(&args), ten - 1).unwrap(); + assert_eq!(capped["pages"][0]["html"], ""); + assert_eq!(capped["pages"][1]["html"], "", "no HTML after the budget ran out: {capped}"); + assert_eq!(capped["pages"][1]["styles_incomplete"], true); +} diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index f8a7a8c9b4..44febbbfad 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -5,6 +5,8 @@ use serde_json::{Value, json}; +use pdfcraft_engine::REDACT_PATTERNS; + use crate::ToolError; #[derive(Clone, Debug)] @@ -30,6 +32,12 @@ fn pages(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "minItems": 1, "description": format!("1-based page numbers {what}.") }) } +/// The Search & Redact pattern ids, from the engine's registry, so the advertised schema cannot +/// drift from the patterns the runtime accepts. +fn pattern_enum() -> Value { + json!({ "type": "string", "enum": REDACT_PATTERNS.iter().map(|p| p.id()).collect::>() }) +} + fn path(what: &str) -> Value { json!({ "type": "array", "items": { "type": "integer", "minimum": 1 }, "description": format!("{what}: 1-based positions from the top level, e.g. [2, 1] = the first child of the second bookmark.") }) } @@ -77,6 +85,12 @@ fn schema(props: Value, required: &[&str]) -> Value { json!({ "type": "object", "properties": props, "required": required, "additionalProperties": false }) } +/// The language codes the engines read, as the `language` enum — from the engine's own table, +/// so the schema and the validator can never disagree. +fn language_enum() -> Value { + Value::Array(pdfcraft_engine::ocr::LANGUAGES.iter().map(|(code, _)| json!(code)).collect()) +} + struct T { name: &'static str, title: &'static str, @@ -134,7 +148,7 @@ pub fn tools() -> Vec { "paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000}, "out_dir": {"type":"string"}, "steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true}, - "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, + "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, "dry_run": {"type":"boolean"}, "ppi": {"type":"number","minimum":9,"maximum":1600}, "quality": {"type":"integer","minimum":1,"maximum":100}, "dpi": {"type":"number","minimum":72,"maximum":600}, @@ -150,7 +164,7 @@ pub fn tools() -> Vec { .cmd("file.open") .with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])), t("doc_list", "List open documents", "List the open documents with their ids, page counts and unsaved state.").ro().with(schema(json!({}), &[])), - t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security and repair notes.") + t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security, repair notes and the objects that could not be read.") .ro() .cmd("file.properties") .with(schema(json!({ "doc": doc() }), &["doc"])), @@ -181,6 +195,13 @@ pub fn tools() -> Vec { "hide_text_block": { "type": "integer", "minimum": 1, "description": "Render an inline editor background without this paragraph (number from text_paragraphs, resolved as text_edit does: refused once an edit merged that paragraph into a neighbour). Does not modify the document." }, "hide_added_text": { "type": "integer", "minimum": 1, "description": "Render a retyping background without this added text item (added index from object_list). Mutually exclusive with hide_text_block. Does not modify the document." } }), &["doc", "page"])), + t("model_render", "Render a 3D model", "Draw a page's 3D model (a 3D annotation's PRC stream) to a PNG, framed to fill the image, with headlight shading on a white background. model is its 1-based position among the page's 3D models (default 1). direction is the view direction [x, y, z] from the eye towards the model (default [-1, 1, -1]); Z is up unless the view looks along Z. perspective defaults to true. U3D models can't be drawn yet and give an error. Does not modify the document.") + .ro() + .with(schema(json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "model": { "type": "integer", "minimum": 1 }, + "width": { "type": "integer", "minimum": 1, "maximum": 4096 }, "height": { "type": "integer", "minimum": 1, "maximum": 4096 }, + "direction": { "type": "array", "items": { "type": "number" }, "minItems": 3, "maxItems": 3 }, + "perspective": { "type": "boolean" } + }), &["doc", "page"])), t("object_list", "List editable objects", "List existing paragraphs, Image/Form artwork and added content once, with source kind/index references, displayed rectangles and document generation. Indexes change when content changes; use generation in object_move to reject stale references.") .ro().with(schema(json!({"doc":doc(),"page":{"type":"integer","minimum":1}}), &["doc","page"])), t("object_move", "Move objects together", "Atomically translate 1–1000 mixed references from object_list by [dx, dy] in displayed points (right/down). Preserves glyph codes, fonts, image data and relative spacing; success is one undo step. Text used as a clipping path and vertical text are refused. A bad/stale reference refuses the entire move.") @@ -195,13 +216,14 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc(), "pages": pages("to count (default: all)"), "exclude_tables": { "type": "boolean" }, "exclude_code": { "type": "boolean" }, "exclude_images": { "type": "boolean" } }), &["doc"])), - t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab.") + t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab. With format \"html\", each page also carries html, the same text as Copy with formatting puts on the clipboard (paragraphs, bold, italic, size, colour and font family), and styles_incomplete when some styles had to be left out. With rect, the html is that text in line order, not split into the column's rows and cells. The HTML of one call is capped at 32 MiB in total; pages past the cap carry empty html with styles_incomplete. html_truncated is set when that cap is reached or when a page's own copy was cut at its span or size cap.") .ro() .cmd("edit.column_select") .with(schema( json!({ "doc": doc(), "pages": pages("to extract (default: all)"), - "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." } + "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." }, + "format": { "type": "string", "enum": ["text", "html"], "description": "text (default) or html: also return each page's text as formatted HTML." } }), &["doc"], )), @@ -233,6 +255,18 @@ pub fn tools() -> Vec { }), &["doc", "page"], )), + t("page_enhance_scan", "Enhance scanned file", "Clean up scanned pages in place (Scan & OCR ▸ Enhance ▸ Scanned document). A page is a scan when its pictures cover at least half of it; the pictures covering at least a tenth of it are cleaned and written back into the same image objects at the same size and colour space (a JPEG stays a JPEG at quality 90, other grey/RGB images are stored losslessly, a 1-bit image stays 1-bit), so the page's invisible OCR text, comments, links and other content stay as they were, and an image drawn on several pages is cleaned once. level (0–100, default 50) is background removal: the paper's illumination is evened out so a grey, yellowed or unevenly lit background turns white, then the contrast is raised so faint strokes darken; colour ink stays colour. edge_shadows (default: on unless level is 0) lightens shadows along the scan's edges (book gutters, a copier's dark border). despeckle (default: from level 60) turns isolated dark specks white; it is the only change a 1-bit scan takes. Returns pages_cleaned (1-based), images (rewritten), not_scanned (pages that are no scan) and skipped ([{pages, name, reason}]: CMYK, indexed, JPEG 2000, JBIG2 or CCITT images, a scan drawn inside grouped Form artwork, broken data, or nothing to change). One undoable step; none when nothing changed.") + .cmd("ocr.enhance") + .with(schema( + json!({ + "doc": doc(), + "pages": pages("to clean up (default: all)"), + "level": { "type": "integer", "minimum": 0, "maximum": 100 }, + "edge_shadows": { "type": "boolean" }, + "despeckle": { "type": "boolean" }, + }), + &["doc"], + )), t("page_delete", "Delete pages", "Delete pages. Undoable until saved.") .destructive() .cmd("page.delete") @@ -323,6 +357,50 @@ pub fn tools() -> Vec { }), &["doc", "title", "page"], )), + t( + "bookmark_export", + "Export bookmarks", + "The bookmark tree, or the subtree at path, as a list of nodes bookmark_import takes back unchanged. A node has title, page (1-based, null when it goes nowhere), open, and when set view ({type, values}: a destination type from ISO 32000-2 §12.3.2.2 Table 149 and its values in PDF user space, as the file stores them), color ([r, g, b], 0-1), bold, italic and children. truncated is true when bookmarks nested deeper than 33 levels were left out. With out, the tree is also written there as a JSON file {\"bookmarks\": [...]}, with \"truncated\": true when that is so; out is overwritten.", + ) + .destructive() + .with(schema( + json!({ "doc": doc(), "path": path("The bookmark whose subtree to export (omit for the whole outline)"), "out": path_arg() }), + &["doc"], + )), + t( + "bookmark_import", + "Import bookmarks", + "Add a tree of bookmarks, as bookmark_export gives it (bookmarks) or as a JSON file it wrote (file), under a parent bookmark (or at the top level) at a position, or with replace in place of the whole outline. A bookmark whose page this document lacks is kept without a destination and listed in unmapped (1-based paths into the given tree). A view takes exactly its type's values (XYZ 3, FitR 4, FitH/FitV/FitBH/FitBV 1, Fit/FitB none). Give exactly one of bookmarks and file. A title may be empty but not missing; a key a node does not have is refused. Every node is checked before anything is written: at most 100000 bookmarks, nested at most 33 levels counting the parent's. truncated is true when the file says its export left bookmarks out. Undoable.", + ) + .destructive() + .with(schema( + json!({ + "doc": doc(), + "bookmarks": { "type": "array", "items": { "type": "object" }, "minItems": 1, "description": "Nodes as bookmark_export returns them." }, + "file": path_arg(), + "parent": path("Parent bookmark (omit for the top level)"), + "position": { "type": "integer", "minimum": 1, "description": "1-based position among the parent's children (default: last)." }, + "replace": { "type": "boolean", "description": "Replace the document's bookmarks instead of adding to them (default false; leave parent out)." } + }), + &["doc"], + )), + t( + "bookmark_copy", + "Copy bookmarks from another document", + "Copy the bookmarks of document from (or the subtree at path) into doc, by page number: under a parent bookmark (or at the top level) at a position, or with replace in place of doc's outline. A bookmark whose page doc lacks is kept without a destination and listed in unmapped (1-based paths into the copied tree). truncated is true when bookmarks nested deeper than 33 levels were left out of the copy. Undoable.", + ) + .destructive() + .with(schema( + json!({ + "doc": doc(), + "from": { "type": "integer", "minimum": 1, "description": "Id of the document to copy from." }, + "path": path("The bookmark whose subtree to copy (omit for the whole outline)"), + "parent": path("Parent bookmark in doc (omit for the top level)"), + "position": { "type": "integer", "minimum": 1, "description": "1-based position among the parent's children (default: last)." }, + "replace": { "type": "boolean", "description": "Replace doc's bookmarks instead of adding to them (default false; leave parent out)." } + }), + &["doc", "from"], + )), t("bookmark_rename", "Rename a bookmark", "Change a bookmark's title. Undoable.") .with(schema(json!({ "doc": doc(), "path": path("The bookmark"), "title": { "type": "string", "minLength": 1 } }), &["doc", "path", "title"])), t("bookmark_delete", "Delete a bookmark", "Delete a bookmark and the bookmarks under it. Undoable.") @@ -469,6 +547,20 @@ pub fn tools() -> Vec { }), &["doc", "page", "path"], )), + t("model_add", "Add a 3D model", "Add a 3D model (Acrobat: Rich Media ▸ Add 3D): the U3D or PRC file at path, recognised by its first bytes, as a 3D annotation in rect [x0, y0, x1, y1] (points from the top-left of the displayed page; each side 1–14400 pt; at most 256 MiB). Its poster, what prints and what readers show until the model is activated, is the picture at poster (a PDF page, poster_page default 1, or an image) fitted inside rect keeping its shape and centred; without poster, a frame labelled \"3D model\". activation: click (default), page_open or page_visible. name is its description (default: the file name). No views are added: readers open the model on its own default view. Undoable.") + .with(schema( + json!({ + "doc": doc(), + "page": { "type": "integer", "minimum": 1 }, + "path": { "type": "string" }, + "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, + "poster": { "type": "string" }, + "poster_page": { "type": "integer", "minimum": 1 }, + "activation": { "type": "string", "enum": ["click", "page_open", "page_visible"] }, + "name": { "type": "string" }, + }), + &["doc", "page", "path", "rect"], + )), t("form_set_image", "Set an image field's picture", "Show an image file (PNG, JPEG, TIFF, GIF or BMP) in an image field or button, scaled to fit and centred (what clicking an image field does). On an XFA form the picture is kept in its data too (base64, as Designer saves it) and the form shows it from there. Undoable.") .with(schema(json!({ "doc": doc(), "field": { "type": "string" }, "path": { "type": "string" } }), &["doc", "field", "path"])), t( @@ -528,7 +620,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -539,7 +631,7 @@ pub fn tools() -> Vec { "words": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "maxItems": 1000 }, "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, - "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "pattern": pattern_enum(), "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), "overlay": { "type": "string" }, @@ -628,13 +720,13 @@ pub fn tools() -> Vec { }), &["doc"], )), - t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style.") + t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style; text painted in gray or CMYK also has gray (its level) or cmyk ([c, m, y, k]).") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t( "page_add_text", "Add text to a page", - "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name), align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", + "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name, painted in RGB) or, for print, gray (0 black to 1 white, painted in DeviceGray) or cmyk ([c, m, y, k], each 0 to 1, painted in DeviceCMYK; [0, 0, 0, 1] is 100 K black), one of the three, align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", ) .cmd("edit.text") .with(schema( @@ -645,6 +737,8 @@ pub fn tools() -> Vec { "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, @@ -661,13 +755,15 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "path": { "type": "string" }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 } }), &["doc", "page", "path"], )), - t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") + t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color or gray or cmyk as page_add_text takes them, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") .with(schema( json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "index": { "type": "integer", "minimum": 1 }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "text": { "type": "string" }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, @@ -782,9 +878,13 @@ pub fn tools() -> Vec { "icon": { "type": "string", "description": "note: Comment, Note, Help, Insert, Key, NewParagraph, Paragraph (the standard names), Check, Circle, Cross, CrossHairs, Star, RightArrow, UpArrow; attachment: PushPin, Paperclip, Graph, Tag." }, "color": color(), "fill": color(), - "opacity": { "type": "number", "minimum": 0, "maximum": 1 }, + "opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The whole comment's opacity (its line or border's, where its fill or text has its own)." }, + "fill_opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The fill's own opacity: rectangle, oval, polygon, cloud, text box, typewriter text or callout." }, + "text_opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The text's own opacity: text box, typewriter text or callout." }, "width": { "type": "number", "minimum": 0, "description": "Line width in points." }, + "border_shape": { "type": "string", "enum": ["rectangle", "oval", "circle"], "description": "A text box's or typewriter text's border shape: an oval or a circle goes around the text box." }, "endings": { "type": "array", "items": { "type": "string", "enum": ["None", "Square", "Circle", "Diamond", "OpenArrow", "ClosedArrow", "Butt", "ROpenArrow", "RClosedArrow", "Slash"] }, "minItems": 1, "maxItems": 2, "description": "Line endings. Two names (start, end) for a line, arrow or polyline; one name for a callout." }, + "ortho": { "type": "boolean", "description": "line/arrow/polygon/cloud/polyline: hold each end or corner level or plumb with the one before, whichever is nearer (Shift while drawing)." }, "font_size": { "type": "number", "exclusiveMinimum": 0 }, }), &["doc", "page", "type"], @@ -797,7 +897,7 @@ pub fn tools() -> Vec { comment_ref(json!({ "status": { "type": "string", "enum": ["none", "accepted", "rejected", "cancelled", "completed"] }, "author": { "type": "string" } })), &["doc", "status"], )), - t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") + t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, whether PdfCraft draws that state into the signature's own appearance as displayed (verdict_shown; never written to the file), the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t("sign_id_create", "Create a digital ID", "Create a self-signed digital ID and save it as a password-protected .p12 file (Acrobat: Configure a new digital ID ▸ Create a new digital ID ▸ Save to file). Default key: 2048-bit RSA, valid 5 years.").with(schema( @@ -915,13 +1015,17 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "sort": { "type": "string", "enum": ["page", "author", "date", "type"] }, "out": save_out, "open": open }), &["doc"], )), - t("comment_edit", "Edit a comment", "Change a comment's text, colour, opacity, line width, font size (text box/typewriter/callout), fill (rectangle/oval/polygon/text box/callout; \"none\" removes it), rectangle (rectangle/oval/text box/stamp), position (`move` [dx, dy] in points) or shape (`points`: a line's two ends, or the corners of connected lines, a polygon or a cloud). Stamps keep their original appearance when resized. One undo step.").with(schema( + t("comment_edit", "Edit a comment", "Change a comment's text, colour, opacity (the whole comment's, or its fill's and text's own), line width, font size (text box/typewriter/callout), fill (rectangle/oval/polygon/text box/callout; \"none\" removes it), border colour (`border_color`, text box/typewriter/callout; \"text\" gives it the text's colour again), rectangle (rectangle/oval/text box/stamp), position (`move` [dx, dy] in points) or shape (`points`: a line's two ends, or the corners of connected lines, a polygon or a cloud). Stamps keep their original appearance when resized. One undo step.").with(schema( comment_ref(json!({ "contents": { "type": "string" }, "color": color(), - "opacity": { "type": "number", "minimum": 0, "maximum": 1 }, + "opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The whole comment's opacity: its fill and text drop opacities of their own." }, + "fill_opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The fill's own opacity (rectangle, oval, polygon, cloud, text box, typewriter text, callout); the line or border keeps the comment's." }, + "text_opacity": { "type": "number", "minimum": 0, "maximum": 1, "description": "The text's own opacity (text box, typewriter text, callout)." }, "width": { "type": "number", "minimum": 0 }, "fill": { "type": "string", "description": "Interior colour of a rectangle, oval, polygon, text box or callout: #RRGGBB, a colour name, or \"none\" for no fill." }, + "border_color": { "type": "string", "description": "A text box's, typewriter text's or callout's border colour: #RRGGBB, a colour name, or \"text\" for the text's colour." }, + "border_shape": { "type": "string", "enum": ["rectangle", "oval", "circle"], "description": "A text box's or typewriter text's border shape: an oval or a circle goes around the text, which keeps its place." }, "font_size": { "type": "number", "minimum": 1, "maximum": 400, "description": "Font size in points of a text box, typewriter text or callout; the box grows to hold the text." }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "move": point(), @@ -932,6 +1036,25 @@ pub fn tools() -> Vec { t("comment_image_preview", "Preview image signature layers", "Return a PNG of the page without the selected image signature/initials (layer=background, default), or its embedded image with alpha (layer=image). Includes the displayed rectangle, document rotation, image_rotation (how far clockwise the image is shown turned) and annotation opacity. Cache these layers for live placement/resizing; commit once with comment_edit. Does not change the document or undo history.") .ro().with(schema(comment_ref(json!({ "layer": { "type": "string", "enum": ["background", "image"], "default": "background" }, "dpi": { "type": "number", "minimum": 1, "maximum": 600, "default": 96 } })), &["doc"])), t("comment_delete", "Delete a comment", "Delete a comment with its pop-up and replies, or one reply (its id, or page and index, from comment_list) with its pop-up and the replies to it; the comment and its other replies stay. Undoable.").destructive().with(schema(comment_ref(json!({})), &["doc"])), + t( + "comments_copy", + "Copy comments", + "The XFDF of these comments, each with its replies: what Copy puts on the clipboard. Paste it into this or another document with comments_paste. `comments`: [[page, index], …], 1-based as comment_list reports them.", + ) + .ro() + .with(schema( + json!({ + "doc": doc(), + "comments": { "type": "array", "items": { "type": "array", "items": { "type": "integer", "minimum": 1 }, "minItems": 2, "maxItems": 2 }, "minItems": 1 } + }), + &["doc", "comments"], + )), + t( + "comments_paste", + "Paste comments", + "Add the comments of an XFDF text (from comments_copy) to `page` as new comments: copies under names of their own, so nothing in the document is replaced and pasting twice gives two sets; replies keep their copied parent. `at` [x, y] (points, origin at the top-left of the displayed page, y down, as comment_add takes them) puts the centre of the copies' bounding box there; without it they keep their positions. Undoable.", + ) + .with(schema(json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "xfdf": { "type": "string" }, "at": point() }), &["doc", "page", "xfdf"])), t( "doc_protect", "Protect with passwords", @@ -1262,31 +1385,79 @@ pub fn tools() -> Vec { )), t("js_enabled", "JavaScript on or off", "Preferences ▸ JavaScript ▸ Enable Acrobat JavaScript: set it with `enabled`, or read it. With JavaScript off, field scripts other than Acrobat's AF calls don't run.") .with(schema(json!({ "enabled": { "type": "boolean" } }), &[])), - t("ocr_recognize", "Recognize text (OCR)", "Scan & OCR ▸ Recognize text: render pages, read the words in them and add them as invisible text over the page image, so scanned pages become searchable and selectable (a searchable image; the image is not changed). Pages that already have text are skipped unless skip_text_pages is false; then pictures on those pages are read too, and recognised words that the page's own text already has (an earlier OCR layer included) are left out and counted as already_on_page. Returns each page's recognised text (left out when the document's security doesn't allow copying), word count or why it was skipped. Needs the OCR models (ocr_status). Undoable as one step.") + t("ocr_recognize", "Recognize text (OCR)", "Scan & OCR ▸ Recognize text: render pages, read the words in them and add them as invisible text over the page image, so scanned pages become searchable and selectable (a searchable image; the image is not changed). Pages that already have text are skipped unless skip_text_pages or force_ocr say otherwise; words overlapping existing text are omitted and counted as already_on_page. With output_mode \"editable\" nothing is applied: a NEW document of visible text at the recognised boxes is written to out (which must not be the source; empty pages stay blank). engine and strategy choose the readers (an engine that cannot run is an error named up front); the four preprocessing flags clean the raster first. region reads just that part of one page ([x0, y0, x1, y1] in points from the top-left of the displayed page). Returns each page's recognised text, word count or why it was skipped. Needs the engines (ocr_status). Searchable output is undoable as one step.") .cmd("ocr.recognize") .with(schema( json!({ "doc": doc(), "pages": pages("to recognise (default: all)"), "dpi": { "type": "number", "minimum": 72, "maximum": 600, "description": "Resolution pages are read at (default 300; a scanned page is read at most at its own resolution)." }, - "language": { "type": "string", "enum": ["en"], "description": "Document language (default en)." }, - "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true). false also reads pictures on such pages, without adding their text again." }, + "language": { "type": "string", "enum": language_enum(), "description": "Document language (default en)." }, + "engine": { "type": "string", "enum": ["auto", "ocrs", "tesseract"], "description": "Which engine reads the pages (default ocrs). auto: ocrs first and tesseract as the second engine, combined by strategy (only the engines that can run take part)." }, + "strategy": { "type": "string", "enum": ["primary", "confidence", "rover"], "description": "How two engines' readings combine: primary = the first engine alone; confidence = the second engine joins when the first reading's mean confidence is under 70 or unknown; rover = both always read, merged. Default: confidence with engine auto, primary otherwise." }, + "output_mode": { "type": "string", "enum": ["searchable", "editable"], "description": "searchable (default): invisible text over the pages of the open document. editable: a NEW _ocr-style file of visible text written to out; the source is not touched." }, + "out": { "type": "string", "description": "Where the editable output is written (required with output_mode editable; a new file: never the source path, never an existing file)." }, + "region": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "[x0, y0, x1, y1] in points from the top-left of the displayed page: read only that part. Needs exactly one page." }, + "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, + "skip_text_files": { "type": "boolean", "description": "Batch: leave whole files that already contain text alone (they come back skipped, not read)." }, + "force_ocr": { "type": "boolean", "description": "Read pages and files even when the skip options would leave them alone (default false)." }, + "auto_rotate": { "type": "boolean", "description": "Turn pages scanned sideways or upside down upright before reading (default false)." }, + "deskew": { "type": "boolean", "description": "Straighten skewed text lines before reading (default false)." }, + "denoise": { "type": "boolean", "description": "A 3×3 median filter against scan speckle (default false)." }, + "binarize": { "type": "boolean", "description": "Sauvola binarization: ink to black, paper to white (default false)." }, }), &["doc"], )), - t("ocr_recognize_files", "Recognize text in multiple files", "Scan & OCR ▸ Recognize text ▸ In multiple files: read every page of each PDF in paths and write the searchable result into folder under the same name (pages that already have text are left alone unless skip_text_pages is false, as in ocr_recognize). Two inputs that would write the same output file are refused before anything is read or written. Returns, per file, the output path, word count and skipped pages, or the error.") + t("ocr_words", "Words of a recognition (for review)", "The words a recognition finds, for inspection without the GUI: per page, each word's text, its box ([x, y, w, h] in points from the top-left of the displayed page), its confidence and band (high >= 90, medium 70–89, low < 70), which engine read it, and whether it is a suspect (low band). suspects counts the words a reviewer should look at first. Changes nothing on the document. retain_review returns a session-local handle for ocr_review_apply (one per document, at most eight; a new retained recognition replaces the old one). Needs the engines (ocr_status).") + .ro() + .with(schema( + json!({ + "doc": doc(), + "retain_review": { "type": "boolean" }, + "region": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, + "pages": pages("to read (default: all)"), + "dpi": { "type": "number", "minimum": 72, "maximum": 600 }, + "language": { "type": "string", "enum": language_enum() }, + "engine": { "type": "string", "enum": ["auto", "ocrs", "tesseract"] }, + "strategy": { "type": "string", "enum": ["primary", "confidence", "rover"] }, + "auto_rotate": { "type": "boolean" }, + "deskew": { "type": "boolean" }, + "denoise": { "type": "boolean" }, + "binarize": { "type": "boolean" }, + "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, + "force_ocr": { "type": "boolean", "description": "Read pages even when the skip options would leave them alone (default false)." }, + }), + &["doc"], + )), + t("ocr_review_apply", "Apply or discard reviewed OCR words", "Accept corrected recognition from ocr_words retain_review. Corrections use 1-based page/word positions and text only; empty text removes a word, omitted words keep their original text and boxes. The stored document guard refuses stale, foreign or consumed reviews. Searchable output is one undoable step; editable requires out and writes a new document (out must not exist yet; the review is kept when it does). discard releases the review without applying it. Input is capped at 65536 words and 4 MiB, never truncated.") + .cmd("ocr.correct") + .with(schema(json!({ "doc": doc(), "review": { "type": "integer", "minimum": 1 }, "discard": { "type": "boolean" }, "output_mode": { "type": "string", "enum": ["searchable", "editable"] }, "out": { "type": "string" }, + "corrections": { "type": "array", "maxItems": 65536, "items": { "type": "object", "additionalProperties": false, "properties": { "page": { "type": "integer", "minimum": 1 }, "word": { "type": "integer", "minimum": 1 }, "text": { "type": "string" } }, "required": ["page", "word", "text"] } } }), &["doc", "review"])), + t("ocr_dictionary", "Read or update an OCR review dictionary", "Read or add/remove words in an OCR Verify dictionary file (one case-insensitive word per UTF-8 line). Choose its per-language ocr-dictionary-.txt path. Paths honor root; updates are atomic. Refuses invalid words, over 65536 entries or over 4 MiB. No document changes.") + .destructive() + .cmd("ocr.correct") + .with(schema(json!({ "path": { "type": "string" }, "add": { "type": "array", "maxItems": 65536, "items": { "type": "string" } }, "remove": { "type": "array", "maxItems": 65536, "items": { "type": "string" } } }), &["path"])), + t("ocr_recognize_files", "Recognize text in multiple files", "Scan & OCR ▸ Recognize text ▸ In multiple files: read every page of each PDF in paths and write the searchable result into folder under the same name. skip_text_files leaves whole files that already contain text alone (they are their own skipped bucket, nothing is written); skip_text_pages leaves single text pages alone; force_ocr reads regardless of both. Every output name is checked before the first page is read: two paths sharing a file name (from different folders) are refused, and a file of that name already in folder aborts the batch, with the names in the error. Each written file carries a low_confidence note when words at or under 60 confidence sit on it (1-based sorted pages). Returns, per file, the output path, word count, skipped pages or the error, plus recognized/words/skipped_files totals. Needs the engines (ocr_status).") .cmd("ocr.recognize_batch") .with(schema( json!({ "paths": { "type": "array", "items": { "type": "string" }, "minItems": 1 }, "folder": { "type": "string" }, "dpi": { "type": "number", "minimum": 72, "maximum": 600 }, - "language": { "type": "string", "enum": ["en"] }, - "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true); see ocr_recognize." }, + "language": { "type": "string", "enum": language_enum() }, + "engine": { "type": "string", "enum": ["auto", "ocrs", "tesseract"] }, + "strategy": { "type": "string", "enum": ["primary", "confidence", "rover"] }, + "skip_text_pages": { "type": "boolean", "description": "Leave pages that already contain text alone (default true)." }, + "skip_text_files": { "type": "boolean", "description": "Leave whole files that already contain text alone (default false)." }, + "force_ocr": { "type": "boolean", "description": "Read even where the skip options would leave things alone (default false)." }, + "auto_rotate": { "type": "boolean" }, + "deskew": { "type": "boolean" }, + "denoise": { "type": "boolean" }, + "binarize": { "type": "boolean" }, }), &["paths", "folder"], )), - t("ocr_status", "OCR status", "Whether text recognition is available (its models are installed: release packages include them; a source build fetches them with `cargo xtask models`; PDFCRAFT_MODELS overrides), where it looks for them, and the languages it reads.") + t("ocr_status", "OCR status", "Whether text recognition is available, where it looks for models (release packages include them; a source build fetches them with `cargo xtask models`; PDFCRAFT_MODELS overrides), its languages, and the engines: ocrs (built in, models) and tesseract (an installed program, with its path).") .ro() .cmd("ocr.recognize") .with(schema(json!({}), &[])), @@ -1307,7 +1478,7 @@ pub fn tools() -> Vec { t( "fill_sign_add", "Fill & Sign: type text or place a mark", - "Fill in a form that has no fields, as Acrobat's Fill & Sign does: type text (`text`, default 10 pt; `font_size` 4–300 pt; optional `color` as #RRGGBB for text/date, default black), place a check, cross, dot or line, today's date (in Preferences ▸ Date format, see fill_sign_date_format, or `format`; `font_size` also applies), or a signature or initials (either `text` drawn in a script font, or `path` to a local PNG/JPEG image up to 4 MiB and 32 megapixels, scaled down to 4 megapixels, preserving transparency; with `scan: true`, `path` is instead a photo of a signature on paper, within the same limits: the signature is found, and the paper, shadows and table around it are removed, with `sensitivity` 0–100, default 50, picking up fainter ink when higher). `at` [x, y] is in points from the top-left of the page: text's top-left, mark's centre, signature's left edge centred vertically. Image signatures fit within 150 pt wide and 32 pt tall (24 pt for initials). Creates movable, undoable annotations. Typed text the standard fonts can't draw is saved and drawn as `?` (`warnings` in the result says so); a typed signature neither the script font nor its fallback can draw is refused (pass `path` instead).", + "Fill in a form that has no fields, as Acrobat's Fill & Sign does: type text (`text`, default 10 pt; `font_size` 4–300 pt; optional `color` as #RRGGBB for text/date, default black; `border` 0–12 pt around it, default none, in its colour or `border_color`), place a check, cross, dot or line, today's date (in Preferences ▸ Date format, see fill_sign_date_format, or `format`; `font_size` also applies), or a signature or initials (either `text` drawn in a script font, or `path` to a local PNG/JPEG image up to 4 MiB and 32 megapixels, scaled down to 4 megapixels, preserving transparency; with `scan: true`, `path` is instead a photo of a signature on paper, within the same limits: the signature is found, and the paper, shadows and table around it are removed, with `sensitivity` 0–100, default 50, picking up fainter ink when higher). `at` [x, y] is in points from the top-left of the page: text's top-left, mark's centre, signature's left edge centred vertically. Image signatures fit within 150 pt wide and 32 pt tall (24 pt for initials). Creates movable, undoable annotations. Typed text the standard fonts can't draw is saved and drawn as `?` (`warnings` in the result says so); a typed signature neither the script font nor its fallback can draw is refused (pass `path` instead).", ) .cmd("sign.fill.text") .with(schema( @@ -1319,6 +1490,9 @@ pub fn tools() -> Vec { "text": { "type": "string", "minLength": 1 }, "font_size": { "type": "number", "minimum": 4, "maximum": 300, "description": "For type text or date (default 10 pt)." }, "color": color(), + "border": { "type": "number", "minimum": 0, "maximum": 12, "description": "For type text or date: a border around it in points, in the text's colour (default 0, none)." }, + "border_color": { "type": "string", "description": "With border: its own colour, #RRGGBB or a colour name." }, + "border_shape": { "type": "string", "enum": ["rectangle", "oval", "circle"], "description": "With border, for type text or date: the border's shape (default rectangle); an oval or a circle goes around the text." }, "path": { "type": "string", "description": "PNG or JPEG for signature/initials; pass either path or text." }, "scan": { "type": "boolean", "description": "With path: scan a photo of a signature on paper (default false)." }, "sensitivity": { "type": "integer", "minimum": 0, "maximum": 100, "description": "With scan: how faint ink may be, 0–100 (default 50)." }, @@ -1460,7 +1634,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/it/action_failures.rs b/crates/automation/tests/it/action_failures.rs new file mode 100644 index 0000000000..b487103a21 --- /dev/null +++ b/crates/automation/tests/it/action_failures.rs @@ -0,0 +1,66 @@ +//! A step that fails fails its file: the file is reported with an error and nothing is written +//! for it. A step that finds nothing to do is still not a failure. + +use std::sync::Arc; + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_engine::Session; +use pdfcraft_engine::actions::{Action, Step, run_on}; +use serde_json::json; + +fn pdf() -> Vec { + Session::new().create_from_text("Original", "Original text").unwrap().as_ref().clone() +} + +fn one_step(step: Step) -> Action { + Action { name: "Custom".into(), description: String::new(), steps: vec![step], builtin: false } +} + +/// The standard fonts can't draw the character, so the watermark step fails and no file is produced. +#[test] +fn failing_watermark_step_fails_the_file_and_publishes_nothing() { + let r = run_on(&one_step(Step::AddWatermark("中".into())), "one.pdf", Arc::new(pdf()), |_, _| {}); + assert!(r.is_err(), "a failed step must fail its file: {r:?}"); +} + +/// A comment step on a document with no comments has nothing to do, which is not a failure. +#[test] +fn nothing_to_do_step_still_succeeds() { + let r = run_on(&one_step(Step::FlattenComments), "one.pdf", Arc::new(pdf()), |_, _| {}); + assert!(r.is_ok(), "a step with nothing to do is not a failure: {r:?}"); +} + +/// Through the tool: the failing file gets an error entry and no output, and the folder stays empty. +#[test] +fn action_run_marks_the_failing_file_as_error_without_output() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("in"); + std::fs::create_dir_all(&input).unwrap(); + let in_file = input.join("one.pdf"); + std::fs::write(&in_file, pdf()).unwrap(); + let out = dir.path().join("out"); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let reply = auto.call( + "action_run", + &json!({ + "paths": [in_file.to_string_lossy()], + "folder": out.to_string_lossy(), + "steps": [{ "step": "add_watermark", "arg": "中" }], + }), + ); + let report = match reply.unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected a report: {other:?}"), + }; + let entry = &report["files"][0]; + assert!(entry["error"].is_string(), "{report}"); + assert!(entry.get("output").is_none(), "{report}"); + assert_eq!(std::fs::read_dir(&out).unwrap().count(), 0, "nothing is published for a failed file"); +} + +/// A removal step with nothing to remove keeps its existing note and succeeds. +#[test] +fn removal_step_with_nothing_to_remove_still_succeeds() { + let r = run_on(&one_step(Step::RemoveHiddenInformation), "one.pdf", Arc::new(pdf()), |_, _| {}); + assert!(r.is_ok(), "nothing to remove is not a failure: {r:?}"); +} diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 30a8f42cb3..8afb33b58c 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -843,10 +843,11 @@ fn arabic_ligature_actual_text_is_one_editable_word_through_save_reopen() { } /// Each of these writes to its required `path` and replaces an existing file there, so the MCP -/// annotations must not tell clients the call is read-only or harmless (#130). +/// annotations must not tell clients the call is read-only or harmless (#130). The OCR +/// dictionary's updates rewrite the file in place (lower-cased and sorted), so it counts too. #[test] fn file_writing_tools_are_not_read_only() { - for name in ["doc_export_data", "accessibility_report", "image_save"] { + for name in ["doc_export_data", "accessibility_report", "image_save", "ocr_dictionary"] { let t = tools().into_iter().find(|t| t.name == name).unwrap(); assert!(!t.read_only, "{name} writes a file but advertises read-only"); assert!(t.destructive, "{name} overwrites its path but advertises non-destructive"); @@ -1715,6 +1716,25 @@ fn mcp_default_lists_every_tool_and_has_no_meta_tools() { assert_eq!(rpc(&mut s, 3, "tools/call", json!({ "name": "tool_search" }))["error"]["code"], -32602); } +#[cfg(feature = "mcp")] +#[test] +/// `tools/list` advertises exactly the Search & Redact patterns the registry supports, in `pattern` +/// and by name in the description, so a schema-obeying client can select every one of them. +fn the_advertised_redact_mark_schema_offers_every_pattern() { + use pdfcraft_engine::REDACT_PATTERNS; + let registry: Vec<&str> = REDACT_PATTERNS.iter().map(|p| p.id()).collect(); + let mut s = McpServer::new(Automation::new()); + let list = rpc(&mut s, 1, "tools/list", json!({})); + let mark = list["result"]["tools"].as_array().unwrap().iter().find(|t| t["name"] == "redact_mark").expect("redact_mark is listed"); + let advertised: Vec<&str> = + mark["inputSchema"]["properties"]["pattern"]["enum"].as_array().unwrap().iter().map(|v| v.as_str().unwrap()).collect(); + assert_eq!(advertised, registry, "the advertised pattern enum must be the registry's ids"); + let description = mark["description"].as_str().unwrap(); + for id in ®istry { + assert!(description.contains(id), "the description must name {id}: {description}"); + } +} + #[cfg(feature = "mcp")] #[test] fn mcp_compact_lists_the_core_tools_and_two_meta_tools() { @@ -2135,6 +2155,104 @@ fn numbering_pages_through_tools() { assert!(matches!(a.call("page_number", &json!({ "doc": doc, "from": 1, "to": 1, "style": "klingon" })), Err(ToolError::InvalidArgs(_)))); } +/// Typed text and text boxes take an oval or a circle for their border through the tools; +/// comment_list reports it (a rectangle is not reported). Only typed text, dates and text boxes +/// have one, and only the three shapes. +#[test] +fn text_borders_take_an_oval_or_a_circle_through_the_tools() { + let dir = workdir("border-shape"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok( + &mut a, + "fill_sign_add", + json!({ "doc": doc, "page": 1, "type": "text", "at": [20, 40], "text": "12", "border": 1, "border_shape": "circle" }), + ); + ok( + &mut a, + "comment_add", + json!({ "doc": doc, "page": 1, "type": "textbox", "rect": [10, 200, 190, 240], "contents": "Hello", "width": 1, "border_shape": "Oval" }), + ); + let list = |a: &mut Automation| ok(a, "comment_list", json!({ "doc": doc, "page": 1 }))["comments"].as_array().unwrap().clone(); + let shapes: Vec = list(&mut a).iter().map(|c| c["border_shape"].clone()).collect(); + assert_eq!(shapes, [json!("circle"), json!("oval")]); + let typed = list(&mut a)[0].clone(); + // Snug around "12": the box no wider than the text needs, the circle as wide as that box. + let r: Vec = typed["rect"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect(); + let w = pdfcraft_engine::annot_text::text_width("12", 10.0) + 10.0; + assert!((r[2] - r[0] - w).abs() < 0.05 && (r[3] - r[1] - w).abs() < 0.05, "{r:?}"); + ok(&mut a, "comment_edit", json!({ "doc": doc, "page": typed["page"], "index": typed["index"], "border_shape": "oval" })); + assert_eq!(list(&mut a)[0]["border_shape"], "oval"); + ok(&mut a, "comment_edit", json!({ "doc": doc, "page": typed["page"], "index": typed["index"], "border_shape": "rectangle" })); + assert!(list(&mut a)[0].get("border_shape").is_none(), "a rectangle is not reported"); + for (tool, args) in [ + ("fill_sign_add", json!({ "doc": doc, "page": 1, "type": "check", "at": [60, 60], "border_shape": "oval" })), + ("fill_sign_add", json!({ "doc": doc, "page": 1, "type": "text", "text": "7", "at": [60, 60], "border_shape": "circle" })), + ("comment_add", json!({ "doc": doc, "page": 1, "type": "rectangle", "rect": [80, 10, 120, 40], "border_shape": "oval" })), + ("comment_edit", json!({ "doc": doc, "page": typed["page"], "index": typed["index"], "border_shape": "star" })), + ] { + assert!(matches!(a.call(tool, &args), Err(ToolError::InvalidArgs(_))), "{args}"); + } + let callout = ok( + &mut a, + "comment_add", + json!({ "doc": doc, "page": 1, "type": "callout", "rect": [110, 100, 190, 140], "to": [40, 60], "contents": "Look" }), + ); + assert!(matches!( + a.call("comment_edit", &json!({ "doc": doc, "page": 1, "index": callout["comment"]["index"], "border_shape": "oval" })), + Err(ToolError::Failed(_)) + )); +} + +/// A rectangle's fill and a text box's text each take an opacity of their own through the tools +/// (the line or border keeps the comment's `opacity`); `opacity` sets the whole comment's again. +#[test] +fn comment_parts_take_opacities_of_their_own_through_the_tools() { + let dir = workdir("part-opacity"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok( + &mut a, + "comment_add", + json!({ "doc": doc, "page": 1, "type": "rectangle", "rect": [10, 10, 60, 40], "fill": "yellow", "fill_opacity": 0.25 }), + ); + ok( + &mut a, + "comment_add", + json!({ "doc": doc, "page": 1, "type": "textbox", "rect": [10, 200, 190, 240], "contents": "Hello", "fill": "#FFFF99", "width": 1, "fill_opacity": 0.2, "text_opacity": 1 }), + ); + let find = |a: &mut Automation, kind: &str| { + let list = ok(a, "comment_list", json!({ "doc": doc, "page": 1 })); + list["comments"].as_array().unwrap().iter().find(|c| c["type"] == kind).unwrap().clone() + }; + let rect = find(&mut a, "Square"); + assert_eq!((rect["opacity"].as_f64(), rect["fill_opacity"].as_f64(), rect.get("text_opacity")), (Some(1.0), Some(0.25), None)); + let tb = find(&mut a, "FreeText"); + assert_eq!((tb["opacity"].as_f64(), tb["fill_opacity"].as_f64(), tb["text_opacity"].as_f64()), (Some(1.0), Some(0.2), Some(1.0))); + // The fill changes, the text stays; an undo step. + ok(&mut a, "comment_edit", json!({ "doc": doc, "page": tb["page"], "index": tb["index"], "fill_opacity": 0.5 })); + let tb = find(&mut a, "FreeText"); + assert_eq!((tb["fill_opacity"].as_f64(), tb["text_opacity"].as_f64()), (Some(0.5), Some(1.0))); + assert_eq!(ok(&mut a, "edit_undo", json!({ "doc": doc }))["undone"], "Change comment properties"); + ok(&mut a, "edit_redo", json!({ "doc": doc })); + // The whole comment's opacity: everything at it again. + ok(&mut a, "comment_edit", json!({ "doc": doc, "page": rect["page"], "index": rect["index"], "opacity": 0.6 })); + let rect = find(&mut a, "Square"); + assert_eq!((rect["opacity"].as_f64(), rect["fill_opacity"].as_f64()), (Some(0.6), Some(0.6))); + // Only the parts a comment has, at 0–1. + for (tool, args) in [ + ("comment_add", json!({ "doc": doc, "page": 1, "type": "line", "from": [0, 0], "to": [50, 50], "fill_opacity": 0.5 })), + ("comment_add", json!({ "doc": doc, "page": 1, "type": "rectangle", "rect": [80, 10, 120, 40], "text_opacity": 0.5 })), + ("comment_edit", json!({ "doc": doc, "page": rect["page"], "index": rect["index"], "fill_opacity": 2 })), + ] { + assert!(matches!(a.call(tool, &args), Err(ToolError::InvalidArgs(_))), "{args}"); + } + assert!(matches!( + a.call("comment_edit", &json!({ "doc": doc, "page": rect["page"], "index": rect["index"], "text_opacity": 0.5 })), + Err(ToolError::Failed(_)) + )); +} + #[test] fn comments_through_tools() { let dir = workdir("comments"); @@ -2420,6 +2538,7 @@ fn protecting_through_tools() { // Copying wasn't allowed either (#568): no text or images come out, though search works. for (tool, args) in [ ("text_extract", json!({ "doc": re })), + ("text_extract", json!({ "doc": re, "format": "html" })), ("text_word_count", json!({ "doc": re })), ("text_paragraphs", json!({ "doc": re, "page": 1 })), ("doc_export_text", json!({ "doc": re, "path": "out.txt" })), @@ -2463,6 +2582,16 @@ fn copy_restricted_text_stays_in_the_document() { } let restyled = ok(&mut a, "text_edit", json!({ "doc": re, "page": 1, "paragraph": 1, "size": 30 })); assert!(!restyled.to_string().contains("Page"), "{restyled}"); + // A path that would carry the page's words OUT of the document — the editable output, or + // the review word list — is refused on a copy-restricted document (no engines needed). + match a.call("ocr_recognize", &json!({ "doc": re, "output_mode": "editable", "out": "copy.pdf" })) { + Err(ToolError::Failed(m)) => assert!(m.contains("don't allow copying"), "{m}"), + other => panic!("the editable output is refused: {other:?}"), + } + match a.call("ocr_words", &json!({ "doc": re })) { + Err(ToolError::Failed(m)) => assert!(m.contains("don't allow copying"), "{m}"), + other => panic!("ocr_words is refused: {other:?}"), + } if ok(&mut a, "ocr_status", json!({}))["available"] != true { eprintln!("skipped the OCR part: OCR models not installed"); return; @@ -3741,6 +3870,62 @@ fn enhance_camera_image_through_tools() { )); } +#[test] +fn enhance_scanned_file_cleans_the_scan_in_place_through_tools() { + let dir = workdir("enhance-scan"); + // A 300 × 400 grey scan: dull paper, a dark gutter at the left, text bars and a speck. + let img = image::GrayImage::from_fn(300, 400, |x, y| { + let light = 0.55 + 0.45 * (x as f32 / 60.0).min(1.0); + let ink = if ((60..360).contains(&y) && (y - 60) % 40 < 6 && (60..240).contains(&x)) || (x, y) == (150, 25) { 0.12 } else { 0.78 }; + image::Luma([(ink * light * 255.0) as u8]) + }); + img.save(dir.join("scan.jpg")).unwrap(); + let mut a = auto(&dir); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["scan.jpg"] }))["doc"].as_u64().unwrap(); + ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "before.jpg" })); + + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": scan, "level": 100 })); + assert_eq!(r["undo"], "Enhance scanned file"); + assert_eq!((r["pages_cleaned"].clone(), r["images"].clone(), r["not_scanned"].clone()), (json!([1]), json!(1), json!([]))); + assert_eq!(r["skipped"], json!([])); + // In place: still one picture of the same size, still a JPEG, now with white paper. + let images = ok(&mut a, "page_images", json!({ "doc": scan, "page": 1 })); + assert_eq!((images["count"].clone(), images["images"][0]["pixels"].clone()), (json!(1), json!([300, 400]))); + let saved = ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "after" })); + assert_eq!(saved["format"], "jpg"); + let (before, after) = (image::open(dir.join("before.jpg")).unwrap().to_luma8(), image::open(dir.join("after.jpg")).unwrap().to_luma8()); + for (x, y) in [(150, 40), (3, 40), (3, 380), (150, 25)] { + assert!( + before.get_pixel(x, y)[0] < 200 && after.get_pixel(x, y)[0] >= 230, + "({x}, {y}): {:?} → {:?}", + before.get_pixel(x, y), + after.get_pixel(x, y) + ); + } + assert!(after.get_pixel(150, 62)[0] <= 60, "a stroke stays dark"); + // Undo puts the scan back as it was. + ok(&mut a, "edit_undo", json!({ "doc": scan })); + ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "undone.jpg" })); + assert_eq!(std::fs::read(dir.join("undone.jpg")).unwrap(), std::fs::read(dir.join("before.jpg")).unwrap()); + + // Switches: despeckle alone leaves everything but the speck. + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": scan, "pages": [1], "level": 0, "despeckle": true, "edge_shadows": false })); + assert_eq!(r["images"], 1); + // A document of text pages has no scans: nothing changes, and it says so. + let text = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": text })); + assert_eq!((r["images"].clone(), r["not_scanned"].clone(), r["undo"].clone()), (json!(0), json!([1, 2, 3]), Value::Null)); + // Bad arguments are refused before anything changes. + for bad in [ + json!({ "doc": scan, "level": 101 }), + json!({ "doc": scan, "pages": [9] }), + json!({ "doc": scan, "despeckle": "yes" }), + json!({ "doc": scan, "pages": [] }), + ] { + assert!(matches!(a.call("page_enhance_scan", &bad), Err(ToolError::InvalidArgs(_))), "{bad}"); + } +} + #[test] fn enhance_camera_image_finds_the_edges_by_itself() { let dir = workdir("enhance-camera-auto"); @@ -3969,6 +4154,48 @@ fn fill_sign_text_and_date_colours_are_saved_and_validated() { } } +#[test] +fn fill_sign_text_border_is_saved_and_validated() { + let dir = workdir("fill-sign-text-border"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "fill_sign_add", json!({ "doc": doc, "page": 1, "type": "text", "at": [20, 40], "text": "Plain" })); + ok(&mut a, "fill_sign_add", json!({ "doc": doc, "page": 1, "type": "text", "at": [20, 80], "text": "Plain", "border": 1.5 })); + let comments = ok(&mut a, "comment_list", json!({ "doc": doc })); + let width = |k: usize| { + let rect = comments["comments"][k]["rect"].as_array().unwrap(); + rect[2].as_f64().unwrap() - rect[0].as_f64().unwrap() + }; + assert!((width(1) - width(0) - 3.0).abs() < 1e-6, "1.5 pt on each side: {comments}"); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "framed.pdf" })); + let saved = std::fs::read(dir.join("framed.pdf")).unwrap(); + let pdf = pdfcraft_cos::Document::open(std::sync::Arc::new(saved)).unwrap(); + let page = pdfcraft_model::pages(&pdf).remove(0); + let annotations = pdf.resolve(page.dict.get(b"Annots").unwrap()); + let annotation = pdf.resolve(annotations.as_array().unwrap().get(1).unwrap()); + let bs = annotation.as_dict().unwrap().get(b"BS").unwrap().as_dict().unwrap().get(b"W").unwrap().as_f64().unwrap(); + assert_eq!(bs, 1.5, "saved annotation has the requested border"); + // Its own colour, then the text's again through comment_edit. + ok( + &mut a, + "fill_sign_add", + json!({ "doc": doc, "page": 1, "type": "text", "at": [20, 160], "text": "Red", "border": 1, "border_color": "#ff0000" }), + ); + let list = ok(&mut a, "comment_list", json!({ "doc": doc, "page": 1 })); + let red = list["comments"].as_array().unwrap().iter().find(|c| c["contents"] == "Red").unwrap()["index"].clone(); + ok(&mut a, "comment_edit", json!({ "doc": doc, "page": 1, "index": red, "border_color": "text" })); + assert!( + a.call("fill_sign_add", &json!({ "doc": doc, "page": 1, "type": "text", "at": [20, 200], "text": "No", "border_color": "red" })).is_err(), + "no border" + ); + for (kind, border) in [("text", -1.0), ("text", 13.0), ("check", 1.0)] { + assert!(matches!( + a.call("fill_sign_add", &json!({ "doc": doc, "page": 1, "type": kind, "at": [20, 120], "text": "No", "border": border })), + Err(ToolError::InvalidArgs(_)) + )); + } +} + #[test] fn image_signatures_through_tools_preserve_transparency_and_survive_save() { let dir = workdir("image-signatures"); @@ -4613,6 +4840,35 @@ fn redacting_italian_international_slash_numbers_through_tools() { std::fs::remove_dir_all(dir).unwrap(); } +#[test] +/// A client obeying the advertised schema can select iban, and the mark made through the tool +/// covers the spaced number and nothing else. +fn redacting_an_iban_through_tools() { + let table = tools(); + let advertised: Vec<&str> = table.iter().find(|t| t.name == "redact_mark").unwrap().input_schema["properties"]["pattern"]["enum"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!(advertised.contains(&"iban"), "the advertised redact_mark schema must offer iban: {advertised:?}"); + let dir = workdir("redact-iban"); + std::fs::write(dir.join("invoice.txt"), "Pay DE89 3704 0044 0532 0130 00 today\nAccount 12345678 is ours\nTotal due").unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "invoice.txt" }))["doc"].as_u64().unwrap(); + let marked = ok(&mut a, "redact_mark", json!({ "doc": doc, "pattern": "iban" })); + assert_eq!(marked["marked"], 1, "the spaced German sample IBAN needs one mark"); + assert!(page_text(&mut a, doc)[0].contains("DE89 3704 0044 0532 0130 00"), "marks alone remove nothing"); + assert_eq!(ok(&mut a, "redact_apply", json!({ "doc": doc }))["applied"], 1); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "redacted.pdf" })); + let reopened = ok(&mut a, "doc_open", json!({ "path": "redacted.pdf" }))["doc"].as_u64().unwrap(); + let text = &page_text(&mut a, reopened)[0]; + assert!(!text.contains("DE89") && !text.contains("0532"), "{text}"); + assert!(text.lines().any(|line| line.trim() == "Pay"), "only the label before the number remains: {text}"); + assert!(text.contains("Account 12345678") && text.contains("Total due"), "text without an IBAN remains: {text}"); + std::fs::remove_dir_all(dir).unwrap(); +} + /// One page with the text "Secret 1234" and a Redact annotation that has neither a usable /// QuadPoints array nor a Rect, so there is no area to redact. fn redact_pdf_with_unreadable_mark() -> Vec { @@ -4959,6 +5215,83 @@ fn adding_content_through_tools() { assert!(matches!(a.call("content_delete", &json!({ "doc": doc, "page": 1, "index": 5 })), Err(ToolError::Failed(_)))); } +/// #6440: `page_add_text` paints text in CMYK or gray for a print file, so its black is 100 K +/// (`0 0 0 1 k`) rather than DeviceRGB; `content_update` restyles it either way. +#[test] +fn added_text_takes_a_cmyk_or_gray_ink_through_tools() { + let dir = workdir("content-ink"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "PROOF", "at": [20, 20], "cmyk": [0, 0, 0, 1] })); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "Note", "at": [20, 80], "gray": 0.5 })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], &items[1]["gray"]), (&json!([0.0, 0.0, 0.0, 1.0]), &json!(0.5))); + // The item streams of the saved file. + let saved_items = |a: &mut Automation, name: &str| -> Vec { + ok(a, "doc_save", json!({ "doc": doc, "path": name })); + let saved = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join(name)).unwrap())).unwrap(); + let page = &pdfcraft_model::pages(&saved)[0]; + let list = saved.resolve(page.dict.get(b"Contents").unwrap()).as_array().cloned().unwrap(); + list.iter() + .filter_map(|c| match &*saved.resolve(c) { + pdfcraft_cos::Object::Stream(s) if s.dict.name(b"PCMark") == Some(&b"Added"[..]) => { + Some(String::from_utf8_lossy(&s.decoded().unwrap()).into_owned()) + } + _ => None, + }) + .collect() + }; + let streams = saved_items(&mut a, "ink.pdf"); + assert!(streams[0].contains("0 0 0 1 k") && !streams[0].contains("rg"), "{}", streams[0]); + assert!(streams[1].contains("0.5 g") && !streams[1].contains("rg"), "{}", streams[1]); + // Restyled: to RGB with color, and back to CMYK. + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 2, "color": "#000000" })); + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 1, "cmyk": [1, 0, 0, 0] })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], items[1].get("gray")), (&json!([1.0, 0.0, 0.0, 0.0]), None)); + let streams = saved_items(&mut a, "restyled.pdf"); + assert!(streams[0].contains("1 0 0 0 k") && streams[1].contains("0 0 0 rg"), "{streams:?}"); + // One colour at a time, four numbers from 0 to 1. + for bad in [ + json!({ "color": "red", "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 1.5 }), + json!({ "cmyk": [0, 0, 1] }), + json!({ "cmyk": [0, 0, 0, 1, 0] }), + json!({ "cmyk": [0, "x", 0, 1] }), + json!({ "cmyk": [0, 0, 0, -0.1] }), + ] { + let mut args = json!({ "doc": doc, "page": 1, "text": "x", "at": [20, 200] }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("page_add_text", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["count"], 2, "nothing refused was added"); + // content_update refuses them too, an image takes no ink, and a refused update changes nothing. + let mut png = Vec::new(); + { + let mut enc = png::Encoder::new(&mut png, 40, 20); + enc.set_color(png::ColorType::Rgb); + let mut w = enc.write_header().unwrap(); + w.write_image_data(&[200u8; 2400]).unwrap(); + } + std::fs::write(dir.join("logo.png"), png).unwrap(); + ok(&mut a, "page_add_image", json!({ "doc": doc, "page": 1, "path": "logo.png", "rect": [100, 200, 180, 240] })); + let before = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + for (index, bad) in [ + (1, json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] })), + (2, json!({ "color": "red", "gray": 0.5 })), + (1, json!({ "size": 30, "cmyk": [0, 0, 0, 2] })), + (2, json!({ "bold": true, "gray": -1 })), + (3, json!({ "gray": 0.5 })), + (3, json!({ "cmyk": [0, 0, 0, 1] })), + ] { + let mut args = json!({ "doc": doc, "page": 1, "index": index }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("content_update", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["items"], before, "a refused update changed nothing"); +} + /// Edit ▸ Copy and Paste headlessly (#2636): an added text box or image, as it is, onto its own /// page beside it, onto a point, and into another document. #[test] @@ -6437,6 +6770,44 @@ fn drawing_comments_through_tools() { assert!(matches!(png[0], Content::Png { .. })); } +/// comment_add `ortho`, Shift's hold while drawing: a line's end is held level or plumb with its +/// start, and each corner of connected lines or a polygon with the corner before. +#[test] +fn lines_and_corners_are_held_level_or_plumb_through_the_tool() { + let dir = workdir("ortho"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + for args in [ + json!({ "type": "line", "from": [20, 40], "to": [120, 52], "ortho": true }), + json!({ "type": "arrow", "from": [180, 40], "to": [170, 140], "ortho": true }), + json!({ "type": "polyline", "points": [[20, 200], [90, 210], [95, 260], [130, 250]], "ortho": true }), + json!({ "type": "line", "from": [20, 280], "to": [120, 290] }), + ] { + let mut args = args; + args.as_object_mut().unwrap().extend([("doc".to_string(), json!(doc)), ("page".to_string(), json!(1))]); + ok(&mut a, "comment_add", args); + } + let list = ok(&mut a, "comment_list", json!({ "doc": doc, "page": 1 })); + // In the order added: the list isn't sorted by index. + let mut listed = list["comments"].as_array().unwrap().clone(); + listed.sort_by_key(|c| c["index"].as_u64()); + let points: Vec> = + listed.iter().map(|c| c["points"].as_array().unwrap().iter().map(|p| [p[0].as_f64().unwrap(), p[1].as_f64().unwrap()]).collect()).collect(); + let want: [&[[f64; 2]]; 4] = [ + &[[20.0, 40.0], [120.0, 40.0]], + &[[180.0, 40.0], [180.0, 140.0]], + &[[20.0, 200.0], [90.0, 200.0], [90.0, 260.0], [130.0, 260.0]], + &[[20.0, 280.0], [120.0, 290.0]], + ]; + assert_eq!(points.len(), want.len(), "{points:?}"); + for (got, want) in points.iter().zip(want) { + assert_eq!(got.len(), want.len(), "{got:?}"); + assert!(got.iter().zip(want).all(|(g, w)| (g[0] - w[0]).abs() < 0.01 && (g[1] - w[1]).abs() < 0.01), "{got:?} for {want:?}"); + } + let refused = a.call("comment_add", &json!({ "doc": doc, "page": 1, "type": "rectangle", "rect": [10, 10, 50, 50], "ortho": true })); + assert!(matches!(refused, Err(ToolError::InvalidArgs(_))), "ortho is for lines and corners: {refused:?}"); +} + #[test] fn signing_with_an_image_typed_or_drawn_graphic_through_tools() { let dir = workdir("signing-graphic"); @@ -6528,6 +6899,43 @@ fn a_dynamic_xfa_forms_signature_box_is_signed_under_its_som_path() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn named_layer_verdicts_are_reported_and_rendered_through_tools() { + use pdfcraft_cos::{Dict, Document, Lexer, ObjRef, Object, SaveOptions, Stream}; + let dir = workdir("named-layer-verdict"); + let mut cos = Document::open(std::sync::Arc::new(fixture(1))).unwrap(); + let dict = |text: &str| Lexer::new(text.as_bytes(), 0).object().unwrap().as_dict().unwrap().clone(); + let form = |resources: Dict, ops: &str| { + let mut d = dict("<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] >>"); + d.set(b"Resources".to_vec(), resources); + Stream::from_raw(d, ops.as_bytes().to_vec()) + }; + let blank = cos.add(form(Dict::new(), "")); + let layers = dict(&format!("<< /XObject << /n1 {blank} /n3 {blank} >> >>")); + let layer_form = cos.add(form(layers, "/n1 Do /n3 Do")); + let normal = cos.add(form(dict(&format!("<< /XObject << /FRM {layer_form} >> >>")), "/FRM Do")); + // Deliberately malformed CMS: an invalid signature still gets the redrawn invalid state. + let value = cos.add(dict("<< /Type /Sig /SubFilter /ETSI.CAdES.detached /ByteRange [0 0 0 0] /Contents <00> >>")); + let widget = cos + .add(dict(&format!("<< /FT /Sig /T (Signature1) /Subtype /Widget /F 4 /Rect [50 50 150 150] /P 4 0 R /V {value} /AP << /N {normal} >> >>"))); + cos.update_dict(ObjRef::new(4, 0), |p| p.set(b"Annots".to_vec(), vec![Object::Ref(widget)])).unwrap(); + cos.update_dict(cos.root().unwrap(), |root| root.set(b"AcroForm".to_vec(), dict(&format!("<< /Fields [{widget}] >>")))).unwrap(); + let original = pdfcraft_cos::write_full(&cos, &SaveOptions::default()).unwrap(); + std::fs::write(dir.join("named.pdf"), &original).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({"path": "named.pdf"}))["doc"].as_u64().unwrap(); + let list = ok(&mut a, "sign_list", json!({"doc": doc})); + assert_eq!(list["signatures"][0]["status"], "invalid"); + assert_eq!(list["signatures"][0]["verdict_shown"], true); + let rendered = a.call("page_render", &json!({"doc": doc, "page": 1, "dpi": 72})).unwrap(); + let Content::Png { data, .. } = &rendered[0] else { panic!("expected a PNG") }; + let pixels = image::load_from_memory_with_format(data, image::ImageFormat::Png).unwrap().to_rgba8(); + let red = pixels.pixels().filter(|p| p.0.iter().zip([0xD7, 0x37, 0x3F]).all(|(a, b)| a.abs_diff(b) < 30)).count(); + assert!(red > 100, "the invalid state is drawn: {red} red pixels"); + ok(&mut a, "doc_save", json!({"doc": doc, "path": "saved.pdf", "full": false})); + assert_eq!(std::fs::read(dir.join("saved.pdf")).unwrap(), original, "the drawn state never enters saves"); +} + #[test] fn digital_ids_signing_and_validation_through_tools() { let dir = workdir("signing"); @@ -6586,6 +6994,8 @@ fn sign_document_and_verify_trust(a: &mut Automation, dir: &Path) -> u64 { assert_eq!(t["trusted"].as_array().unwrap().len(), 1); let list = ok(a, "sign_list", json!({ "doc": doc })); assert_eq!((list["all_valid"].as_bool(), list["signatures"][0]["status"].as_str()), (Some(true), Some("valid"))); + // PdfCraft's own appearance has no status layers to draw into: only the page badge shows it. + assert_eq!(list["signatures"][0]["verdict_shown"], false); ok(a, "comment_add", json!({ "doc": doc, "page": 1, "type": "note", "at": [20, 20], "contents": "ok" })); let s = &ok(a, "sign_list", json!({ "doc": doc }))["signatures"][0]; assert_eq!((s["status"].as_str(), s["modification"].as_str()), (Some("valid"), Some("allowed"))); @@ -7220,6 +7630,8 @@ fn ocr_tools_make_a_scan_searchable() { let mut a = auto(&dir); let status = ok(&mut a, "ocr_status", json!({})); assert_eq!(status["languages"][0]["code"], "en"); + assert_eq!(status["engines"][0]["id"], "ocrs", "{status}"); + assert!(status["engines"][1]["id"] == "tesseract" && status["engines"][1]["available"].is_boolean(), "{status}"); if status["available"] != true { eprintln!("skipped: OCR models not installed"); return; @@ -7234,6 +7646,64 @@ fn ocr_tools_make_a_scan_searchable() { let again = ok(&mut a, "ocr_recognize", json!({ "doc": scan, "pages": [1] })); assert!(again["pages"][0]["skipped"].is_string(), "{again}"); assert!(matches!(a.call("ocr_recognize", &json!({ "doc": scan, "language": "xx" })), Err(ToolError::InvalidArgs(_)))); + // An engine that cannot run is named up front, before anything is read. + if status["engines"][1]["available"] != true { + let why = a.call("ocr_recognize", &json!({ "doc": scan, "engine": "tesseract", "skip_text_pages": false })).unwrap_err(); + assert!(matches!(why, ToolError::Failed(_)), "{why}"); + assert!(why.to_string().contains("tesseract"), "{why}"); + } + // ocr_words: the recognition's words with confidence, band and suspects, changing nothing. + // ocrs reports no confidences, so `auto` + a confidence merge is asked for when the second + // engine is there to supply them; without it confidence and band are honestly null. + ok(&mut a, "doc_close", json!({ "doc": scan, "discard_changes": true })); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); + let mut args = json!({ "doc": scan }); + if status["engines"][1]["available"] == true { + args["engine"] = json!("auto"); + args["strategy"] = json!("confidence"); + } + let words = ok(&mut a, "ocr_words", args); + assert!(words["words"].as_u64().unwrap() >= 4, "{words}"); + let first = &words["pages"][0]["words"][0]; + assert!(first["text"].is_string() && first["box"].as_array().is_some_and(|b| b.len() == 4), "{words}"); + if status["engines"][1]["available"] == true { + let with_confidence = words["pages"][0]["words"] + .as_array() + .expect("words") + .iter() + .filter(|w| w["confidence"].is_number() && w["band"].as_str().is_some()) + .count(); + assert!(with_confidence >= 4, "the second engine's confidences arrive: {words}"); + } else { + assert!(first["confidence"].is_null() && first["band"].is_null() && first["suspect"] == false, "{words}"); + } + assert!(a.call("edit_undo", &json!({ "doc": scan })).is_err(), "ocr_words changes nothing, so there is no undo"); +} + +/// An editable-text recognition writes a NEW file of visible text, never the source. +#[test] +fn ocr_recognize_editable_output_writes_visible_text() { + let dir = workdir("ocr-editable"); + let mut a = auto(&dir); + if ok(&mut a, "ocr_status", json!({}))["available"] != true { + eprintln!("skipped: OCR models not installed"); + return; + } + let text = ok(&mut a, "doc_create", json!({ "from": "text", "text": "Editable output words" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_images", json!({ "doc": text, "folder": "scan", "dpi": 150, "pages": [1] })); + let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_save", json!({ "doc": scan, "path": "src/scan.pdf" })); + // The output never lands on the source. + let why = a.call("ocr_recognize", &json!({ "doc": scan, "pages": [1], "output_mode": "editable", "out": "src/scan.pdf" })).unwrap_err(); + assert!(why.to_string().contains("source"), "{why}"); + let r = ok(&mut a, "ocr_recognize", json!({ "doc": scan, "pages": [1], "output_mode": "editable", "out": "out/ocr.pdf" })); + assert!(r["words"].as_u64().unwrap() >= 3, "{r}"); + let ocr = ok(&mut a, "doc_open", json!({ "path": "out/ocr.pdf" }))["doc"].as_u64().unwrap(); + // The text is visible (the engine may misread a letter at this dpi — look for a word that + // survives). + let found = ok(&mut a, "text_find", json!({ "doc": ocr, "query": "output" })); + assert!(found["count"].as_u64() >= Some(1), "the new file's text is visible: {found}"); } /// #1028: `skip_text_pages: false` reads a picture on a page that has its own text, and the @@ -7276,6 +7746,46 @@ fn ocr_recognize_files_refuses_paths_outside_the_root() { assert!(!dir.join("out").exists(), "nothing was started"); } +/// Two inputs of one file name from different folders would write the same output in the +/// batch folder: the batch is refused up front, with or without the OCR engines, naming the +/// file, and nothing is written. +#[test] +fn ocr_recognize_files_refuses_inputs_that_share_a_file_name() { + let dir = workdir("ocr-files-same-name"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "one/scan.pdf" })); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": "two/scan.pdf" })); + let r = a.call("ocr_recognize_files", &json!({ "paths": ["one/scan.pdf", "two/scan.pdf"], "folder": "out" })); + assert!(matches!(&r, Err(ToolError::InvalidArgs(m)) if m.contains("scan.pdf") && m.contains("overwrite")), "{r:?}"); + assert!(std::fs::read_dir(dir.join("out")).map_or(true, |mut d| d.next().is_none()), "nothing was written"); +} + +/// `out` belongs to the editable output. A searchable run (the default) writes into the open +/// document, so an `out` is refused rather than silently ignored, with or without the engines. +#[test] +fn ocr_recognize_refuses_an_out_that_nothing_would_use() { + let dir = workdir("ocr-out-ignored"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let r = a.call("ocr_recognize", &json!({ "doc": doc, "out": "../outside.pdf" })); + assert!(matches!(&r, Err(ToolError::InvalidArgs(m)) if m.contains("editable")), "{r:?}"); + assert!(!dir.join("../outside.pdf").exists()); +} + +/// The editable output is a NEW file: an existing `out` is refused before anything runs (with +/// or without the engines), and the file is left as it was. +#[test] +fn ocr_recognize_editable_output_never_replaces_an_existing_file() { + let dir = workdir("ocr-editable-exists"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + std::fs::write(dir.join("taken.pdf"), b"keep me").unwrap(); + let r = a.call("ocr_recognize", &json!({ "doc": doc, "output_mode": "editable", "out": "taken.pdf" })); + assert!(matches!(&r, Err(e) if e.to_string().contains("already exists")), "{r:?}"); + assert_eq!(std::fs::read(dir.join("taken.pdf")).unwrap(), b"keep me"); +} + #[test] fn ocr_recognize_files_writes_searchable_copies() { let dir = workdir("ocr-files"); @@ -7289,9 +7799,22 @@ fn ocr_recognize_files_writes_searchable_copies() { let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); ok(&mut a, "doc_save", json!({ "doc": scan, "path": "in/scan.pdf" })); + // A file that already has text: the skip-files option passes it over as its own bucket. + let typed = ok(&mut a, "doc_create", json!({ "from": "text", "text": "Already typed" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_save", json!({ "doc": typed, "path": "in/typed.pdf" })); + // Collision pre-flight: the existing out/scan.pdf (written below) stops a rerun. let r = ok(&mut a, "ocr_recognize_files", json!({ "paths": ["in/scan.pdf", "missing.pdf"], "folder": "out" })); assert!(r["files"][0]["words"].as_u64().unwrap() >= 3, "{r}"); assert!(r["files"][1]["error"].is_string(), "{r}"); + let why = a.call("ocr_recognize_files", &json!({ "paths": ["in/scan.pdf"], "folder": "out" })).unwrap_err(); + assert!(matches!(why, ToolError::Failed(_)), "{why}"); + assert!(why.to_string().contains("scan.pdf"), "{why}"); + // The skip-files bucket and the low-confidence note. + let r = ok(&mut a, "ocr_recognize_files", json!({ "paths": ["in/typed.pdf", "in/scan.pdf"], "folder": "out2", "skip_text_files": true })); + assert_eq!(r["skipped_files"].as_u64(), Some(1), "{r}"); + assert!(r["files"][0]["skipped"].is_string(), "{r}"); + assert_eq!(r["recognized"].as_u64(), Some(1), "{r}"); + assert!(r["files"][1]["output"].is_string(), "{r}"); let out = ok(&mut a, "doc_open", json!({ "path": "out/scan.pdf" }))["doc"].as_u64().unwrap(); let found = ok(&mut a, "text_find", json!({ "doc": out, "query": "recognition" })); assert!(found.to_string().contains("\"page\""), "{found}"); @@ -7301,18 +7824,6 @@ fn ocr_recognize_files_writes_searchable_copies() { /// the second output replaces the first while both results report success. The batch is refused /// up front, naming the file, before the folder is created, anything is read or the OCR engines /// load, so the outcome is the same with or without the models installed. -#[test] -fn ocr_recognize_files_refuses_inputs_that_share_a_file_name() { - let dir = workdir("ocr-files-same-name"); - let mut a = auto(&dir); - let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); - ok(&mut a, "doc_save", json!({ "doc": doc, "path": "one/scan.pdf" })); - ok(&mut a, "doc_save", json!({ "doc": doc, "path": "two/scan.pdf" })); - let r = a.call("ocr_recognize_files", &json!({ "paths": ["one/scan.pdf", "two/scan.pdf"], "folder": "out" })); - assert!(matches!(&r, Err(ToolError::InvalidArgs(m)) if m.contains("scan.pdf")), "{r:?}"); - assert!(!dir.join("out").exists(), "nothing was started: the output folder was not created"); -} - /// File names are compared without regard to case on Windows and macOS (`scan.pdf` and `Scan.PDF` /// are one file there), and exactly elsewhere, where they are two files and the batch is not /// refused. @@ -8917,27 +9428,15 @@ fn a_deleted_image_is_not_kept_by_reduce_or_optimize() { /// page is turned with /Rotate 90. fn table_pdf() -> Vec { let body = "BT /F1 12 Tf 20 250 Td (Name) Tj 130 0 Td (Qty) Tj -130 -20 Td (Apple) Tj 130 0 Td (12) Tj -130 -20 Td (Pear) Tj 130 0 Td (7) Tj ET"; - let objs = [ - "<< /Type /Catalog /Pages 2 0 R >>".to_string(), - "<< /Type /Pages /Kids [4 0 R 5 0 R] /Count 2 /MediaBox [0 0 300 300] >>".to_string(), - "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), - "<< /Type /Page /Parent 2 0 R /Contents 6 0 R /Resources << /Font << /F1 3 0 R >> >> >>".to_string(), - "<< /Type /Page /Parent 2 0 R /Rotate 90 /Contents 6 0 R /Resources << /Font << /F1 3 0 R >> >> >>".to_string(), - format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()), - ]; - let mut out = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for (i, o) in objs.iter().enumerate() { - offsets.push(out.len()); - out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); - } - let xref = out.len(); - out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); - for o in offsets { - out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); - } - out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); - out + let stream = format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()); + pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [4 0 R 5 0 R] /Count 2 /MediaBox [0 0 300 300] >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "<< /Type /Page /Parent 2 0 R /Contents 6 0 R /Resources << /Font << /F1 3 0 R >> >> >>", + "<< /Type /Page /Parent 2 0 R /Rotate 90 /Contents 6 0 R /Resources << /Font << /F1 3 0 R >> >> >>", + &stream, + ]) } /// Issue #740: `text_extract` with `rect` takes what Column select takes, in the displayed-page @@ -8978,6 +9477,56 @@ fn text_extract_with_a_rect_takes_one_column() { } } +/// Copy with formatting, headless: `format: "html"` adds each page's text as the HTML fragment +/// Copy puts on the clipboard, for the whole page or only what `rect` takes. +#[test] +fn text_extract_as_html_keeps_the_formatting() { + let dir = workdir("rich-copy"); + std::fs::write(dir.join("table.pdf"), table_pdf()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "table.pdf" }))["doc"].as_u64().unwrap(); + let page = &ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "format": "html" }))["pages"][0]; + let html = page["html"].as_str().unwrap(); + assert!(html.starts_with("

Name"), "{html}"); + assert!(html.contains("Pear") && html.contains("Qty"), "{html}"); + assert_eq!(page["text"], "Name\nApple\nPear\nQty\n12\n7", "the plain text is unchanged"); + assert_eq!(page["styles_incomplete"], false); + // Plain text by default: no html key. + assert!(ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1] }))["pages"][0].get("html").is_none()); + // Only what the rect takes; nothing inside gives empty HTML. + let column = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [120, 0, 300, 300], "format": "html" })); + let html = column["pages"][0]["html"].as_str().unwrap(); + assert!(html.contains("Qty") && !html.contains("Name"), "{html}"); + let none = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [0, 0, 5, 5], "format": "html" })); + assert_eq!(none["pages"][0]["html"], ""); + // An unknown format is refused. + for bad in [json!("rtf"), json!(1)] { + let e = a.call("text_extract", &json!({ "doc": doc, "format": bad })).unwrap_err(); + assert!(matches!(e, ToolError::InvalidArgs(_)), "{bad}: {e}"); + } + // Bold, italic, colour and size come through end to end. + let body = "BT /F1 18 Tf 20 250 Td (Big) Tj ET BT /F2 12 Tf 20 200 Td (Strong) Tj ET \ + BT /F3 12 Tf 20 150 Td (Slanted) Tj ET BT 0 0 1 rg /F1 12 Tf 20 100 Td (Blue) Tj ET"; + let stream = format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()); + let styled = pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 300] >>", + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R /F3 7 0 R >> >> >>", + &stream, + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Oblique >>", + ]); + std::fs::write(dir.join("styled.pdf"), styled).unwrap(); + let doc = ok(&mut a, "doc_open", json!({ "path": "styled.pdf" }))["doc"].as_u64().unwrap(); + let page = &ok(&mut a, "text_extract", json!({ "doc": doc, "format": "html" }))["pages"][0]; + let html = page["html"].as_str().unwrap(); + assert!(html.contains("font-size:18pt\">Big"), "{html}"); + assert!(html.contains("Strong"), "{html}"); + assert!(html.contains("Slanted"), "{html}"); + assert!(html.contains("color:#0000FF\">Blue"), "{html}"); +} + #[test] fn mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools() { use pdfcraft_cos::{Dict, Document, Object, Stream, write_incremental}; @@ -10225,6 +10774,31 @@ fn changing_a_lists_options_through_tools_keeps_its_saved_selection_on_the_same_ assert_eq!(saved_selection_indices(&dir.join("letters.pdf"), "letters"), Some(vec![0, 1]), "B and C are now items 0 and 1"); } +/// Clearing a field's length limit through `form_set_props` (`"max_length": 0`) leaves it with +/// none, even when its parent field has one it would otherwise inherit. +#[test] +fn clearing_a_kids_length_limit_through_tools_does_not_bring_back_its_parents() { + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 400 300] >>", + "<< /Type /Page /Parent 2 0 R /Annots [6 0 R] >>", + "<< /Fields [5 0 R] /DA (/Helv 12 Tf 0 g) /DR << /Font << /Helv 7 0 R >> >> >>", + "<< /T (address) /FT /Tx /MaxLen 3 /Kids [6 0 R] >>", + "<< /Type /Annot /Subtype /Widget /T (city) /Parent 5 0 R /MaxLen 8 /Rect [10 250 150 268] /P 3 0 R >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + ]; + let dir = workdir("kid-max-len"); + std::fs::write(dir.join("form.pdf"), pdf_of(&objs)).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "form.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "form_set_props", json!({ "doc": doc, "field": "address.city", "max_length": 0 })); + ok(&mut a, "form_fill", json!({ "doc": doc, "values": { "address.city": "Absalom" } })); + let fields = ok(&mut a, "form_fields", json!({ "doc": doc })); + let city = fields["fields"].as_array().unwrap().iter().find(|f| f["name"] == "address.city").unwrap().clone(); + assert!(city.get("max_length").is_none(), "no limit, not the parent's 3: {city}"); + assert_eq!(city["value"], "Absalom", "the value is not cut to the parent's 3 characters"); +} + /// Clearing a field's default through `form_set_props` (`"default": ""`) leaves it with none, /// even when its parent field has one it would otherwise inherit. #[test] @@ -10494,6 +11068,162 @@ fn simple_rule_scan_drawn(content: &[u8]) -> Vec { write_full(&doc, &SaveOptions::default()).unwrap() } +/// Reviewing headlessly: corrections keep the boxes, a stale or foreign review is refused, and +/// applying consumes the handle (the pages already accepted were applied once). +#[test] +fn ocr_review_corrections_keep_boxes_are_guarded_and_apply_once() { + fn render(a: &mut Automation, doc: u64) -> Vec { + let mut result = a.call("page_render", &json!({ "doc": doc, "page": 1, "dpi": 72 })).unwrap(); + match result.remove(0) { + Content::Png { data, .. } => data, + other => panic!("expected a rendered page, got {other:?}"), + } + } + let dir = workdir("ocr-review"); + let mut a = auto(&dir); + if ok(&mut a, "ocr_status", json!({}))["available"] != true { + eprintln!("skipped: OCR models not installed"); + return; + } + let text = ok(&mut a, "doc_create", json!({ "from": "text", "text": "Review these original words" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_images", json!({ "doc": text, "folder": "scan", "dpi": 150, "pages": [1] })); + let png = std::fs::read_dir(dir.join("scan")).unwrap().next().unwrap().unwrap().path(); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": [png.to_string_lossy()] }))["doc"].as_u64().unwrap(); + let before = render(&mut a, scan); + let found = ok(&mut a, "ocr_words", json!({ "doc": scan, "retain_review": true })); + assert!(found["words"].as_u64().unwrap() >= 3, "{found}"); + let review = found["review"].as_u64().unwrap(); + assert!(page_text(&mut a, scan).iter().all(String::is_empty)); + for args in [ + json!({ "doc": text, "review": review }), + json!({ "doc": scan, "review": review, "corrections": [{"page":1,"word":999999,"text":"bad"}] }), + json!({ "doc": scan, "review": review, "corrections": [{"page":1,"word":1,"text":"bad","origin":[0,0]}] }), + json!({ "doc": scan, "review": review, "corrections": [{"page":1,"word":1,"text":"a".repeat(4*1024*1024+1)}] }), + json!({ "doc": scan, "review": review, "corrections": [{"page":1,"word":1,"text":"a"},{"page":1,"word":1,"text":"b"}] }), + ] { + assert!(a.call("ocr_review_apply", &args).is_err()); + assert!(page_text(&mut a, scan).iter().all(String::is_empty)); + } + let result = ok( + &mut a, + "ocr_review_apply", + json!({ "doc": scan, "review": review, "corrections": [{"page":1,"word":1,"text":"日本語"},{"page":1,"word":2,"text":""}] }), + ); + assert!(result["applied"].as_u64().unwrap() >= 2); + let extracted = page_text(&mut a, scan).join("\n"); + assert!(extracted.contains("日本語"), "{extracted}"); + assert_eq!(render(&mut a, scan), before, "corrected text remains invisible at its original geometry"); + assert!(a.call("ocr_review_apply", &json!({ "doc": scan, "review": review })).is_err(), "consumed reviews cannot be replayed"); + ok(&mut a, "edit_undo", json!({ "doc": scan })); + assert!(page_text(&mut a, scan).iter().all(String::is_empty)); + assert!(a.call("edit_undo", &json!({ "doc": scan })).is_err(), "review acceptance is one undo step"); + + let found = ok(&mut a, "ocr_words", json!({ "doc": scan, "retain_review": true })); + let stale = found["review"].as_u64().unwrap(); + ok(&mut a, "page_rotate", json!({ "doc": scan, "pages": [1], "degrees": 90 })); + let error = a.call("ocr_review_apply", &json!({ "doc": scan, "review": stale })).unwrap_err().to_string(); + assert!(error.contains("changed since recognition"), "{error}"); + assert!(page_text(&mut a, scan).iter().all(String::is_empty)); + ok(&mut a, "ocr_review_apply", json!({ "doc": scan, "review": stale, "discard": true })); + assert!(a.call("ocr_review_apply", &json!({ "doc": scan, "review": stale })).is_err()); + ok(&mut a, "edit_undo", json!({ "doc": scan })); + + let first = ok(&mut a, "ocr_words", json!({ "doc": scan, "retain_review": true }))["review"].as_u64().unwrap(); + let second = ok(&mut a, "ocr_words", json!({ "doc": scan, "retain_review": true }))["review"].as_u64().unwrap(); + assert_ne!(first, second); + assert!(a.call("ocr_review_apply", &json!({ "doc": scan, "review": first })).is_err()); + ok( + &mut a, + "ocr_review_apply", + json!({ "doc": scan, "review": second, "output_mode": "editable", "out": "reviewed.pdf", "corrections": [{"page":1,"word":1,"text":"Corrected"}] }), + ); + assert!(page_text(&mut a, scan).iter().all(String::is_empty)); + let output = ok(&mut a, "doc_open", json!({ "path": "reviewed.pdf" }))["doc"].as_u64().unwrap(); + assert!(page_text(&mut a, output).join("\n").contains("Corrected")); + // The editable output is a new file: an existing one is refused and the review is kept. + let third = ok(&mut a, "ocr_words", json!({ "doc": scan, "retain_review": true }))["review"].as_u64().unwrap(); + let again = a.call("ocr_review_apply", &json!({ "doc": scan, "review": third, "output_mode": "editable", "out": "reviewed.pdf" })); + assert!(matches!(&again, Err(e) if e.to_string().contains("already exists")), "{again:?}"); + ok(&mut a, "ocr_review_apply", json!({ "doc": scan, "review": third, "discard": true })); +} + +#[test] +fn ocr_dictionary_updates_are_atomic_bounded_and_confined() { + let dir = workdir("ocr-dictionary"); + let mut a = auto(&dir); + let path = "ocr-dictionary-en.txt"; + let out = ok(&mut a, "ocr_dictionary", json!({ "path": path, "add": ["Quartz", "quartz", "日本語"] })); + assert_eq!(out["count"], 2); + assert_eq!(pdfcraft_engine::ocr::dictionary::words(&std::fs::read_to_string(dir.join(path)).unwrap()).unwrap().len(), 2); + let before = std::fs::read(dir.join(path)).unwrap(); + for args in [ + json!({ "path": path, "add": ["a\nb"] }), + json!({ "path": path, "add": ["x".repeat(1025)] }), + json!({ "path": path, "add": ["İ".repeat(512)] }), + json!({ "path": path, "add": vec!["x"; 65537] }), + json!({ "path": "../outside-ocr-dictionary.txt", "add": ["x"] }), + ] { + assert!(a.call("ocr_dictionary", &args).is_err()); + } + assert_eq!(std::fs::read(dir.join(path)).unwrap(), before); + assert_eq!(ok(&mut a, "ocr_dictionary", json!({ "path": path, "remove": ["QUARTZ"] }))["words"], json!(["日本語"])); + assert_eq!(ok(&mut a, "ocr_dictionary", json!({ "path": path }))["count"], 1); + std::fs::write(dir.join("ocr-dictionary-large.txt"), vec![b'a'; 4 * 1024 * 1024 + 1]).unwrap(); + let too_big = a.call("ocr_dictionary", &json!({ "path": "ocr-dictionary-large.txt", "add": ["x"] })).unwrap_err().to_string(); + assert!(!too_big.contains("must name a dictionary file"), "refused for its size, not its name: {too_big}"); + assert_eq!(std::fs::metadata(dir.join("ocr-dictionary-large.txt")).unwrap().len(), 4 * 1024 * 1024 + 1); +} + +/// The tool rewrites what it is pointed at (lower-cased, sorted), so it only takes dictionary file +/// names: another small text file under the root is neither read nor changed. +#[test] +fn ocr_dictionary_refuses_files_that_are_not_dictionaries() { + let dir = workdir("ocr-dictionary-names"); + let mut a = auto(&dir); + let env = b"TOKEN=Zebra +APPLE=1 +"; + for name in [".env", "notes.txt", "ocr-dictionary-en.md", "ocr-dictionary-e n.txt", "sub/readme.txt"] { + if let Some(parent) = std::path::Path::new(name).parent() { + std::fs::create_dir_all(dir.join(parent)).unwrap(); + } + std::fs::write(dir.join(name), env).unwrap(); + for args in [json!({ "path": name }), json!({ "path": name, "add": [] }), json!({ "path": name, "add": ["x"] })] { + let err = a.call("ocr_dictionary", &args).unwrap_err().to_string(); + assert!(err.contains("must name a dictionary file"), "{name} {args}: {err}"); + } + assert_eq!(std::fs::read(dir.join(name)).unwrap(), env, "{name} is untouched"); + } +} + +#[test] +fn ocr_retained_reviews_have_eight_document_cap_and_close_releases_slot() { + let dir = workdir("ocr-review-cap"); + let mut a = auto(&dir); + if ok(&mut a, "ocr_status", json!({}))["available"] != true { + eprintln!("skipped: OCR models not installed"); + return; + } + let mut docs = Vec::new(); + let mut handles = Vec::new(); + for _ in 0..9 { + let doc = ok(&mut a, "doc_create", json!({"from":"text", "text":"Already recognized text"}))["doc"].as_u64().unwrap(); + docs.push(doc); + } + for doc in docs.iter().take(8) { + handles.push(ok(&mut a, "ocr_words", json!({"doc":doc,"retain_review":true}))["review"].as_u64().unwrap()); + } + let error = a.call("ocr_words", &json!({"doc":docs[8],"retain_review":true})).unwrap_err().to_string(); + assert!(error.contains("at most 8"), "{error}"); + let replacement = ok(&mut a, "ocr_words", json!({"doc":docs[0],"retain_review":true}))["review"].as_u64().unwrap(); + assert_ne!(replacement, handles[0]); + assert!(a.call("ocr_review_apply", &json!({"doc":docs[0],"review":handles[0],"discard":true})).is_err()); + ok(&mut a, "doc_close", json!({"doc":docs[0],"discard_changes":true})); + assert!(a.call("ocr_review_apply", &json!({"doc":docs[0],"review":replacement})).is_err()); + let final_handle = ok(&mut a, "ocr_words", json!({"doc":docs[8],"retain_review":true}))["review"].as_u64().unwrap(); + ok(&mut a, "ocr_review_apply", json!({"doc":docs[8],"review":final_handle,"discard":true})); +} + #[test] fn editor_crop_group_move_and_arrange_survive_added_item_deletion_through_tools() { let dir = workdir("editor-crop-group-arrange"); @@ -10673,3 +11403,214 @@ fn multiline_fill_sign_engine_round_trip_preserves_lines_and_appearance() { assert_eq!(rendered, re.call("page_render", &json!({"doc":reopened,"page":1,"dpi":72})).unwrap(), "saved appearances are identical"); std::fs::remove_dir_all(dir).unwrap(); } + +/// Export, then import into another document and copy between open documents: the tree comes +/// back unchanged, a page the target lacks is reported, and the import undoes in one step. +#[test] +fn bookmarks_export_import_and_copy_between_documents() { + let dir = workdir("bookmark-io"); + let mut a = auto(&dir); + let src = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "bookmark_add", json!({ "doc": src, "title": "Intro", "page": 1 })); + ok(&mut a, "bookmark_add", json!({ "doc": src, "title": "Detail", "page": 3, "parent": [1] })); + ok(&mut a, "bookmark_add", json!({ "doc": src, "title": "End", "page": 2 })); + ok(&mut a, "bookmark_set_style", json!({ "doc": src, "path": [1], "color": [1, 0, 0], "bold": true })); + let exported = ok(&mut a, "bookmark_export", json!({ "doc": src, "out": "marks.json" })); + assert_eq!(exported["truncated"], false); + let tree = exported["bookmarks"].clone(); + assert_eq!((tree[0]["title"].clone(), tree[0]["color"].clone(), tree[0]["bold"].clone()), (json!("Intro"), json!([1.0, 0.0, 0.0]), json!(true))); + assert_eq!((tree[0]["children"][0]["title"].clone(), tree[0]["children"][0]["page"].clone()), (json!("Detail"), json!(3))); + assert_eq!(tree[0]["view"]["type"], "XYZ"); + let subtree = ok(&mut a, "bookmark_export", json!({ "doc": src, "path": [1, 1] }))["bookmarks"].clone(); + assert_eq!(subtree, json!([tree[0]["children"][0]])); + + // b.pdf has two pages: Detail goes to the third, so it arrives without a destination. + let dst = ok(&mut a, "doc_open", json!({ "path": "b.pdf" }))["doc"].as_u64().unwrap(); + let imported = ok(&mut a, "bookmark_import", json!({ "doc": dst, "file": "marks.json" })); + assert_eq!(imported["unmapped"], json!([[1, 1]])); + let back = ok(&mut a, "bookmark_export", json!({ "doc": dst }))["bookmarks"].clone(); + assert_eq!(back[0]["children"][0]["page"], json!(null)); + assert_eq!((back[0]["title"].clone(), back[1].clone()), (tree[0]["title"].clone(), tree[1].clone())); + ok(&mut a, "edit_undo", json!({ "doc": dst })); + assert_eq!(ok(&mut a, "bookmark_list", json!({ "doc": dst }))["bookmarks"], json!([])); + + // Into a document with every page, the tree is unchanged, wherever it is copied from. + let same = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "bookmark_add", json!({ "doc": same, "title": "Old", "page": 1 })); + let copied = ok(&mut a, "bookmark_copy", json!({ "doc": same, "from": src, "replace": true })); + assert_eq!(copied["unmapped"], json!([])); + assert_eq!(ok(&mut a, "bookmark_export", json!({ "doc": same }))["bookmarks"], tree); + ok(&mut a, "bookmark_import", json!({ "doc": same, "bookmarks": subtree, "parent": [2], "position": 1 })); + assert_eq!(ok(&mut a, "bookmark_list", json!({ "doc": same }))["bookmarks"][1]["children"][0]["title"], "Detail"); +} + +/// A one-page document whose outline has an untitled item, a /FitR destination with three of +/// its four numbers, and a chain nested `deep` levels below them. +fn odd_outline(deep: usize) -> Vec { + let first_deep = 7; + let mut objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /Outlines 4 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R >>".into(), + "<< /Type /Outlines /First 5 0 R /Last 6 0 R >>".into(), + "<< /Title () /Parent 4 0 R /Next 6 0 R /Dest [3 0 R /Fit] >>".into(), + format!( + "<< /Title (Short) /Parent 4 0 R /Prev 5 0 R /Dest [3 0 R /FitR 0 0 10]{} >>", + if deep > 0 { format!(" /First {first_deep} 0 R /Last {first_deep} 0 R") } else { String::new() } + ), + ]; + for i in 0..deep { + let (me, parent) = (first_deep + i, if i == 0 { 6 } else { first_deep + i - 1 }); + let child = if i + 1 < deep { format!(" /First {0} 0 R /Last {0} 0 R", me + 1) } else { String::new() }; + objs.push(format!("<< /Title (Level) /Parent {parent} 0 R /Dest [3 0 R /Fit]{child} >>")); + } + pdf_of(&objs.iter().map(String::as_str).collect::>()) +} + +/// What a real file holds and the JSON form can't say exactly still copies and round-trips: an +/// empty title stays empty, a short /FitR keeps its page and drops the view, and bookmarks past +/// the listing depth are reported as left out at every step. +#[test] +fn odd_bookmarks_round_trip_and_a_cut_tree_says_so() { + let dir = workdir("bookmark-io-odd"); + std::fs::write(dir.join("odd.pdf"), odd_outline(0)).unwrap(); + std::fs::write(dir.join("deep.pdf"), odd_outline(40)).unwrap(); + let mut a = auto(&dir); + let src = ok(&mut a, "doc_open", json!({ "path": "odd.pdf" }))["doc"].as_u64().unwrap(); + let tree = ok(&mut a, "bookmark_export", json!({ "doc": src, "out": "odd.json" }))["bookmarks"].clone(); + assert_eq!((tree[0]["title"].clone(), tree[1]["page"].clone(), tree[1].get("view")), (json!(""), json!(1), None)); + let dst = ok(&mut a, "doc_open", json!({ "path": "b.pdf" }))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut a, "bookmark_import", json!({ "doc": dst, "file": "odd.json", "replace": true }))["truncated"], false); + // A destination with no view is written as XYZ with nulls: the same page, the reader's view. + let back = ok(&mut a, "bookmark_export", json!({ "doc": dst }))["bookmarks"].clone(); + assert_eq!((back[0].clone(), back[1]["title"].clone(), back[1]["page"].clone()), (tree[0].clone(), json!("Short"), json!(1))); + assert_eq!(back[1]["view"], json!({ "type": "XYZ", "values": [null, null, null] })); + let copied = ok(&mut a, "bookmark_copy", json!({ "doc": dst, "from": src, "replace": true })); + assert_eq!((copied["unmapped"].clone(), copied["truncated"].clone()), (json!([]), json!(false))); + match a.call("bookmark_export", &json!({ "doc": src, "path": [3] })) { + Err(ToolError::Failed(m)) => assert!(m.contains("no bookmark at [3]"), "{m}"), + other => panic!("expected no bookmark, got {other:?}"), + } + + let deep = ok(&mut a, "doc_open", json!({ "path": "deep.pdf" }))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut a, "bookmark_export", json!({ "doc": deep, "out": "deep.json" }))["truncated"], true); + let saved: Value = serde_json::from_slice(&std::fs::read(dir.join("deep.json")).unwrap()).unwrap(); + assert_eq!(saved["truncated"], true); + assert_eq!(ok(&mut a, "bookmark_import", json!({ "doc": dst, "file": "deep.json", "replace": true }))["truncated"], true); + assert_eq!(ok(&mut a, "bookmark_copy", json!({ "doc": dst, "from": deep, "replace": true }))["truncated"], true); + let export = tools().into_iter().find(|t| t.name == "bookmark_export").unwrap(); + assert!(export.destructive && !export.read_only, "bookmark_export overwrites out"); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_misspelt_bookmark_key_is_refused_not_dropped() { + let dir = workdir("bookmark-io-key"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let tree = json!([{ "title": "Parent", "children": [{ "title": "Red", "page": 1, "colour": [1, 0, 0] }] }]); + match a.call("bookmark_import", &json!({ "doc": doc, "bookmarks": tree })) { + Err(ToolError::InvalidArgs(m)) => assert!(m.contains("[1, 1]") && m.contains("\"colour\""), "{m}"), + other => panic!("expected InvalidArgs, got {other:?}"), + } + assert_eq!(ok(&mut a, "bookmark_list", json!({ "doc": doc }))["bookmarks"], json!([])); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_bad_bookmark_tree_is_refused_whole_and_names_the_node() { + let dir = workdir("bookmark-io-bad"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let refused = |a: &mut Automation, bookmarks: Value| match a.call("bookmark_import", &json!({ "doc": doc, "bookmarks": bookmarks })) { + Err(ToolError::InvalidArgs(m)) => m, + other => panic!("expected InvalidArgs, got {other:?}"), + }; + let fine = json!({ "title": "Fine", "page": 1 }); + let m = refused(&mut a, json!([fine, { "title": "Parent", "children": [{ "page": 1 }] }])); + assert!(m.contains("[2, 1]") && m.contains("title"), "{m}"); + let m = refused(&mut a, json!([{ "title": 5 }])); + assert!(m.contains("[1]") && m.contains("title"), "{m}"); + let m = refused(&mut a, json!([fine, { "title": "Bad", "page": 0 }])); + assert!(m.contains("[2]") && m.contains("page"), "{m}"); + let m = refused(&mut a, json!([{ "title": "Bad", "color": [2, 0, 0] }])); + assert!(m.contains("color"), "{m}"); + let m = refused(&mut a, json!([{ "title": "Bad", "view": { "type": "XYZ", "values": [1, 2, 3, 4, 5] } }])); + assert!(m.contains("view"), "{m}"); + let mut deep = json!({ "title": "Leaf" }); + for _ in 0..40 { + deep = json!({ "title": "Level", "children": [deep] }); + } + assert!(refused(&mut a, json!([deep])).contains("33 levels")); + let m = refused(&mut a, json!([fine, { "title": "Z", "page": 1, "view": { "type": "Zoom" } }])); + assert!(m.contains("[2]") && m.contains("XYZ (3 values"), "{m}"); + let m = refused(&mut a, json!([fine, { "title": "Z", "page": 1, "view": { "type": "FitH", "values": [1, 2] } }])); + assert!(m.contains("[2]") && m.contains("FitH view takes 1"), "{m}"); + let m = refused(&mut a, json!([{ "title": "R", "view": { "type": "FitR", "values": [0, null, 1, 1] } }])); + assert!(m.contains("all four"), "{m}"); + assert!(matches!(a.call("bookmark_import", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); + assert!(matches!(a.call("bookmark_import", &json!({ "doc": doc, "bookmarks": [fine], "file": "x.json" })), Err(ToolError::InvalidArgs(_)))); + // A real bookmark file outside the root is refused for being outside, not for being missing. + let outside = dir.with_extension("outside.json"); + std::fs::write(&outside, br#"{"bookmarks": [{"title": "Outside", "page": 1}]}"#).unwrap(); + let name = format!("../{}", outside.file_name().unwrap().to_string_lossy()); + match a.call("bookmark_import", &json!({ "doc": doc, "file": name })) { + Err(ToolError::Failed(m)) => assert!(m.contains("outside the allowed directory"), "{m}"), + other => panic!("expected the root to refuse it, got {other:?}"), + } + let _ = std::fs::remove_file(&outside); + let out = dir.with_extension("out.json"); + let name = format!("../{}", out.file_name().unwrap().to_string_lossy()); + match a.call("bookmark_export", &json!({ "doc": doc, "out": name })) { + Err(ToolError::Failed(m)) => assert!(m.contains("outside the allowed directory"), "{m}"), + other => panic!("expected the root to refuse it, got {other:?}"), + } + assert!(!out.exists()); + assert_eq!(ok(&mut a, "bookmark_list", json!({ "doc": doc }))["bookmarks"], json!([])); +} + +/// Copy and paste comments between documents through the tools: the copies carry their replies, +/// land on the asked-for page under their own names (a second paste gives a second set), and a +/// plain text is refused. +#[test] +fn comments_copy_and_paste_between_documents_through_tools() { + let dir = workdir("comments-clipboard"); + let mut a = auto(&dir); + let src = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "comment_add", json!({ "doc": src, "page": 2, "type": "note", "at": [30, 30], "contents": "Please review", "author": "Ada" })); + ok(&mut a, "comment_add", json!({ "doc": src, "page": 2, "type": "rectangle", "rect": [20, 20, 80, 60], "contents": "Box" })); + ok(&mut a, "comment_reply", json!({ "doc": src, "page": 2, "index": 1, "text": "Done", "author": "Bob" })); + // Positions as comment_list reports them (a note's pop-up takes a position of its own). + let listed = ok(&mut a, "comment_list", json!({ "doc": src, "page": 2 })); + let pairs: Vec = listed["comments"].as_array().unwrap().iter().map(|c| json!([c["page"], c["index"]])).collect(); + assert_eq!(pairs.len(), 2, "{listed}"); + let copied = ok(&mut a, "comments_copy", json!({ "doc": src, "comments": pairs })); + assert_eq!(copied["comments"], 2); + let xfdf = copied["xfdf"].as_str().unwrap().to_owned(); + assert!(xfdf.contains("inreplyto=") && xfdf.contains("> = + placed["comments"].as_array().unwrap().iter().map(|c| c["rect"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect()).collect(); + let (x0, y0, x1, y1) = + rects.iter().fold((f64::MAX, f64::MAX, f64::MIN, f64::MIN), |(a, b, c, d), r| (a.min(r[0]), b.min(r[1]), c.max(r[2]), d.max(r[3]))); + assert!(((x0 + x1) / 2.0 - 100.0).abs() < 1.0 && ((y0 + y1) / 2.0 - 150.0).abs() < 1.0, "centred at [100, 150]: {rects:?}"); + assert!(matches!(a.call("comments_paste", &json!({ "doc": dst, "page": 1, "xfdf": "hello" })), Err(ToolError::InvalidArgs(_)))); + assert!(a.call("comments_copy", &json!({ "doc": src, "comments": [[9, 9]] })).is_err(), "no such comment"); + assert!(matches!(a.call("comments_copy", &json!({ "doc": src, "comments": [[2]] })), Err(ToolError::InvalidArgs(_)))); + assert_eq!(ok(&mut a, "comment_list", json!({ "doc": src }))["comments"].as_array().unwrap().len(), 2, "the source is untouched"); +} diff --git a/crates/automation/tests/it/automation_roots.rs b/crates/automation/tests/it/automation_roots.rs index a96870ca02..e65c582a73 100644 --- a/crates/automation/tests/it/automation_roots.rs +++ b/crates/automation/tests/it/automation_roots.rs @@ -145,6 +145,8 @@ fn tools_that_read_a_path_are_held_to_the_read_root() { ok(&mut server, "doc_open", json!({"path": s(&base.join("in/a.pdf"))})); let (secret, pic, gone) = (s(&base.join("outside/secret.pdf")), s(&base.join("outside/pic.png")), s(&base.join("outside/gone.bin"))); let out = s(&base.join("out/result.pdf")); + std::fs::write(base.join("in/m.prc"), b"PRC model").unwrap(); + let model = s(&base.join("in/m.prc")); for (tool, args) in [ ("page_insert_file", json!({"doc": 1, "path": secret, "at": 1})), ("page_replace", json!({"doc": 1, "pages": [1], "path": secret})), @@ -153,6 +155,8 @@ fn tools_that_read_a_path_are_held_to_the_read_root() { ("doc_create", json!({"from": "images", "paths": [pic]})), ("stamp_custom", json!({"doc": 1, "page": 1, "path": secret, "at": [10, 10]})), ("stamp_custom", json!({"doc": 1, "page": 1, "path": secret, "rect": [10, 10, 493, 300]})), + ("model_add", json!({"doc": 1, "page": 1, "path": secret, "rect": [10, 10, 100, 100]})), + ("model_add", json!({"doc": 1, "page": 1, "path": model, "rect": [10, 10, 100, 100], "poster": pic})), ("page_add_image", json!({"doc": 1, "page": 1, "path": pic})), ("image_edit", json!({"doc": 1, "page": 1, "image": 1, "action": "replace", "path": pic})), ("comment_add", json!({"doc": 1, "page": 1, "type": "attachment", "at": [10, 10], "path": secret})), diff --git a/crates/automation/tests/it/batch.rs b/crates/automation/tests/it/batch.rs index af5e0c1272..8b7650fd3f 100644 --- a/crates/automation/tests/it/batch.rs +++ b/crates/automation/tests/it/batch.rs @@ -238,3 +238,61 @@ fn mcp_batch_errors_keep_the_structured_per_file_report() { assert_eq!(report["result"]["isError"], true, "{report}"); assert_eq!(report["result"]["structuredContent"]["failed"], 1); } + +#[test] +fn dry_run_planned_files_are_not_counted_as_skipped() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + let original = pdf(); + std::fs::write(dir.path().join("input/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("input/sub/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("out/a.pdf"), b"precious").unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["optimize"],"out_dir":"out","dry_run":true})); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["skipped"], 0, "planned files are not skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 2, "{report}"); + assert!(files.iter().all(|file| file["status"] == "planned"), "{report}"); + assert!(files.iter().all(|file| file["steps"] == json!(["optimize"])), "planned files list the ordered steps: {report}"); + assert!(files[0]["output"].as_str().unwrap().ends_with("out/a (2).pdf"), "existing output gets the publish collision name: {report}"); + assert!(files[1]["output"].as_str().unwrap().ends_with("out/a (3).pdf"), "later planned files reserve their names: {report}"); + assert_eq!(std::fs::read(dir.path().join("out/a.pdf")).unwrap(), b"precious"); + assert!(!dir.path().join("out/a (2).pdf").exists(), "dry run writes nothing: {report}"); + #[cfg(unix)] + { + // An output subfolder that resolves outside out_dir is a per-file error: + // it counts as failed, honors stop_on_error, and later files are skipped. + for outside_root in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/z.pdf"), pdf()).unwrap(); + let target = if outside_root { outside.path().to_path_buf() } else { dir.path().join("outside") }; + if !outside_root { + std::fs::create_dir(&target).unwrap(); + } + std::os::unix::fs::symlink(&target, dir.path().join("out/sub")).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result( + &mut auto, + json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","recursive":true,"dry_run":true,"stop_on_error":true}), + ); + assert_eq!(report["failed"], 1, "preview target failure is per-file: {report}"); + assert_eq!(report["stopped"], true, "{report}"); + assert_eq!(report["skipped"], 1, "only the unprocessed file is skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 3, "{report}"); + assert_eq!(files[0]["status"], "planned", "{report}"); + assert_eq!(files[1]["status"], "error", "{report}"); + assert_eq!(files[2]["status"], "skipped", "stop_on_error leaves later files unplanned: {report}"); + assert!(!dir.path().join("out/a.pdf").exists(), "dry run writes nothing: {report}"); + } + } +} diff --git a/crates/automation/tests/it/batch_failures.rs b/crates/automation/tests/it/batch_failures.rs new file mode 100644 index 0000000000..f993d85eab --- /dev/null +++ b/crates/automation/tests/it/batch_failures.rs @@ -0,0 +1,89 @@ +//! A batch run that fails a file publishes only the completed outputs, and a +//! publish the file system refuses publishes nothing. + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_engine::Session; +use serde_json::{Value, json}; + +fn result(auto: &mut Automation, args: Value) -> Value { + match auto.call("doc_batch", &args).unwrap().remove(0) { + Content::Json(value) => value, + _ => panic!("expected a report"), + } +} + +fn pdf() -> Vec { + Session::new().create_from_text("Original", "Original text").unwrap().as_ref().clone() +} + +/// Sets a folder's permission bits. +#[cfg(unix)] +fn set_mode(path: &std::path::Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); +} + +/// Puts back the write permission the temporary folder needs before it is removed. +#[cfg(unix)] +struct RestoreMode(std::path::PathBuf); + +#[cfg(unix)] +impl Drop for RestoreMode { + fn drop(&mut self) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)); + } +} + +#[test] +fn failed_inputs_publish_only_completed_outputs() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + std::fs::write(dir.path().join("input/good.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/bad.pdf"), "not a pdf").unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","recursive":false})); + assert_eq!(report["completed"], 1, "{report}"); + assert_eq!(report["failed"], 1, "{report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 2, "{report}"); + let bad = files.iter().find(|file| file["path"].as_str().is_some_and(|path| path.ends_with("bad.pdf"))).unwrap(); + assert_eq!(bad["status"], "error", "{report}"); + assert!(bad["output"].is_null(), "a failed file reports no output: {report}"); + let names: Vec = + std::fs::read_dir(dir.path().join("out")).unwrap().map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()).collect(); + // Publish stages its bytes under a dot-prefixed name in the output folder, so + // no entry carrying that prefix may survive the run. + for name in &names { + assert!(!name.starts_with('.'), "temporary name left behind: {name} ({names:?})"); + } + assert_eq!(names, vec!["good.pdf".to_string()], "only the completed output is published"); +} + +#[cfg(unix)] +#[test] +fn read_only_output_folder_publishes_nothing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + std::fs::write(dir.path().join("input/good.pdf"), pdf()).unwrap(); + let out = dir.path().join("out"); + std::fs::create_dir(&out).unwrap(); + set_mode(&out, 0o555); + let _restore = RestoreMode(out.clone()); + // A root process ignores the mode bits, and then the assertions below would + // pass without testing the refusal. Fail loudly instead of skipping. + assert!(std::fs::write(out.join("probe"), b"").is_err(), "run as a non-root user: the read-only output folder accepted a new file"); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","recursive":false})); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 1, "{report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 1, "{report}"); + // Publish cannot create its staging file in a folder it may not write, so + // the entry carries the system's error and no output. + assert_eq!(files[0]["status"], "error", "{report}"); + assert!(files[0]["output"].is_null(), "a refused publish reports no output: {report}"); + let error = files[0]["error"].as_str().unwrap(); + assert!(!error.is_empty(), "{report}"); + assert_eq!(std::fs::read_dir(&out).unwrap().count(), 0, "the refused publish writes nothing into the read-only folder: {report}"); +} diff --git a/crates/automation/tests/it/batch_secrets.rs b/crates/automation/tests/it/batch_secrets.rs new file mode 100644 index 0000000000..08223f0955 --- /dev/null +++ b/crates/automation/tests/it/batch_secrets.rs @@ -0,0 +1,84 @@ +//! Tool results and errors from the folder tools carry no known test password. +//! +//! The passwords are set on the document and passed to the tool, and must never come back out: a +//! result that echoes one has leaked it into logs, transcripts and MCP callers. + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_engine::{Changes, Edit, Protection, Session}; +use serde_json::{Value, json}; + +/// The known test passwords; neither may appear in a result or an error. +const OPEN_PASSWORD: &str = "PLANT-OPEN-5f2c9a"; +const OWNER_PASSWORD: &str = "PLANT-OWNER-71de0b"; + +/// A one-page PDF locked with the known test passwords. +/// +/// The open password is the only way in. The batch bates tool rewrites page content, and +/// `Changes::None` forbids that, so this fixture permits every change except extracting pages; +/// the restrictions stay real, since the open password never lifts them. +fn protected_pdf() -> Vec { + let mut session = Session::new(); + let bytes = session.create_from_text("Batch secrets", "Planted password test").unwrap(); + let id = session.open("locked.pdf", None, bytes, None).unwrap(); + session + .apply( + id, + Edit::Protect(Protection { + open_password: Some(OPEN_PASSWORD.into()), + permissions_password: Some(OWNER_PASSWORD.into()), + changes: Changes::AnyExceptExtract, + ..Default::default() + }), + ) + .unwrap(); + session.save_bytes(id).unwrap().as_ref().clone() +} + +fn json_result(a: &mut Automation, tool: &str, args: Value) -> Value { + match a.call(tool, &args) { + Ok(mut content) => match content.remove(0) { + Content::Json(value) => value, + other => panic!("{tool}: expected JSON, got {other:?}"), + }, + Err(error) => panic!("{tool} {args}: {error}"), + } +} + +#[test] +fn results_and_errors_hold_no_planted_password_for_bates_or_batch() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("locked.pdf"), protected_pdf()).unwrap(); + let mut a = Automation::new().with_root(dir.path()).unwrap(); + + // The right open password: numbering succeeds and reports the numbers only. + let numbered = + json_result(&mut a, "doc_bates_multi", json!({"files":[{"input":"locked.pdf","output":"locked-numbered.pdf","password":OPEN_PASSWORD}]})); + assert_eq!(numbered["completed"], 1, "{numbered}"); + assert_eq!(numbered["files"][0]["status"], "ok", "{numbered}"); + // The report carries the per-file entry the Bates tool builds, so a field added to that entry + // lands in the text checked below. + assert_eq!(numbered["files"][0]["first"], 1, "{numbered}"); + assert!(numbered["files"][0]["output"].as_str().is_some_and(|path| path.ends_with("locked-numbered.pdf")), "{numbered}"); + assert!(dir.path().join("locked-numbered.pdf").exists(), "the numbered copy is written"); + + // The wrong open password: the tool refuses the file with an error. + let refused = match a + .call("doc_bates_multi", &json!({"files":[{"input":"locked.pdf","output":"locked-refused.pdf","password":"not-the-open-password"}]})) + { + Err(error) => error.to_string(), + Ok(value) => panic!("a wrong open password must be refused: {value:?}"), + }; + + // The folder job refuses the encrypted file without a password, in its per-file report. + let batch = json_result(&mut a, "doc_batch", json!({"paths":["locked.pdf"],"steps":["optimize"],"out_dir":"out"})); + assert_eq!(batch["failed"], 1, "{batch}"); + assert_eq!(batch["files"][0]["status"], "error", "{batch}"); + assert!(batch["files"][0]["error"].as_str().is_some_and(|text| !text.is_empty()), "{batch}"); + assert!(!dir.path().join("out/locked.pdf").exists(), "a refused file publishes nothing"); + + for report in [serde_json::to_string(&numbered).unwrap(), refused, serde_json::to_string(&batch).unwrap()] { + for secret in [OPEN_PASSWORD, OWNER_PASSWORD] { + assert!(!report.contains(secret), "a result or error echoed a planted password: {report}"); + } + } +} diff --git a/crates/automation/tests/it/main.rs b/crates/automation/tests/it/main.rs index d36dbe93a9..92e54c99a6 100644 --- a/crates/automation/tests/it/main.rs +++ b/crates/automation/tests/it/main.rs @@ -1,14 +1,20 @@ //! The crate's integration tests, linked into one test binary. +mod action_failures; mod attachment_key; mod automation; mod automation_roots; mod backdrop; mod batch; +mod batch_failures; +mod batch_secrets; mod button_caption; mod form_bulk; mod hostile_args; +mod model_3d; +mod model_render; mod optimize_content_boundaries; mod redact_after_stray_q_form; mod redact_text_after_q; mod render_warnings; +mod unreadable_objects; diff --git a/crates/automation/tests/it/model_3d.rs b/crates/automation/tests/it/model_3d.rs new file mode 100644 index 0000000000..179192f421 --- /dev/null +++ b/crates/automation/tests/it/model_3d.rs @@ -0,0 +1,120 @@ +//! `model_add` embeds a U3D or PRC model as a 3D annotation (ISO 32000-2 §13.6.2) with a poster, +//! end to end: the tool, the saved file, undo, and the refusals. + +use pdfcraft_automation::{Automation, Content, ToolError}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use serde_json::{Value, json}; + +fn tool(a: &mut Automation, name: &str, args: Value) -> Value { + match a.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +fn png(w: u32, h: u32) -> Vec { + let mut out = Vec::new(); + let mut encoder = png::Encoder::new(&mut out, w, h); + encoder.set_color(png::ColorType::Rgb); + encoder.set_depth(png::BitDepth::Eight); + encoder.write_header().unwrap().write_image_data(&vec![90; (w * h * 3) as usize]).unwrap(); + out +} + +fn stream(doc: &Document, obj: &Object) -> Stream { + match doc.resolve(obj).as_ref() { + Object::Stream(s) => s.clone(), + other => panic!("expected a stream: {other:?}"), + } +} + +/// The 3D annotations on the first page of the saved file. +fn saved_3d(dir: &std::path::Path) -> (Document, Vec) { + let doc = Document::open(std::sync::Arc::new(std::fs::read(dir.join("out.pdf")).unwrap())).unwrap(); + let page = &pdfcraft_model::pages(&doc)[0]; + let list = page.dict.get(b"Annots").map(|a| doc.resolve(a).as_array().cloned().unwrap()).unwrap_or_default(); + let found = list.iter().map(|o| doc.resolve(o).as_dict().unwrap().clone()).filter(|d| d.name(b"Subtype") == Some(&b"3D"[..])).collect(); + (doc, found) +} + +#[test] +fn model_add_embeds_the_model_with_a_fitted_poster() { + let dir = tempfile::tempdir().unwrap(); + let mut prc = b"PRC".to_vec(); + prc.extend_from_slice(&[7; 200]); + std::fs::write(dir.path().join("bracket.prc"), &prc).unwrap(); + std::fs::write(dir.path().join("poster.png"), png(80, 40)).unwrap(); + let mut a = Automation::new().with_root(dir.path()).unwrap(); + let doc = tool(&mut a, "doc_create", json!({ "from": "blank", "width": 600, "height": 800 }))["doc"].as_u64().unwrap(); + // A 300 × 300 box from the top-left of the page: user space [100, 400, 400, 700]. + let call = + json!({ "doc": doc, "page": 1, "path": "bracket.prc", "rect": [100, 100, 400, 400], "poster": "poster.png", "activation": "page_open" }); + tool(&mut a, "model_add", call); + tool(&mut a, "doc_save", json!({ "doc": doc, "path": "out.pdf" })); + + let (saved, found) = saved_3d(dir.path()); + let [annot] = found.as_slice() else { panic!("one 3D annotation: {found:?}") }; + let rect: Vec = annot.get(b"Rect").unwrap().as_array().unwrap().iter().filter_map(Object::as_f64).collect(); + assert_eq!(rect, [100.0, 400.0, 400.0, 700.0]); + assert_eq!(annot.get(b"Contents").and_then(Object::as_string).map(|s| s.to_text()).as_deref(), Some("bracket")); + let model = stream(&saved, annot.get(b"3DD").unwrap()); + assert_eq!(model.dict.name(b"Type"), Some(&b"3D"[..])); + assert_eq!(model.dict.name(b"Subtype"), Some(&b"PRC"[..])); + assert_eq!(model.dict.name(b"Filter"), Some(&b"FlateDecode"[..])); + assert_eq!(model.decoded().unwrap().to_vec(), prc, "the model is stored unchanged"); + let activation = saved.resolve(annot.get(b"3DA").unwrap()).as_dict().unwrap().clone(); + assert_eq!(activation.name(b"A"), Some(&b"PO"[..])); + // The 80 × 40 picture fitted in the 300 × 300 box: 300 × 150, centred top to bottom. + let ap = saved.resolve(annot.get(b"AP").unwrap()).as_dict().unwrap().clone(); + let poster = stream(&saved, ap.get(b"N").unwrap()); + let content = String::from_utf8(poster.decoded().unwrap().to_vec()).unwrap(); + assert!(content.contains("300 0 0 150 100 475 cm /Pic Do"), "{content}"); + + assert_eq!(tool(&mut a, "edit_undo", json!({ "doc": doc }))["undone"], "Add 3D model"); + tool(&mut a, "doc_save", json!({ "doc": doc, "path": "out.pdf" })); + assert!(saved_3d(dir.path()).1.is_empty(), "undo removes it"); +} + +#[test] +fn model_add_without_a_poster_draws_a_labelled_frame_and_activates_on_click() { + let dir = tempfile::tempdir().unwrap(); + let mut u3d = b"U3D\0".to_vec(); + u3d.extend_from_slice(&[0; 60]); + std::fs::write(dir.path().join("part.u3d"), &u3d).unwrap(); + let mut a = Automation::new().with_root(dir.path()).unwrap(); + let doc = tool(&mut a, "doc_create", json!({ "from": "blank", "width": 600, "height": 800 }))["doc"].as_u64().unwrap(); + tool(&mut a, "model_add", json!({ "doc": doc, "page": 1, "path": "part.u3d", "rect": [50, 50, 250, 200], "name": "Part" })); + tool(&mut a, "doc_save", json!({ "doc": doc, "path": "out.pdf" })); + + let (saved, found) = saved_3d(dir.path()); + let [annot] = found.as_slice() else { panic!("one 3D annotation: {found:?}") }; + assert_eq!(stream(&saved, annot.get(b"3DD").unwrap()).dict.name(b"Subtype"), Some(&b"U3D"[..])); + let activation = saved.resolve(annot.get(b"3DA").unwrap()).as_dict().unwrap().clone(); + assert_eq!(activation.name(b"A"), Some(&b"XA"[..]), "the standard's default, stated"); + let ap = saved.resolve(annot.get(b"AP").unwrap()).as_dict().unwrap().clone(); + let content = String::from_utf8(stream(&saved, ap.get(b"N").unwrap()).decoded().unwrap().to_vec()).unwrap(); + assert!(content.contains("re S") && content.contains("(3D model) Tj"), "{content}"); +} + +#[test] +fn model_add_refuses_what_is_not_a_model_and_adds_nothing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("notes.txt"), b"PR not a model").unwrap(); + std::fs::write(dir.path().join("part.prc"), b"PRC....").unwrap(); + let mut a = Automation::new().with_root(dir.path()).unwrap(); + let doc = tool(&mut a, "doc_create", json!({ "from": "blank", "width": 600, "height": 800 }))["doc"].as_u64().unwrap(); + let not_a_model = a.call("model_add", &json!({ "doc": doc, "page": 1, "path": "notes.txt", "rect": [10, 10, 100, 100] })); + assert!(matches!(¬_a_model, Err(ToolError::Failed(m)) if m.contains("U3D") && m.contains("PRC")), "{not_a_model:?}"); + for (args, why) in [ + (json!({ "rect": [10, 10, 10, 100] }), "zero width"), + (json!({ "rect": [0, 0, 20000, 100] }), "wider than a PDF page can be"), + (json!({}), "no rect"), + (json!({ "rect": [10, 10, 100, 100], "activation": "now" }), "an unknown activation"), + ] { + let mut call = json!({ "doc": doc, "page": 1, "path": "part.prc" }); + call.as_object_mut().unwrap().extend(args.as_object().unwrap().clone()); + assert!(matches!(a.call("model_add", &call), Err(ToolError::InvalidArgs(_))), "{why}"); + } + tool(&mut a, "doc_save", json!({ "doc": doc, "path": "out.pdf" })); + assert!(saved_3d(dir.path()).1.is_empty()); +} diff --git a/crates/automation/tests/it/model_render.rs b/crates/automation/tests/it/model_render.rs new file mode 100644 index 0000000000..615a5346a9 --- /dev/null +++ b/crates/automation/tests/it/model_render.rs @@ -0,0 +1,112 @@ +//! `model_render` draws a page's 3D model headlessly: a synthetic PDF holding a synthetic PRC. + +use std::path::PathBuf; + +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +/// One triangle at z = 0 facing +z, with corners (-1,-1), (1,-1) and (0,1), as a PRC file of one +/// file structure with no product tree, in hexadecimal for the stream's `/ASCIIHexDecode`. +const TRIANGLE_PRC_HEX: &str = "505243c91f0000c91f00000100000002000000030000000400000000000000000000000000000000000000010000000500000006000000070000000800000000000000060000006b0000009a000000a3000000ab000000cd000000d5000000dd000000e500000000000000505243c91f0000c91f0000050000000600000007000000080000000000000000000000000000000000000000000000789c63000000010001789c030000000001789c9b71c087758da2422b43084fe3140e038713464cd759381a162800006ed70828789c030000000001789c030000000001789c030000000001"; + +/// A one-page PDF whose page holds a 3D annotation over a stream of `/Subtype /{subtype}`. +fn pdf(subtype: &str, hex: &str) -> Vec { + let stream = format!("<< /Type /3D /Subtype /{subtype} /Filter /ASCIIHexDecode /Length {} >>\nstream\n{hex}>\nendstream", hex.len() + 1); + let objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [<< /Type /Annot /Subtype /Square /Rect [0 0 10 10] >> 4 0 R] >>".into(), + "<< /Type /Annot /Subtype /3D /Rect [20 20 180 180] /3DD 5 0 R >>".into(), + stream, + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// A fresh folder with `prc.pdf` and `u3d.pdf` in `in`, `outside.pdf` beside it, and an +/// automation rooted at `in`. +fn setup(test: &str) -> (PathBuf, Automation) { + let dir = std::env::temp_dir().join(format!("pdfcraft-model3d-{test}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("in")).unwrap(); + std::fs::write(dir.join("in/prc.pdf"), pdf("PRC", TRIANGLE_PRC_HEX)).unwrap(); + std::fs::write(dir.join("in/u3d.pdf"), pdf("U3D", "553344")).unwrap(); + std::fs::write(dir.join("outside.pdf"), pdf("PRC", TRIANGLE_PRC_HEX)).unwrap(); + let a = Automation::new().with_root(dir.join("in")).unwrap(); + (dir, a) +} + +fn open(a: &mut Automation, path: &str) -> u64 { + match a.call("doc_open", &json!({ "path": path })).unwrap().remove(0) { + Content::Json(v) => v["doc"].as_u64().unwrap(), + other => panic!("expected JSON: {other:?}"), + } +} + +/// The RGB of pixel (x, y) of a PNG. +fn rgb(data: &[u8], x: usize, y: usize) -> [u8; 3] { + let mut reader = png::Decoder::new(std::io::Cursor::new(data)).read_info().unwrap(); + let mut pixels = vec![0; reader.output_buffer_size().unwrap()]; + let info = reader.next_frame(&mut pixels).unwrap(); + let channels = info.line_size / info.width as usize; + let at = y * info.line_size + x * channels; + [pixels[at], pixels[at + 1], pixels[at + 2]] +} + +#[test] +fn a_prc_model_is_drawn_over_the_background() { + let (_dir, mut a) = setup("prc"); + let doc = open(&mut a, "prc.pdf"); + let args = json!({ "doc": doc, "page": 1, "width": 64, "height": 48, "direction": [0, 0, -1], "perspective": false }); + let out = a.call("model_render", &args).unwrap(); + let Content::Png { data, width, height } = &out[0] else { panic!("expected a PNG") }; + assert_eq!((*width, *height), (64, 48)); + assert_eq!(rgb(data, 0, 0), [255, 255, 255], "the corner is background"); + assert_eq!(rgb(data, 32, 24), [190, 192, 200], "the centre is the triangle, lit head on"); + // In perspective the headlight meets the face at a slight angle, so only not-background holds. + let out = a.call("model_render", &json!({ "doc": doc, "page": 1, "width": 64, "height": 48, "direction": [0, 0, -1] })).unwrap(); + let Content::Png { data, .. } = &out[0] else { panic!("expected a PNG") }; + assert_ne!(rgb(data, 32, 24), [255, 255, 255], "the centre is drawn in perspective too"); + // The same model named explicitly, and the default isometric view, both draw. + assert!(a.call("model_render", &json!({ "doc": doc, "page": 1, "model": 1 })).is_ok()); +} + +#[test] +fn a_u3d_model_or_a_missing_one_is_a_clear_error() { + let (_dir, mut a) = setup("u3d"); + let doc = open(&mut a, "u3d.pdf"); + let err = a.call("model_render", &json!({ "doc": doc, "page": 1 })).unwrap_err().to_string(); + assert!(err.contains("U3D"), "{err}"); + let doc = open(&mut a, "prc.pdf"); + let err = a.call("model_render", &json!({ "doc": doc, "page": 1, "model": 2 })).unwrap_err().to_string(); + assert!(err.contains("has 1 3D models, not 2"), "{err}"); + for bad in [json!({ "direction": [0, 0, 0] }), json!({ "direction": [1, 2] }), json!({ "width": 0 }), json!({ "height": 5000 })] { + let mut args = json!({ "doc": doc, "page": 1 }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(a.call("model_render", &args).is_err(), "{bad}"); + } +} + +#[test] +fn a_model_outside_the_root_is_never_opened_to_be_drawn() { + let (dir, mut a) = setup("root"); + let outside = dir.join("outside.pdf"); + for path in [Value::from("../outside.pdf"), Value::from(outside.to_string_lossy().into_owned())] { + assert!(a.call("doc_open", &json!({ "path": path })).is_err(), "{path} lies outside the root"); + } + // The tool takes no path of its own: only a document opened inside the root reaches it. + assert!(a.call("model_render", &json!({ "doc": 1, "page": 1 })).is_err()); + let doc = open(&mut a, "prc.pdf"); + assert!(a.call("model_render", &json!({ "doc": doc, "page": 1 })).is_ok()); +} diff --git a/crates/automation/tests/it/unreadable_objects.rs b/crates/automation/tests/it/unreadable_objects.rs new file mode 100644 index 0000000000..f19a19078a --- /dev/null +++ b/crates/automation/tests/it/unreadable_objects.rs @@ -0,0 +1,64 @@ +//! `doc_info` lists the objects of a file that could not be read: each reads as null, so the file +//! opens, but what it held is lost and nothing was repaired. + +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +/// A one-page PDF whose document information dictionary (object 4) is `info`. +fn with_info(info: &str) -> Vec { + let bodies = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + info, + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, body) in bodies.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", bodies.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R /Info 4 0 R >>\nstartxref\n{xref}\n%%EOF\n", bodies.len() + 1).as_bytes()); + out +} + +fn info_of(a: &mut Automation, path: &str) -> Value { + let doc = match a.call("doc_open", &json!({ "path": path })).unwrap().remove(0) { + Content::Json(v) => v["doc"].as_u64().unwrap(), + other => panic!("expected JSON: {other:?}"), + }; + match a.call("doc_info", &json!({ "doc": doc })).unwrap().remove(0) { + Content::Json(v) => v, + other => panic!("expected JSON: {other:?}"), + } +} + +/// An unparseable `/Info` is listed with its number and why, the rest of the file reads, and +/// nothing is reported as repaired. +#[test] +fn doc_info_lists_an_object_that_could_not_be_read() { + let dir = std::env::temp_dir().join(format!("pdfcraft-unreadable-objects-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("broken.pdf"), with_info("<< /Title (Report) /Author ) >>")).unwrap(); + std::fs::write(dir.join("sound.pdf"), with_info("<< /Title (Report) >>")).unwrap(); + let mut a = Automation::new().with_root(&dir).unwrap(); + + let info = info_of(&mut a, "broken.pdf"); + let lines = info["unreadable_objects"].as_array().unwrap(); + assert_eq!(lines.len(), 1, "{info}"); + let line = lines[0].as_str().unwrap(); + assert!(line.starts_with("object 4 0 could not be read: syntax error"), "{line}"); + assert_eq!(info["repairs"], json!([]), "nothing was repaired"); + assert_eq!(info["pages"].as_array().unwrap().len(), 1, "the rest of the file reads"); + + let info = info_of(&mut a, "sound.pdf"); + assert_eq!(info["unreadable_objects"], json!([]), "{info}"); + assert_eq!(info["title"], "Report"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/automation/tests/pattern_redaction.rs b/crates/automation/tests/pattern_redaction.rs new file mode 100644 index 0000000000..667779cef6 --- /dev/null +++ b/crates/automation/tests/pattern_redaction.rs @@ -0,0 +1,146 @@ +//! Rectangle redaction must not report a pattern-cell secret safely removed (#3647). + +use std::sync::Arc; + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_engine::DocId; +use serde_json::{Value, json}; + +fn tool(automation: &mut Automation, name: &str, args: Value) -> Value { + match automation.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +/// `retained`: paint with the pattern once more, away from the mark (x 290..340 lies right of it). +/// A path the mark hits is removed whole (F-008) and a pattern nothing surviving paints with is +/// pruned, so only a pattern that some surviving paint still uses reaches the guard. +fn fixture(cell: &[u8], partial: bool, compressed: bool, retained: bool) -> Vec { + let mut pattern = Dict::from_iter([ + (b"Type".to_vec(), Object::name("Pattern")), + (b"PatternType".to_vec(), Object::Int(1)), + (b"PaintType".to_vec(), Object::Int(1)), + (b"TilingType".to_vec(), Object::Int(1)), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"XStep".to_vec(), Object::Int(500)), + (b"YStep".to_vec(), Object::Int(300)), + ]); + pattern.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Font".to_vec(), + Object::Dict(Dict::from_iter([(b"F1".to_vec(), Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))])), + )])), + ); + let width = if partial { 400 } else { 170 }; + let survivor = if retained { " 290 115 50 30 re f" } else { "" }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET /Pattern cs /P0 scn 35 110 {width} 35 re f{survivor}"); + let mut page_stream = Vec::new(); + pdfcraft_cos::serialize(&Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes())), &mut page_stream); + let mut pattern_stream = Vec::new(); + let stream = if compressed { Stream::flate(pattern, cell) } else { Stream::from_raw(pattern, cell.to_vec()) }; + pdfcraft_cos::serialize(&Object::Stream(stream), &mut pattern_stream); + let objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >> >>" + .to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + page_stream, + pattern_stream, + ]; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend(format!("{} 0 obj\n", index + 1).as_bytes()); + bytes.extend(object); + bytes.extend(b"\nendobj\n"); + } + let xref = bytes.len(); + bytes.extend(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes()); + for offset in offsets { + bytes.extend(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes()); + bytes +} + +fn saved_has_secret(bytes: Vec) -> bool { + let document = Document::open(Arc::new(bytes)).unwrap(); + document.object_numbers().into_iter().any(|num| { + let object = document.try_get(num).unwrap(); + matches!(&*object, Object::Stream(stream) if stream.decoded().unwrap().windows(10).any(|part| part == b"CELLSECRET")) + }) +} + +#[test] +fn pattern_redaction_tools_remove_or_refuse_without_leaking_or_changing_undo() { + let dir = tempfile::tempdir().unwrap(); + for (partial, retained) in [(false, false), (true, false), (true, true)] { + for compressed in [false, true] { + let cell = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET BT /F1 20 Tf 300 120 Td (SAFE) Tj ET"; + let bytes = fixture(cell, partial, compressed, retained); + assert!(saved_has_secret(bytes.clone())); + std::fs::write(dir.path().join("input.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "input.pdf"}))["doc"].as_u64().unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let before = automation.session().get(DocId(id)).unwrap().bytes.clone(); + let pixels = automation.render_pixels(id, 1, 1.0).unwrap(); + match automation.call("redact_apply", &json!({"doc": id})) { + Ok(_) => { + assert!(!retained, "a text-bearing cell still in use must not be reported safe (partial: {partial})"); + tool(&mut automation, "doc_save", json!({"doc": id, "path": "redacted.pdf"})); + assert!( + !saved_has_secret(std::fs::read(dir.path().join("redacted.pdf")).unwrap()), + "successful apply/save left the pattern-cell text recoverable" + ); + } + Err(error) => { + assert!(retained, "a pattern nothing surviving paints with should apply (partial: {partial}): {error}"); + assert!(error.to_string().contains("tiling pattern"), "{error}"); + assert!(!error.to_string().contains("CELLSECRET")); + let document = automation.session().get(DocId(id)).unwrap(); + assert_eq!(document.bytes, before); + assert_eq!(document.redaction_marks(), 1); + assert_eq!(document.can_undo(), Some("Add redaction mark")); + assert_eq!(automation.render_pixels(id, 1, 1.0).unwrap(), pixels); + tool(&mut automation, "edit_undo", json!({"doc": id})); + assert_eq!(automation.session().get(DocId(id)).unwrap().redaction_marks(), 0); + } + } + } + } +} + +#[test] +fn pattern_redaction_vector_control_saves_reopens_and_preserves_outside_pixels() { + let dir = tempfile::tempdir().unwrap(); + // The guard's control: the cell is plain vectors and a paint outside the mark still uses it, + // so it is inspected and kept, not refused, and what that paint draws is unchanged. + let bytes = fixture(b"0.2 0.4 0.6 rg 40 120 15 15 re f 300 120 30 20 re f", true, true, true); + std::fs::write(dir.path().join("vectors.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "vectors.pdf"}))["doc"].as_u64().unwrap(); + let (width, height, original) = automation.render_pixels(id, 1, 1.0).unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let applied = tool(&mut automation, "redact_apply", json!({"doc": id})); + assert_eq!(applied["applied"], 1); + let save = tool(&mut automation, "doc_save", json!({"doc": id, "path": "vectors-redacted.pdf"})); + assert_eq!(save["incremental"], false); + let reopened = tool(&mut automation, "doc_open", json!({"path": "vectors-redacted.pdf"}))["doc"].as_u64().unwrap(); + let (new_width, new_height, actual) = automation.render_pixels(reopened, 1, 1.0).unwrap(); + assert_eq!((new_width, new_height), (width, height)); + for (index, (before, after)) in original.as_chunks::<4>().0.iter().zip(actual.as_chunks::<4>().0).enumerate() { + let (x, y) = (index % width as usize, index / width as usize); + if x >= 280 { + assert_eq!(before, after, "unmarked pattern pixels changed at {x},{y}"); + } + if (35..205).contains(&x) && (155..190).contains(&y) { + assert_eq!(after, &[0, 0, 0, 255], "marked interior must be black"); + } + } +} diff --git a/crates/compare/src/lib.rs b/crates/compare/src/lib.rs index 91245cbc21..26a50eaf71 100644 --- a/crates/compare/src/lib.rs +++ b/crates/compare/src/lib.rs @@ -41,6 +41,8 @@ pub struct Side { /// The page the change is on (for a pure insertion or deletion, where the other side's /// words would have been). pub page: usize, + /// The page its last word is on: words changed across a page break are one change. + pub last_page: usize, pub rects: Vec<[f64; 4]>, /// For an empty side (the other side's words were inserted or deleted): the box of the /// word next to where they would be. @@ -245,6 +247,7 @@ fn side(words: &[&Word], neighbour: Option<&Word>) -> Side { Side { text: words.iter().map(|w| w.text.as_str()).collect::>().join(" "), page: words.first().or(neighbour.as_ref()).map_or(0, |w| w.page), + last_page: words.last().or(neighbour.as_ref()).map_or(0, |w| w.page), rects: rects(words), near: if words.is_empty() { neighbour.map(|w| w.rect) } else { None }, } @@ -349,6 +352,17 @@ mod tests { assert_eq!(c.count(Kind::Inserted), 2); } + #[test] + fn a_change_across_a_page_break_knows_both_pages() { + let mut old = words("alpha beta", 0); + old.extend(words("gamma delta", 1)); + let mut new = words("alpha one", 0); + new.extend(words("two delta", 1)); + let c = compare(&old, &new); + assert_eq!(c.changes.len(), 1); + assert_eq!((c.changes[0].new.page, c.changes[0].new.last_page), (0, 1)); + } + #[test] fn deletions_across_pages_keep_their_pages() { let mut old = words("alpha beta gamma", 0); diff --git a/crates/cos/Cargo.toml b/crates/cos/Cargo.toml index a2fb3f4c4e..0bf2e894d2 100644 --- a/crates/cos/Cargo.toml +++ b/crates/cos/Cargo.toml @@ -15,6 +15,13 @@ thiserror = { workspace = true } # PDF date strings (`pdf_date`) go through chrono (no hand-rolled calendar code). chrono = { workspace = true } +# On the web, a rewrite's fresh file identifier takes its entropy from the browser +# (`crypto.getRandomValues`). Both crates are already in the dependency tree of the web shell +# (`js-sys` 0.3 / `wasm-bindgen` 0.2); native builds compile neither. +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" +wasm-bindgen = "0.2" + [dev-dependencies] pdfcraft-oracle = { workspace = true } pdfcraft-testkit = { workspace = true } diff --git a/crates/cos/README.md b/crates/cos/README.md index 7b576f293f..9f5fb56725 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -53,7 +53,20 @@ another. Edits never touch the original bytes. - **Every cross-reference form:** classic tables with their quirky subsections, cross-reference streams, hybrid-reference files (`/XRefStm`), and object streams. - **Repair.** A file whose xref is wrong or missing is reconstructed by scanning for `obj` - headers. Repairs are *recorded*, never silent, so the UI can say the file was damaged. + headers. Repairs are *recorded*, never silent, so the UI can say the file was damaged. A live + in-file reference may match either its xref generation or a same-number object header at + that indexed offset; a missing, free, or neither-matching slot still resolves to null. + Header probes inspect at most 128 bytes at the indexed and header-relative offsets, without + parsing object bodies. On open, disagreements in retained xref revisions are recorded once + per object, up to 256 objects plus an omitted-details summary. This diagnostic scan does + not expand compact xref ranges; the winning xref entry remains authoritative for free + slots. Compressed objects require generation zero. Explicit `set` overlays accept only the + supplied generation; `set_existing` and `update_dict` retain both accepted generations of a + damaged live slot, including through snapshots, and require full rewrite. + `accepted_generations(num)` lists the generations a slot accepts, so code comparing two + revisions can tell whether a reference still reaches the same slot. Headers beyond + this bounded probe are not additionally recovered. Materialized primary and hybrid rows are + diagnosed before folding, so a newer row does not hide a retained disagreement. - **Tolerant, never fatal.** The lexer works on a byte slice and a position; it allocates nothing for skipped data, bounds-checks every index, and caps nesting depth. Malformed input yields `CosError`, never a panic — this crate is the first thing a hostile file meets. Exhausted xref entries are skipped with a repair log; exhausted /Size hints allocate unused numbers without overflowing or replacing existing objects. @@ -66,14 +79,25 @@ another. Edits never touch the original bytes. ## Writing -`write_incremental` appends only the edited objects, a cross-reference section **in the same -style as the file's last one**, and a trailer with `/Prev`. The original bytes are never modified, -which is what keeps a signature over an earlier revision valid. +`write_incremental` ordinarily appends only the edited objects, a cross-reference section **in +the same style as the file's last one**, and a trailer with `/Prev`, preserving the original +bytes and signatures over earlier revisions. It delegates to `write_full` when reconstruction, +encryption changes or `full_save_required()` demand a rewrite. Repair-aware edits to a live +xref/header generation disagreement use this fallback rather than append a contradictory row. `write_full` writes only what the trailer can reach, renumbered densely — the Save As and garbage-collecting path. By default (`SaveOptions::object_streams`) it packs non-stream objects into compressed object streams with a cross-reference stream (PDF 1.5+); turning that off writes a -classic table any reader accepts. +classic table any reader accepts. Both forms map accepted xref/header aliases to one output +object, retaining shared identity such as page-tree parent links; unrelated generations stay null. + +A full save keeps the file's `/ID` as it is, unless the document asked for a new identifier with +`Document::require_full_save_with_new_id` (redaction and sanitizing do). Then both strings are +regenerated, and `SaveOptions::id_entropy` can supply entropy for reproducible output (tests pin +output this way). Without it the identifier mixes in fresh entropy from the host — the browser's +`crypto.getRandomValues` on the web, the operating system's random hash seed on the desktop — so a +fresh copy of the program cannot reproduce it. An encrypted file keeps the first string, because its +key is derived from it. Dictionaries keep their key order and streams keep their *encoded* bytes, so rewritten objects stay recognisable in a diff and byte-stable when re-serialized. diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 376b108be3..65b750e7b3 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -25,6 +25,12 @@ pub const MAX_PAGE_DECODED: usize = 256 << 20; /// Form XObjects one page's content walk may enter, each entry counted again. pub const MAX_PAGE_FORMS: usize = 50_000; +/// The most pixels an image redaction decodes, to clear the ones under a mark or to store again +/// without the `/JBIG2Globals` it shares with a redacted image (`pdfcraft_redact`). Optimize has +/// JBIG2 images share a dictionary only within it (`pdfcraft_optimize`), so that redaction can +/// store each of them again: one it couldn't would refuse the redaction. +pub const MAX_REDACTED_IMAGE_PIXELS: u64 = 64 << 20; + /// The steps a sample of a JPEG costs: a few bytes can hold a page of one grey, and decoding, /// resampling and storing it took about 20 to 45 ns a sample. const JPEG_WORK: u64 = 2; diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 2aaf33ec63..9fc72599ed 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -19,12 +19,13 @@ mod reader; mod state; mod xref; +pub(crate) use objects::Held; #[cfg(test)] pub(crate) use reader::find; #[cfg(test)] use reader::rfind; pub(crate) use state::ObjectReader; -use state::{ObjStm, Slot}; +use state::{ObjStm, Slot, Unreadable}; pub use state::{ReadState, SourceIdentity}; #[cfg(test)] use state::{SECTION_READS, held_bytes, objstm_member}; @@ -47,8 +48,14 @@ pub struct Document { /// `fill_xref_gaps`. located: Arc>, repair_log: Arc>, + /// What loading each file object gave, a failure as well: a damaged object is otherwise + /// loaded again on every read, and a member of an object stream that can't be decoded + /// decodes the stream again (`Document::load`). cache: Arc>, objstms: Arc>>>, + /// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the + /// source, and is reset wherever they are. + unreadable: Arc>, /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). jbig2_globals: Arc>>>>, @@ -62,10 +69,17 @@ pub struct Document { security: Option>, /// Object number of the `/Encrypt` dictionary (never encrypted itself). encrypt_num: Option, + /// What that dictionary and `/ID` refer to, which the key comes from, and so is as stored. + encrypt_members: Arc>, + /// Object numbers a scan of the file didn't find: the file doesn't change, so it never will. + unfound: Arc>>, /// Encryption was added, changed or removed since opening: only a full save can apply it. encryption_changed: bool, /// Set by edits that must not leave earlier revisions in the file (redaction). full_save: bool, + /// The next full save also gives the file a new identifier (`/ID`), see + /// [`Document::require_full_save_with_new_id`]. + new_id: bool, /// The handler and `/Encrypt` object number that saves use, when encryption changed. out_security: Option>, out_encrypt_num: Option, diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 4efb185f1a..6eabb2c5b3 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,6 +1,10 @@ +use std::collections::HashSet; + use super::reader::find; -use super::state::{LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, held_bytes, objstm_member}; -use super::xref::{Revision, XrefEntry}; +use super::state::{ + LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, note_overlay_read, objstm_member, overlay_reads, +}; +use super::xref::{Revision, XrefEntry, XrefIndex}; use super::{ Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, }; @@ -86,6 +90,39 @@ impl Document { &self.repair_log } + /// Objects of the file that could not be read, lowest number first, as lines such as + /// "object 11 0 could not be read: syntax error at byte 512: ...". [`Self::get`] reads each + /// as null (ISO 32000-2 §7.3.10), so a document with one opens, but what it held is lost: + /// unlike [`Self::repair_log`], nothing was repaired. Lists the objects read so far, by this + /// document, its clones or a scan of it, that have not since been edited; past 100, a last + /// line says there were more. Opening again, authenticating or rebuilding the index starts + /// the list afresh, as it does the object cache. + pub fn unreadable_objects(&self) -> Vec { + let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() }; + unreadable.lines(|num| self.overlay.contains_key(&num)) + } + + /// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file + /// whose broken object is asked for again and again pays for it once. + fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { + // A poisoned lock only means the object goes unlisted; the read itself still fails. + let Ok(mut unreadable) = self.unreadable.lock() else { return }; + let hidden = |n: u32| self.overlay.contains_key(&n); + if unreadable.wants(num, hidden) { + unreadable.note(num, self.generation(num), &reason(), hidden); + } + } + + /// Why an object listed at `offset` could not be read: what parsing it there, without resolving + /// an indirect `/Length`, found. A best account, not the only one: the reader also tried + /// elsewhere. + fn why_unreadable(&self, offset: usize) -> String { + match parse_indirect_shared(&self.data, offset, &|_| None, &self.endstreams) { + Err(e) => e.to_string(), + Ok(_) => "no readable copy was found at its cross-reference offset or elsewhere in the file".into(), + } + } + /// Add a line to [`Self::repair_log`]: a leniency applied while reading that the user should /// know about (fidelity: never silent). pub fn note_repair(&mut self, line: String) { @@ -129,8 +166,8 @@ impl Document { pub fn xref_entry_is_exact(&self, num: u32) -> bool { match self.xref_entry(num) { None | Some(XrefEntry::Free { .. }) => true, - Some(XrefEntry::InFile { offset, .. }) => { - let at = |off: usize| matches!(parse_indirect_shared(&self.data, off, &|_| None, &self.endstreams), Ok((id, _)) if id.num == num); + Some(XrefEntry::InFile { offset, generation }) => { + let at = |off: usize| matches!(parse_indirect_shared(&self.data, off, &|_| None, &self.endstreams), Ok((id, _)) if id == ObjRef::new(num, generation)); // Offsets may count from a header that does not start the file. usize::try_from(offset).is_ok_and(|off| at(off) || (self.header_offset != 0 && at(off.saturating_add(self.header_offset)))) } @@ -151,19 +188,131 @@ impl Document { // ── object access ─────────────────────────────────────────────────────────────────────── - /// Fetch an object by reference. Missing objects are `Null` (ISO 32000-2 §7.3.10). + /// Fetch an object by reference. Undefined references are `Null` (ISO 32000-2 §7.3.10). + /// A live file slot also accepts its actual header generation when a damaged xref row + /// disagrees; the disagreement is recorded in [`Self::repair_log`]. Free slots never do. pub fn get(&self, r: ObjRef) -> Arc { - self.try_get(r.num).unwrap_or_else(|_| Arc::new(Object::Null)) + self.try_get_reference(r).unwrap_or_else(|_| Arc::new(Object::Null)) } - /// Follow a reference if `o` is one; otherwise return `o` itself. - pub fn resolve(&self, o: &Object) -> Arc { - match o { - Object::Ref(r) => self.get(*r), - other => Arc::new(other.clone()), + pub(super) fn reference_matches(&self, r: ObjRef) -> bool { + match self.overlay.get(&r.num) { + Some(Slot::Set(generation, _)) => r.generation == *generation, + Some(Slot::RepairedSet(generation, alias, _)) => r.generation == *generation || r.generation == *alias, + Some(Slot::Freed(_)) => false, + None => match self.entries.lookup(r.num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + r.generation == generation || self.indexed_header_reference(r.num, offset) == Some(r) + } + Some(XrefEntry::InStream { .. }) => r.generation == 0, + None | Some(XrefEntry::Free { .. }) => false, + }, } } + /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), + /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged + /// file slot (or a repair-aware edit of one) also the generation its object header carries. + /// Code comparing revisions (signature validation) uses it to tell whether the references + /// that reached an object in one revision still reach the same slot in another. + pub fn accepted_generations(&self, num: u32) -> Vec { + // Mirrors `reference_matches`, which stays allocation-free for every lookup. + let mut generations = match self.overlay.get(&num) { + Some(Slot::Set(generation, _)) => vec![*generation], + Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], + Some(Slot::Freed(_)) => Vec::new(), + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() + } + Some(XrefEntry::InStream { .. }) => vec![0], + None | Some(XrefEntry::Free { .. }) => Vec::new(), + }, + }; + generations.sort_unstable(); + generations.dedup(); + generations + } + + pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { + if !self.reference_matches(r) { + return Ok(Arc::new(Object::Null)); + } + self.try_get(r.num) + } + + /// A repair probe, not a whole-file search: only the indexed header (or the same offset + /// relative to a shifted PDF header) can establish this additional generation. A small + /// window bounds hostile whitespace/tokens; ordinary xref-matching reads are unchanged. + pub(super) fn indexed_header_reference(&self, num: u32, offset: u64) -> Option { + let at = |off: usize| { + let bytes = self.data.get(off..off.saturating_add(128).min(self.data.len()))?; + let mut lexer = Lexer::new(bytes, 0); + lexer.skip_ws(); + let number = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; + lexer.skip_ws(); + let generation = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; + (number == num && lexer.eat_keyword(b"obj")).then_some(ObjRef::new(number, generation)) + }; + let off = usize::try_from(offset).ok()?; + at(off).or_else(|| if self.header_offset == 0 { None } else { off.checked_add(self.header_offset).and_then(at) }) + } + + /// Add at most one bounded diagnostic per object. False stops diagnostic probing only; + /// parsers must still read the remaining xref rows needed to build their index. + pub(super) fn inspect_xref_generation_repair(&self, num: u32, entry: XrefEntry, recorded: &mut HashSet, log: &mut Vec) -> bool { + let XrefEntry::InFile { offset, generation } = entry else { return true }; + if recorded.contains(&num) { + return true; + } + let Some(header) = self.indexed_header_reference(num, offset) else { return true }; + if header.generation == generation { + return true; + } + if recorded.len() == 256 { + log.push("additional xref/header generation disagreements omitted after 256 objects".into()); + return false; + } + recorded.insert(num); + log.push(format!("object {num}: xref generation {generation} disagrees with object header generation {}", header.generation)); + true + } + + /// Diagnose once at open without expanding compact ranges or decoding an xref twice. + /// Materialized readers retain bounded notes while they read accepted sections, before + /// newer rows hide older ones. Discarded parse attempts/rebuilt indexes carry no notes. + pub(super) fn note_xref_generation_repairs(&self, log: &mut Vec) { + match &*self.entries { + XrefIndex::Materialized(_, notes) => log.extend(notes.iter().cloned()), + XrefIndex::Compact { revisions, .. } => { + let mut recorded = HashSet::new(); + for revision in revisions { + for section in std::iter::once(&revision.primary).chain(revision.hybrid.iter()) { + for num in section.numbers() { + if let Some(entry) = section.lookup(num, &self.data) + && !self.inspect_xref_generation_repair(num, entry, &mut recorded, log) + { + return; + } + } + } + } + } + } + } + + /// Follow a reference if `o` is one; otherwise return `o` itself. An object can be nothing but + /// a reference to another, which is followed too, up to [`MAX_REFERENCE_HOPS`] of them. + pub fn resolve(&self, o: &Object) -> Arc { + let Object::Ref(r) = o else { return Arc::new(o.clone()) }; + let mut held = self.get(*r); + for _ in 0..MAX_REFERENCE_HOPS { + let Object::Ref(r) = &*held else { break }; + held = self.get(*r); + } + held + } + /// `s` with the `/DecodeParms` its filters use made direct, so [`Stream::decoded`] reads them: /// an indirect dictionary resolved, and of an array (itself perhaps indirect), the entries at /// the filters' places (`/Crypt` counts, ISO 32000-2 §7.4.1) resolved, the rest dropped. Of @@ -218,7 +367,7 @@ impl Document { } /// `o`, or the object it refers to, without copying either. - fn held<'o>(&self, o: &'o Object) -> Held<'o> { + pub(crate) fn held<'o>(&self, o: &'o Object) -> Held<'o> { match o { Object::Ref(_) => Held::Resolved(self.resolve(o)), o => Held::Direct(o), @@ -383,26 +532,38 @@ impl Document { } pub fn try_get(&self, num: u32) -> Result, CosError> { - match self.overlay.get(&num) { - Some(Slot::Set(_, o)) => return Ok(o.clone()), - Some(Slot::Freed(_)) => return Ok(Arc::new(Object::Null)), - None => {} + if let Some(slot) = self.overlay.get(&num) { + note_overlay_read(); + return Ok(match slot { + Slot::Set(_, o) | Slot::RepairedSet(_, _, o) => o.clone(), + Slot::Freed(_) => Arc::new(Object::Null), + }); } - if let Some(o) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { - return Ok(o.clone()); + if let Some(loaded) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + return loaded.clone(); } // Loading can re-enter (indirect stream `/Length`, object streams); damaged files make // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let obj = self.load(num, 0)?; - let obj = Arc::new(obj); - // The editing cache is bounded like the scan reader's: re-parsing a file-backed view - // is cheap, and returned objects outlive eviction. - let owned = held_bytes(&obj, &self.data, 0); - self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone(), owned); - self.evict_editor_cache_if_over_budget(num)?; - Ok(obj) + let _scope = crate::security::FollowScope::enter(); + let edits_read = overlay_reads(); + let loaded = self.load(num, 0).map(Arc::new); + if let Err(e) = &loaded + && !matches!(e, CosError::Poisoned) + { + self.note_unreadable(num, || e.to_string()); + } + // A failure is kept like an object, or every read of a damaged object loads it again. A + // poisoned lock says nothing about the object. Clones share the cache, so a load that read + // this document's edits (an edited object stream or `/Length`) is not kept. + if overlay_reads() == edits_read && !matches!(loaded, Err(CosError::Poisoned)) { + // The editing cache is bounded like the scan reader's: re-parsing a file-backed view + // is cheap, and returned objects outlive eviction. + self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone(), loaded_bytes(&loaded, &self.data)); + self.evict_editor_cache_if_over_budget(num)?; + } + loaded } /// Bounds for the editing cache: roomier than the scan reader's (128 objects / 8 MB) @@ -514,11 +675,20 @@ impl Document { None | Some(XrefEntry::Free { .. }) => Ok(Object::Null), Some(XrefEntry::InFile { offset, .. }) => { let resolve = |r: ObjRef| { - if let Some(known) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { + if self.overlay.contains_key(&r.num) { + note_overlay_read(); + } + // A length resolved through this document's edits counts as an edit read + // on every reuse. + if let Some((known, edits)) = LENGTHS.with(|m| m.borrow().get(&r).copied()) { + if edits { + note_overlay_read(); + } return known; } - let length = self.try_get(r.num).ok().and_then(|o| o.as_int()); - LENGTHS.with(|m| m.borrow_mut().insert(r.num, length)); + let edits_read = overlay_reads(); + let length = self.try_get_reference(r).ok().and_then(|o| o.as_int()); + LENGTHS.with(|m| m.borrow_mut().insert(r, (length, overlay_reads() != edits_read))); length }; let off = offset as usize; @@ -531,7 +701,10 @@ impl Document { .or_else(|| if self.header_offset == 0 { None } else { at(off.saturating_add(self.header_offset)) }) .or_else(|| self.located.get(&num).and_then(|(o, _)| usize::try_from(*o).ok()).and_then(at)) .or_else(|| self.scan_for(num).map(|(id, o)| self.decrypted(id, o))) - .ok_or(CosError::MissingObject(num)) + .ok_or_else(|| { + self.note_unreadable(num, || self.why_unreadable(off)); + CosError::MissingObject(num) + }) } Some(XrefEntry::InStream { stream, index }) => { let stm = self.objstm(stream)?; @@ -544,7 +717,9 @@ impl Document { } pub(super) fn objstm(&self, num: u32) -> Result, CosError> { - if let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + // An edited or freed stream is this document's alone: clones share the parsed streams. + let edited = self.overlay.contains_key(&num); + if !edited && let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { return Ok(s.clone()); } // A file entry that cannot place the stream (one that says it is itself compressed @@ -552,6 +727,7 @@ impl Document { // header, as a plain object with a wrong offset is. An edited or freed stream, in the // overlay or in a saved revision's table, is what that says it is: the file's copy // must not come back. + let edits_read = overlay_reads(); let loaded = match self.try_get(num) { Ok(o) if matches!(*o, Object::Stream(_)) => o, _ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => { @@ -559,10 +735,18 @@ impl Document { } _ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?), }; + // A stream whose load read this document's edits (its `/Length`) is as much its own as an edited one. + let own = edited || overlay_reads() != edits_read; let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) }; let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize; let first = usize::try_from(s.dict.int(b"First").unwrap_or(0).max(0)).ok(); - let data = s.decoded_within(self.stream_limit)?; + // Its parameters can be given by reference, and can sit in another object stream. A scan of + // the file's object streams comes here without a read around it. + let _scope = crate::security::FollowScope::enter(); + let readable = crate::security::following(Cow::Borrowed(s), || self.with_direct_parms(s)); + #[cfg(test)] + super::tests::OBJSTM_DECODES.with(|decodes| decodes.set(decodes.get() + 1)); + let data = readable.decoded_within(self.stream_limit)?; let mut lx = Lexer::new(&data, 0); let mut index = Vec::with_capacity(n); for _ in 0..n { @@ -570,6 +754,9 @@ impl Document { index.push(objstm_member(onum, off, first, data.len())); } let stm = Arc::new(ObjStm { index, data: Arc::new(data) }); + if own { + return Ok(stm); + } // Same bound as the object cache: decoded object streams can be megabytes each. // Insert first so the retained `num` is actually in the map. self.objstms.lock().map_err(|_| CosError::Poisoned)?.insert(num, stm.clone()); @@ -577,8 +764,14 @@ impl Document { Ok(stm) } - /// Last-resort lookup: scan the file for `num G obj`. + /// Last-resort lookup: scan the file for `num G obj`. What isn't there is remembered: each miss + /// reads the whole file, and a file can refer to such an object from everywhere. fn scan_for(&self, num: u32) -> Option<(ObjRef, Object)> { + if self.unfound.lock().is_ok_and(|unfound| unfound.contains(&num)) { + return None; + } + #[cfg(test)] + super::tests::FILE_SCANS.with(|scans| scans.set(scans.get() + 1)); let needle = format!("{num} "); let data = &self.data; let mut found = None; @@ -594,6 +787,11 @@ impl Document { found = Some((id, o)); // keep the last (newest) definition } } + if found.is_none() + && let Ok(mut unfound) = self.unfound.lock() + { + unfound.insert(num); + } found } @@ -615,11 +813,44 @@ impl Document { pub fn set(&mut self, r: ObjRef, obj: impl Into) { let obj = self.adopt(obj.into()); self.overlay.insert(r.num, Slot::Set(r.generation, Arc::new(obj))); + self.forget_member_failures(r.num); if r.num >= self.next_num { self.next_num = r.num.saturating_add(1); } } + /// Replace an accepted existing reference without losing a damaged source slot's aliases. + /// Unlike [`Self::set`], this preserves both xref/header generations when they disagree. + /// Such an edit forces a full rewrite so the output has consistent references, rather than + /// appending a new contradictory xref row. Missing, free and stale references are errors. + pub fn set_existing(&mut self, r: ObjRef, obj: impl Into) -> Result<(), CosError> { + if !self.reference_matches(r) { + return Err(CosError::MissingObject(r.num)); + } + let alias = self.alternate_generation(r); + if let Some(alias) = alias { + let obj = self.adopt(obj.into()); + self.overlay.insert(r.num, Slot::RepairedSet(r.generation, alias, Arc::new(obj))); + } else { + self.set(r, obj); + } + Ok(()) + } + + /// Other accepted generation for a known-live reference to a damaged slot. + pub(super) fn alternate_generation(&self, r: ObjRef) -> Option { + match self.overlay.get(&r.num) { + Some(Slot::RepairedSet(generation, alias, _)) => Some(if r.generation == *generation { *alias } else { *generation }), + Some(_) => None, + None => match self.entries.lookup(r.num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => self.indexed_header_reference(r.num, offset).and_then(|header| { + (header.generation != generation).then_some(if r.generation == generation { header.generation } else { generation }) + }), + _ => None, + }, + } + } + /// Add a new indirect object and return its reference. pub fn add(&mut self, obj: impl Into) -> ObjRef { let r = ObjRef::new(self.allocate_num(), 0); @@ -642,6 +873,16 @@ impl Document { /// Delete an object (its number becomes free with the next generation). pub fn free(&mut self, r: ObjRef) { self.overlay.insert(r.num, Slot::Freed(r.generation.saturating_add(1))); + self.forget_member_failures(r.num); + } + + /// Forget cached failures of the members `stream` held in the file: they were failures of + /// the file's copy, and an edit of `stream` may read. Clones share the cache, so theirs are + /// searched again too, which costs time only. A poisoned lock leaves the cache as it is. + fn forget_member_failures(&self, stream: u32) { + if let Ok(mut cache) = self.cache.lock() { + cache.forget_failures(|n| matches!(self.entries.lookup(*n, &self.data), Some(XrefEntry::InStream { stream: s, .. }) if s == stream)); + } } /// Free what was added since the file was read and nothing refers to any more: the objects @@ -659,8 +900,9 @@ impl Document { /// /// Used when the working snapshot was restored from an earlier revision and the next /// incremental save must still append to this file (the last saved revision). Objects - /// that already match are left untouched. Reads go through private readers so neither - /// document's editing cache is filled. + /// that already match are left untouched, except members of an object stream of this file + /// whose stream number `current` frees or reuses: those are written again on their own. + /// Reads go through private readers so neither document's editing cache is filled. pub fn overlay_changes_from(&mut self, current: &Document) -> Result<(), CosError> { let saved_nums = self.object_numbers(); let current_nums = current.object_numbers(); @@ -678,12 +920,29 @@ impl Document { let generation = current.generation(num); let same = saved_nums.binary_search(&num).is_ok() && self.generation(num) == generation - && saved_reader.try_get(ObjRef::new(num, 0))? == current_reader.try_get(ObjRef::new(num, 0))?; + && self.alternate_generation(ObjRef::new(num, generation)) == current.alternate_generation(ObjRef::new(num, generation)) + && saved_reader.try_get(ObjRef::new(num, generation))? == current_reader.try_get(ObjRef::new(num, generation))?; if !same { changed.push(num); } } } + // An object stream of this file that is freed or replaced takes its members with it, so + // a member that is otherwise unchanged is written again on its own. + let gone = |n: u32| freed.binary_search(&n).is_ok() || changed.binary_search(&n).is_ok(); + let orphaned: Vec = current_nums + .iter() + .copied() + .filter(|&num| changed.binary_search(&num).is_err()) + .filter(|&num| { + let packed_in = match self.xref_entry(num) { + Some(XrefEntry::InStream { stream, .. }) => Some(stream), + _ => None, + }; + packed_in.is_some_and(gone) + }) + .collect(); + changed.extend(orphaned); for num in freed { let generation = self.generation(num); self.free(ObjRef::new(num, generation)); @@ -691,8 +950,14 @@ impl Document { let mut current_reader = current.object_reader(); for num in changed { let generation = current.generation(num); - let object = current_reader.try_get(ObjRef::new(num, 0))?; - self.set(ObjRef::new(num, generation), (*object).clone()); + let object = current_reader.try_get(ObjRef::new(num, generation))?; + if let Some(alias) = current.alternate_generation(ObjRef::new(num, generation)) { + let object = self.adopt((*object).clone()); + self.overlay.insert(num, Slot::RepairedSet(generation, alias, Arc::new(object))); + self.next_num = self.next_num.max(num.saturating_add(1)); + } else { + self.set(ObjRef::new(num, generation), (*object).clone()); + } } *self.trailer_mut() = current.trailer().clone(); Ok(()) @@ -703,14 +968,13 @@ impl Document { let mut obj = (*self.get(r)).clone(); let d = obj.as_dict_mut().ok_or(CosError::NotADictionary(r))?; f(d); - self.set(r, obj); - Ok(()) + self.set_existing(r, obj) } /// Current generation of an object number. pub fn generation(&self, num: u32) -> u16 { match self.overlay.get(&num) { - Some(Slot::Set(g, _)) => *g, + Some(Slot::Set(g, _) | Slot::RepairedSet(g, _, _)) => *g, Some(Slot::Freed(g)) => *g, None => match self.entries.lookup(num, &self.data) { Some(XrefEntry::InFile { generation, .. }) => generation, @@ -722,7 +986,7 @@ impl Document { pub(crate) fn overlay_entries(&self) -> impl Iterator>)> + '_ { self.overlay.iter().map(|(n, s)| match s { - Slot::Set(g, o) => (*n, *g, Some(o.clone())), + Slot::Set(g, o) | Slot::RepairedSet(g, _, o) => (*n, *g, Some(o.clone())), Slot::Freed(g) => (*n, *g, None), }) } @@ -745,7 +1009,7 @@ impl Document { if source.peek().is_some_and(|n| *n == num) { source.next(); } - if matches!(slot, Slot::Set(..)) { + if matches!(slot, Slot::Set(..) | Slot::RepairedSet(..)) { numbers.push(num); } } @@ -763,7 +1027,7 @@ impl Document { pub(super) fn root_is_catalog(&self) -> bool { let Some(r) = self.root() else { return false }; - matches!(self.try_get(r.num).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog") || d.contains(b"Pages")) + matches!(self.try_get_reference(r).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog") || d.contains(b"Pages")) } } @@ -786,9 +1050,13 @@ const DECODE_PARMS_KEYS: [&[u8]; 13] = [ b"ColorTransform", ]; +/// How many references in a row [`Document::resolve`] follows: enough for any file, and it ends a +/// cycle of them. +const MAX_REFERENCE_HOPS: usize = 32; + /// An object read in place, or the object a reference names as the document holds it: either way /// nothing is copied ([`Document::held`]). -enum Held<'o> { +pub(crate) enum Held<'o> { Direct(&'o Object), Resolved(Arc), } diff --git a/crates/cos/src/document/open.rs b/crates/cos/src/document/open.rs index 395424b771..b628cea89c 100644 --- a/crates/cos/src/document/open.rs +++ b/crates/cos/src/document/open.rs @@ -1,7 +1,11 @@ use super::reader::find; -use super::state::Credentials; +use super::state::{Credentials, Slot}; use super::xref::XrefIndex; -use super::{Arc, BTreeMap, CosError, Dict, Document, ObjRef, Object}; +use super::{Arc, BTreeMap, CosError, Dict, Document, HashSet, ObjRef, Object}; + +/// More objects than `/Encrypt` and `/ID` refer to in any file, so that one which refers to the page +/// tree, say, doesn't have a whole document read to open it. +const MAX_ENCRYPT_MEMBERS: usize = 64; impl Document { /// A new document with an empty page tree (for split / extract / combine). It has no @@ -19,6 +23,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: crate::object::MAX_DECODED, overlay: BTreeMap::new(), @@ -27,8 +32,11 @@ impl Document { version: "1.7".into(), security: None, encrypt_num: None, + encrypt_members: Arc::default(), + unfound: Arc::default(), encryption_changed: false, full_save: false, + new_id: false, out_security: None, out_encrypt_num: None, }; @@ -103,6 +111,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: stream_limit.min(crate::object::MAX_DECODED), overlay: BTreeMap::new(), @@ -111,8 +120,11 @@ impl Document { version, security: None, encrypt_num: None, + encrypt_members: Arc::default(), + unfound: Arc::default(), encryption_changed: false, full_save: false, + new_id: false, out_security: None, out_encrypt_num: None, }; @@ -148,6 +160,9 @@ impl Document { } doc.reconstruct(&mut log)?; } + // Header generations are plaintext, including in encrypted files. Inspect only the + // bounded headers, not object bodies, and retain the compact xref representation. + doc.note_xref_generation_repairs(&mut log); doc.repair_log = Arc::new(log); // Reconstruction may have found a different trailer: (re)authenticate against it. if (!ok || !authenticated) @@ -185,6 +200,8 @@ impl Document { }; let params = crate::security::encrypt_dict(self, &dict); let id0 = self.first_id(); + // Read here, while nothing is decrypted yet. + let members = self.encryption_members(enc); let handler = match credentials { Credentials::Password(password) => Arc::new(pdfcraft_crypt::SecurityHandler::open(params, &id0, *password).map_err(|e| match e { pdfcraft_crypt::CryptError::WrongPassword if password.is_none() => CosError::NeedsPassword, @@ -197,18 +214,52 @@ impl Document { }; self.security = Some(handler); self.encrypt_num = num; + self.encrypt_members = Arc::new(members); // Objects read while locating the catalog were read before decryption was possible. self.cache = Arc::default(); self.objstms = Arc::default(); + self.unreadable = Arc::default(); Ok(()) } /// The first file identifier, part of the encryption key (empty when there is none). fn first_id(&self) -> Vec { - match self.trailer.get(b"ID") { - Some(Object::Array(a)) => a.first().and_then(|s| s.as_string()).map(|s| s.bytes.clone()).unwrap_or_default(), - _ => Vec::new(), + let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); + let first = id.as_deref().and_then(Object::as_array).and_then(|a| a.first()).map(|s| self.resolve(s)); + first.as_deref().and_then(Object::as_string).map(|s| s.bytes.clone()).unwrap_or_default() + } + + /// The objects `/Encrypt`, given as `enc`, and `/ID` refer to, the ones they refer to, and so + /// on, up to [`MAX_ENCRYPT_MEMBERS`]. Their strings aren't encrypted (ISO 32000-2 §7.6.2): the + /// key is made from them, so they have to read as stored, and stay so. A number with no object + /// behind it, and a stream, which those dictionaries never hold, isn't one: an object added + /// under that number later, or a page's content a hostile `/Encrypt` names, is encrypted. + fn encryption_members(&self, enc: &Object) -> HashSet { + let mut pending = Vec::new(); + crate::writer::collect_refs(enc, &mut |r| pending.push(r)); + if let Some(id) = self.trailer.get(b"ID") { + crate::writer::collect_refs(id, &mut |r| pending.push(r)); } + let mut members = HashSet::new(); + while let Some(r) = pending.pop() { + if members.len() >= MAX_ENCRYPT_MEMBERS || members.contains(&r.num) { + continue; + } + let held = self.get(r); + if matches!(&*held, Object::Null | Object::Stream(_)) { + continue; + } + members.insert(r.num); + crate::writer::collect_refs(&held, &mut |x| pending.push(x)); + } + members + } + + /// Whether object `num` is written as it is held, being one of [`Document::encryption_members`], + /// unedited, and the save writing the same `/Encrypt`. What was edited or added is encrypted + /// whatever its number. + pub(crate) fn stored_as_is(&self, num: u32) -> bool { + !self.encryption_changed && !self.is_edited(num) && self.encrypt_members.contains(&num) } /// Lift the restrictions of a document opened with its user password by entering the @@ -244,9 +295,23 @@ impl Document { self.full_save = true; } + /// Like [`Document::require_full_save`], and the rewritten file also gets a new file identifier + /// (`/ID`), so that a redacted or sanitized copy never claims to be its source (ISO 32000-2 + /// §14.4). An encrypted file keeps the first string of its identifier, which its key depends on. + pub fn require_full_save_with_new_id(&mut self) { + self.full_save = true; + self.new_id = true; + } + /// The next save must rewrite the whole file (see [`Document::require_full_save`]). + /// Repair-aware edits retaining two accepted generations cannot use one incremental slot. pub fn full_save_required(&self) -> bool { - self.full_save + self.full_save || self.overlay.values().any(|slot| matches!(slot, Slot::RepairedSet(..))) + } + + /// The next full save gives the file a new identifier (see [`Document::require_full_save_with_new_id`]). + pub fn new_id_required(&self) -> bool { + self.new_id } /// Security was set or removed since the document was opened (the next save applies it). @@ -270,8 +335,9 @@ impl Document { /// always a full rewrite. Returns the handler (authenticated as owner). pub fn set_encryption(&mut self, params: &pdfcraft_crypt::NewEncryption) -> Result<(), CosError> { // The file identifier is part of the key; make sure it exists and keep it. - let id0 = match self.trailer.get(b"ID") { - Some(Object::Array(a)) if a.len() == 2 => a[0].as_string().map(|s| s.bytes.clone()).unwrap_or_default(), + let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); + let id0 = match id.as_deref() { + Some(Object::Array(a)) if a.len() == 2 => self.first_id(), _ => { let id = pdfcraft_crypt::file_id(¶ms.seed); let s = Object::String(crate::PdfString { bytes: id.clone(), hex: true }); @@ -338,7 +404,9 @@ impl Document { /// Decrypt a freshly parsed indirect object if the document is encrypted. pub(super) fn decrypted(&self, id: ObjRef, o: Object) -> Object { match &self.security { - Some(h) if Some(id.num) != self.encrypt_num => crate::security::transform(h, &o, id.num, id.generation, crate::security::Pass::Decrypt), + Some(h) if Some(id.num) != self.encrypt_num && !self.encrypt_members.contains(&id.num) => { + crate::security::transform(self, h, &o, id.num, id.generation, crate::security::Pass::Decrypt) + } _ => o, } } diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index d21bf9403f..2797a04e9a 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -1,5 +1,6 @@ #[cfg(test)] use super::state::SECTION_READS; +use super::state::Unreadable; use super::xref::{ MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData, }; @@ -17,8 +18,8 @@ impl Document { return Ok((index, trailer, revisions, Vec::new())); } } - let (entries, trailer, revisions, oddities) = self.read_xref_chain_materialized(log)?; - Ok((XrefIndex::Materialized(Arc::new(entries)), trailer, revisions, oddities)) + let (entries, trailer, revisions, oddities, generation_repairs) = self.read_xref_chain_materialized(log)?; + Ok((XrefIndex::Materialized(Arc::new(entries), Arc::new(generation_repairs)), trailer, revisions, oddities)) } fn read_compact_xref_chain(&self, log: &mut Vec) -> Result)>, CosError> { @@ -323,7 +324,7 @@ impl Document { pub(super) fn read_xref_chain_materialized( &self, log: &mut Vec, - ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>), CosError> { + ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { let data = &self.data; let tail_start = data.len().saturating_sub(4096); let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?; @@ -343,6 +344,9 @@ impl Document { // At most MAX_XREF_OBJECTS entries are kept in all; a stream past that is read again. let mut hybrids: HashMap>> = HashMap::new(); let (mut hybrid_rows, mut noted) = (0usize, HashSet::new()); + let mut generation_repairs = Vec::new(); + let mut recorded = HashSet::new(); + let mut diagnose = true; let mut next = Some(first); while let Some(off) = next.take() { if !visited.insert(off) || visited.len() > 10_000 { @@ -371,6 +375,12 @@ impl Document { } }; revisions.push(Revision { xref_offset: off, is_stream, header_relative }); + for (&num, &entry) in § { + if !diagnose { + break; + } + diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); + } // Hybrid files (§7.5.8.4): the /XRefStm entries supplement the table and replace the // free entries it gives objects only a cross-reference stream can locate. let hybrid = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0); @@ -390,6 +400,12 @@ impl Document { let mut stm = BTreeMap::new(); let read = match self.read_section(x as usize, &mut stm, &mut odd, log) { Ok(_) => { + for (&num, &entry) in &stm { + if !diagnose { + break; + } + diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); + } for (&num, &e) in &stm { match sect.get(&num) { None | Some(XrefEntry::Free { .. }) => { @@ -427,7 +443,7 @@ impl Document { for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { trailer.remove(k); } - Ok((entries, trailer, revisions, oddities)) + Ok((entries, trailer, revisions, oddities, generation_repairs)) } /// Read one section at `off`, adding entries not already known (newer sections win). `odd` @@ -606,7 +622,7 @@ impl Document { } if found > 0 { log.push(format!("the cross-reference data leaves out {found} objects the file contains; located them by their object headers")); - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(filled))); + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(filled), Arc::default())); } self.located = Arc::new(located); } @@ -631,8 +647,9 @@ impl Document { if entries.is_empty() { return Err(CosError::Syntax { offset: 0, detail: "no objects found".into() }); } - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()))); + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()), Arc::default())); self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Objects inside object streams. for num in objstms { if let Ok(o) = self.try_get(num) @@ -648,7 +665,9 @@ impl Document { } } let object_count = entries.len(); - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries))); + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries), Arc::default())); + // Failures from the scan stand: the members added change no scanned entry, and a + // `/Length` found only now cannot make a stream unreadable. self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a // usable /Root (last in the file, whichever kind), else the catalog itself. One whose @@ -658,7 +677,7 @@ impl Document { // A file whose only trailers are cross-reference streams (and whose `startxref` is gone // or points nowhere) keeps its /Info, /ID and /Encrypt this way. let rank = |at: usize, d: &Dict| { - let root = self.try_get(d.reference(b"Root")?.num).ok()?; + let root = self.try_get_reference(d.reference(b"Root")?).ok()?; let Object::Dict(c) = &*root else { return None }; Some((c.name(b"Type") == Some(b"Catalog") || c.contains(b"Pages"), at)) }; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 96dde01c0f..2f35174824 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -6,8 +6,20 @@ thread_local! { static LOADING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; /// Indirect `/Length` values resolved during the current outermost load: a damaged file /// retries a parse several ways, and each retry would resolve the same lengths again - /// (exponential in the chain of streams whose lengths point at further streams). - pub(super) static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); + /// (exponential in the chain of streams whose lengths point at further streams). Each is + /// kept with whether resolving it read unsaved edits. + pub(super) static LENGTHS: std::cell::RefCell, bool)>> = std::cell::RefCell::new(HashMap::new()); + /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). + static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Unsaved edits read so far on this thread: a load during which this changes depended on one. +pub(super) fn overlay_reads() -> u64 { + OVERLAY_READS.with(std::cell::Cell::get) +} + +pub(super) fn note_overlay_read() { + OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1))); } /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream @@ -30,6 +42,14 @@ impl LoadGuard { } } +/// What loading a file object gave, kept in the document's cache either way. +pub(super) type Loaded = Result, CosError>; + +/// The bytes a load result holds of its own ([`held_bytes`]); a failure holds none. +pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc>) -> usize { + loaded.as_ref().map_or(0, |o| held_bytes(o, data, 0)) +} + impl Drop for LoadGuard { fn drop(&mut self) { let outermost = LOADING.with(|l| { @@ -61,10 +81,67 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: us (start < len).then_some((num, start)) } +/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). +/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that +/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole +/// list holds at most `MAX` plus the objects edited in clones. +#[derive(Debug, Default)] +pub(super) struct Unreadable { + pub(super) objects: BTreeMap, + pub(super) more: bool, +} + +impl Unreadable { + pub(super) const MAX: usize = 100; + /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. + const REASON: usize = 160; + + /// Whether `num` is to be recorded: not listed yet and there is room among the entries the + /// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]), + /// so a clone's edits never use up its own allowance. A full list notes that there were more, + /// so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool { + if self.objects.contains_key(&num) { + return false; + } + if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX { + self.more = true; + return false; + } + true + } + + /// Record `num` once; the first reason given is kept. + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) { + if self.wants(num, hidden) { + self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); + } + } + + /// One line per recorded object not `hidden`, then one saying there were more, if there were. + pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { + // Clones share the list and each hides its own edits, so the list may hold more than + // `MAX` entries in all; each document still shows at most `MAX`. + let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num)); + let mut lines: Vec = shown + .by_ref() + .take(Self::MAX) + .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) + .collect(); + if self.more || shown.next().is_some() { + lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); + } + lines + } +} + /// Where an overlay slot came from. #[derive(Clone, Debug, PartialEq)] pub(super) enum Slot { Set(u16, Arc), + /// Both generations accepted from a damaged live source slot survive a repair-aware edit. + /// A full rewrite is required: a new xref slot cannot encode two generations. + RepairedSet(u16, u16, Arc), Freed(u16), } @@ -103,6 +180,7 @@ impl ReadState { a == b && match (x, y) { (Slot::Set(g, o), Slot::Set(h, p)) => g == h && Arc::ptr_eq(o, p), + (Slot::RepairedSet(g, a, o), Slot::RepairedSet(h, b, p)) => g == h && a == b && Arc::ptr_eq(o, p), (Slot::Freed(g), Slot::Freed(h)) => g == h, _ => false, } @@ -133,7 +211,7 @@ impl ObjectReader { } pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { - let object = self.document.try_get(reference.num); + let object = self.document.try_get_reference(reference); // A missed eviction here only costs a re-parse, so never fail the read on a poisoned // lock. The editing-cache evictors propagate instead; they guard user edits. if let Ok(mut streams) = self.document.objstms.lock() { @@ -178,7 +256,7 @@ thread_local! { pub(super) static SECTION_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// Parsed objects by number, each with what it holds ([`held_bytes`]) measured once, when it goes +/// What loading each object gave (a failure as well), by number, each with what it holds ([`held_bytes`]) measured once, when it goes /// in, and their sum kept beside them. /// /// Both caches are bounded by those bytes. Summing them on every read walked every cached object @@ -187,12 +265,12 @@ thread_local! { /// (#5873). #[derive(Default)] pub(crate) struct ObjectCache { - entries: HashMap, usize)>, + entries: HashMap, owned: usize, } impl ObjectCache { - pub(crate) fn get(&self, num: &u32) -> Option<&Arc> { + pub(crate) fn get(&self, num: &u32) -> Option<&Loaded> { self.entries.get(num).map(|(object, _)| object) } @@ -201,8 +279,8 @@ impl ObjectCache { self.entries.contains_key(num) } - /// Keep `object` as `num`, which holds `owned` bytes (what [`held_bytes`] says of it). - pub(crate) fn insert(&mut self, num: u32, object: Arc, owned: usize) { + /// Keep what loading `num` gave, which holds `owned` bytes ([`loaded_bytes`]). + pub(crate) fn insert(&mut self, num: u32, object: Loaded, owned: usize) { if let Some((_, before)) = self.entries.insert(num, (object, owned)) { self.owned = self.owned.saturating_sub(before); } @@ -228,6 +306,11 @@ impl ObjectCache { self.owned = 0; } + /// Drop the cached failures `stale` picks. A failure holds no bytes, so the total stands. + pub(crate) fn forget_failures(&mut self, mut stale: impl FnMut(&u32) -> bool) { + self.entries.retain(|num, (loaded, _)| loaded.is_ok() || !stale(num)); + } + pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) { self.entries.retain(|num, _| keep(num)); self.owned = self.entries.values().fold(0usize, |n, (_, owned)| n.saturating_add(*owned)); @@ -235,7 +318,7 @@ impl ObjectCache { #[cfg(test)] pub(crate) fn values(&self) -> impl Iterator> { - self.entries.values().map(|(object, _)| object) + self.entries.values().filter_map(|(object, _)| object.as_ref().ok()) } } diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 4c7b3eb652..2cab25e7f1 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1,5 +1,149 @@ use super::*; +std::thread_local! { + /// How many times `Document::scan_for` read the file on this thread. + pub(super) static FILE_SCANS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// How many times `Document::objstm` decoded an object stream on this thread. + pub(super) static OBJSTM_DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// A scan that doesn't find an object isn't made again: each reads the whole file, and a file can +/// refer to such an object from every object it has. +#[test] +fn what_a_scan_of_the_file_did_not_find_is_remembered() { + let body = b"%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n"; + let mut out = body.to_vec(); + let xref = out.len(); + out.extend_from_slice(b"xref\n0 4\n0000000000 65535 f \n"); + out.extend_from_slice(format!("{:010} 00000 n \n", 9).as_bytes()); + out.extend_from_slice(format!("{:010} 00000 n \n", body.iter().position(|b| *b == b'\n').unwrap() + 49).as_bytes()); + out.extend_from_slice(b"0000000001 00000 n \n"); + out.extend_from_slice(format!("trailer << /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + FILE_SCANS.with(|scans| scans.set(0)); + for _ in 0..5 { + assert!(doc.try_get(3).is_err()); + } + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1); + let clone = doc.clone(); + assert!(clone.try_get(3).is_err()); + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "a clone has the same file"); +} + +/// An object can be nothing but a reference to another. A chain of them is followed, and one that +/// goes round ends in what isn't a dictionary or a name, without hanging. +#[test] +fn references_to_references_are_followed() { + let mut doc = Document::new_empty(); + let name = doc.add(Object::name("Target")); + let one = doc.add(Object::Ref(name)); + let two = doc.add(Object::Ref(one)); + assert_eq!(doc.resolve(&Object::Ref(two)).as_name(), Some(b"Target".as_slice())); + let a = doc.add(Object::Null); + let b = doc.add(Object::Ref(a)); + doc.set(a, Object::Ref(b)); + assert_eq!(doc.resolve(&Object::Ref(a)).as_name(), None); +} + +/// An object stream's `/DecodeParms` can be given by reference: without them a predictor isn't +/// undone, so its members were lost. +#[test] +fn an_object_stream_with_parameters_by_reference_is_read() { + let members = b"3 0 << /Hidden (yes) >>"; + // PNG Sub, one row. + let mut rows = vec![1u8]; + let mut before = 0u8; + for b in members { + rows.push(b.wrapping_sub(before)); + before = *b; + } + let coded = pdfcraft_filters::encode_flate(&rows); + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 7]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /DecodeParms 6 0 R /Length {} >>\nstream\n", coded.len()).into_bytes(); + stm.extend_from_slice(&coded); + stm.extend_from_slice(b"\nendstream"); + obj(&mut out, 4, &stm); + let data = [2u8, 0, 4, 0]; + let mut xs = format!("<< /Type /XRef /Size 7 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); + xs.extend_from_slice(&data); + xs.extend_from_slice(b"\nendstream"); + obj(&mut out, 5, &xs); + obj(&mut out, 6, format!("<< /Predictor 11 /Columns {} >>", members.len()).as_bytes()); + let table = out.len(); + out.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); + for (n, off) in offs.iter().enumerate().skip(1) { + let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(format!("trailer << /Size 7 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + let hidden = doc.get(ObjRef::new(3, 0)); + assert_eq!(hidden.as_dict().and_then(|d| d.get(b"Hidden")).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"yes".to_vec())); +} + +/// Each entry of an `/Encrypt` dictionary, and of a crypt filter in it, reads the same by reference. +/// Read as absent, `/StmF` or `/CFM` was `/Identity`, so a whole file's streams stayed ciphertext. +#[test] +fn encrypt_dictionary_entries_read_the_same_by_reference() { + let string = |s: &str| Object::String(crate::PdfString::text(s)); + let mut filter = Dict::new(); + filter.set(b"CFM".to_vec(), Object::name("AESV2")); + filter.set(b"Length".to_vec(), Object::Int(5)); + let cf = |filter: Dict| { + let mut cf = Dict::new(); + cf.set(b"StdCF".to_vec(), Object::Dict(filter)); + cf.set(b"Other".to_vec(), Object::Dict(Dict::new())); + Object::Dict(cf) + }; + let mut d = Dict::new(); + let entries = [ + ("Filter", Object::name("Standard")), + ("V", Object::Int(4)), + ("R", Object::Int(4)), + ("Length", Object::Int(128)), + ("P", Object::Int(-44)), + ("O", string("owner")), + ("U", string("user")), + ("EncryptMetadata", Object::Bool(false)), + ("CF", cf(filter.clone())), + ("StmF", Object::name("StdCF")), + ("StrF", Object::name("Other")), + ("EFF", Object::name("StdCF")), + ]; + for (key, value) in &entries { + d.set(key.as_bytes().to_vec(), value.clone()); + } + let mut doc = Document::new_empty(); + let direct = crate::security::encrypt_dict(&doc, &d); + assert_eq!((direct.stm_f.as_slice(), direct.str_f.as_slice(), direct.ef_f.as_slice()), (&b"StdCF"[..], &b"Other"[..], &b"StdCF"[..])); + assert_eq!(direct.length_bits, 40, "the filter's own length, in bytes"); + assert!(!direct.encrypt_metadata); + assert_eq!(direct.crypt_filters.iter().filter(|(_, m)| *m == pdfcraft_crypt::Method::Aes128).count(), 1); + for (key, value) in &entries { + let mut one = d.clone(); + let r = doc.add(value.clone()); + one.set(key.as_bytes().to_vec(), Object::Ref(r)); + assert_eq!(crate::security::encrypt_dict(&doc, &one), direct, "/{key}"); + } + for key in ["CFM", "Length"] { + let mut by_reference = filter.clone(); + let r = doc.add(filter.get(key.as_bytes()).unwrap().clone()); + by_reference.set(key.as_bytes().to_vec(), Object::Ref(r)); + let mut one = d.clone(); + one.set(b"CF".to_vec(), cf(by_reference)); + assert_eq!(crate::security::encrypt_dict(&doc, &one), direct, "/{key} in the filter"); + } +} + /// /DecodeParms given by reference, whole, as array entries or as the numbers inside, are read /// once made direct: here a Flate predictor, which the stream alone doesn't see. Only the entries the filters use are /// resolved, and nothing is for a chain longer than a decoder takes. @@ -457,7 +601,7 @@ pub(crate) fn build(bodies: &[&str], trailer: &str) -> Vec { fn assert_compact_matches_materialized(doc: &Document) { assert!(matches!(doc.entries.as_ref(), XrefIndex::Compact { .. }), "valid xrefs should not retain a per-entry map"); - let (entries, _, revisions, _) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); + let (entries, _, revisions, _, _) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); assert_eq!(revisions.as_slice(), doc.revisions()); let max = entries.keys().next_back().copied().unwrap_or(0); assert_eq!(doc.entries.max_num(), max); @@ -545,7 +689,7 @@ fn malformed_size_uses_the_materialized_compatibility_path() { let at = bytes.windows(marker.len()).position(|window| window == marker).unwrap(); bytes[at..at + marker.len()].copy_from_slice(b"/Size 2"); let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(matches!(doc.entries.as_ref(), XrefIndex::Materialized(_))); + assert!(matches!(doc.entries.as_ref(), XrefIndex::Materialized(..))); assert_eq!(doc.next_num, 4); assert_eq!(doc.object_numbers(), vec![1, 2, 3]); } @@ -588,6 +732,163 @@ fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() { assert!(doc.xref_entry_is_exact(1) && doc.xref_entry_is_exact(2)); } +/// An object that can't be read at its entry or anywhere else is searched for in the whole file +/// once: every later read, in a clone too, gives the same error without searching again. An +/// edit of that number still answers for it. +#[test] +fn an_unreadable_object_is_searched_for_once() { + let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"]; + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap(); + FILE_SCANS.with(|scans| scans.set(0)); + let first = doc.try_get(3); + assert!(first.is_err(), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + } + let mut edited = doc.clone(); + assert_eq!(edited.try_get(3), first); + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "searches for an object that isn't there"); + let report = doc.unreadable_objects(); + assert_eq!(report.len(), 1, "cached failures must stay listed exactly once: {report:?}"); + assert_eq!(edited.unreadable_objects(), report, "clones retain the cached failure diagnostic"); + edited.set(ObjRef::new(3, 0), Object::Int(7)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Int(7))); + assert_eq!(doc.try_get(3), first, "the edit is the clone's alone"); + edited.free(ObjRef::new(3, 0)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); +} + +/// A member of an object stream that can't be decoded (here, one that decodes past the document's +/// stream limit) is decoded once: every later read, in a clone too, gives the same error without +/// decoding the stream again. The stream itself loads, so no search of the file is involved. +#[test] +fn a_member_of_an_undecodable_object_stream_is_decoded_once() { + let file = object_stream_file("3 0 ", &format!("<< /Hidden ({}) >>", "x".repeat(4096)), 0); + let doc = Document::open_with_stream_limit(Arc::new(file), None, 1024).unwrap(); + OBJSTM_DECODES.with(|decodes| decodes.set(0)); + let first = doc.try_get(3); + assert!(matches!(first, Err(CosError::Filter(_))), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + } + assert_eq!(doc.clone().try_get(3), first); + assert_eq!(OBJSTM_DECODES.with(std::cell::Cell::get), 1, "decodes an object stream that can't be decoded"); +} + +/// Clones share the cache, and an edit is one clone's alone: a member of an object stream that +/// one clone freed or replaced still reads, in the others, from the file's stream. +#[test] +fn an_edited_object_stream_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); + let doc = Document::open(file.clone()).unwrap(); + let mut freed = doc.clone(); + freed.free(ObjRef::new(4, 0)); + assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4))); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the free reached another clone"); + let doc = Document::open(file).unwrap(); + let mut replaced = doc.clone(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + replaced.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec())); + assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no")); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); +} + +/// A member that failed to load from the file's object stream is kept as a failure, but replacing +/// or freeing that stream forgets it: the member then reads from the edit. +#[test] +fn an_edited_object_stream_forgets_its_members_failures() { + let file = Arc::new(object_stream_file("3 0 ", ") broken", 0)); + let mut doc = Document::open(file.clone()).unwrap(); + assert!(doc.try_get(3).is_err(), "the file's member does not parse"); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + doc.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (fixed) >>".to_vec())); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("fixed"), "the cached failure outlived the edit"); + assert!(doc.try_get(3).is_ok()); + let mut doc = Document::open(file).unwrap(); + assert!(matches!(doc.try_get(3), Err(CosError::Syntax { .. })), "the file's member does not parse"); + doc.free(ObjRef::new(4, 0)); + assert_eq!(doc.try_get(3), Err(CosError::MissingObject(4)), "the freed stream's member"); +} + +/// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: +/// the others still read its members within the file's length. +#[test] +fn an_edited_object_stream_length_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); + let doc = Document::open(file).unwrap(); + let mut edited = doc.clone(); + // Ends the stream at the `endstream` inside the member's string. + let cut = "3 0 << /Hidden (cut".len(); + edited.set(ObjRef::new(6, 0), Object::Int(cut as i64)); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + +/// A `/Length` read through one clone's edits and reused within the same load still keeps that +/// load to the clone. Here the file's offsets are header-relative behind junk, so object 10's +/// offset first lands on object 12, whose length (object 8, in the edited object stream 9) is +/// read; object 10's own length then loads object stream 4, framed by object 8 too. +#[test] +fn a_reused_edited_length_leaves_other_clones_members_alone() { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 13]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let stream = |dict: &str, data: &str| format!("<< {dict} >>\nstream\n{data}\nendstream").into_bytes(); + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let inner = "11 0 3 2 5 << /Hidden (cut endstream here) >>"; + obj(&mut out, 4, &stream("/Type /ObjStm /N 2 /First 9 /Length 8 0 R", inner)); + let lengths = format!("8 0 {}", inner.len()); + obj(&mut out, 9, &stream(&format!("/Type /ObjStm /N 1 /First 4 /Length {}", lengths.len()), &lengths)); + obj(&mut out, 12, &stream("/Length 8 0 R", &"x".repeat(40))); + obj(&mut out, 10, &stream("/Length 11 0 R", "PDATA")); + // A cross-reference stream: a hybrid file's /XRefStm is not looked for behind junk. + let compressed = [(3, 4, 1), (8, 9, 0), (11, 4, 0)]; + let mut data = Vec::new(); + for (n, off) in offs.iter().enumerate() { + let row = match compressed.iter().find(|c| c.0 == n) { + Some(&(_, stream, index)) => [2, 0, stream, index], + None if *off == 0 => [0, 0, 0, 0], + None => { + let [.., hi, lo] = u32::try_from(*off).unwrap().to_be_bytes(); + [1, hi, lo, 0] + } + }; + data.extend_from_slice(&row); + } + let table = out.len(); + let dict = format!("<< /Type /XRef /Size 14 /W [1 2 1] /Index [0 13] /Root 1 0 R /Length {} >>", data.len()); + out.extend_from_slice(format!("13 0 obj\n{dict}\nstream\n").as_bytes()); + out.extend_from_slice(&data); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{table}\n%%EOF\n").as_bytes()); + let mut file = vec![b'J'; offs[10] - offs[12] - 1]; + file.push(b'\n'); + file.extend_from_slice(&out); + let doc = Document::open(Arc::new(file)).unwrap(); + let mut edited = doc.clone(); + let cut = "11 0 3 2 5 << /Hidden (cut".len(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + edited.set(ObjRef::new(9, 0), crate::Stream::from_raw(dict, format!("8 0 {cut}").into_bytes())); + assert!(matches!(&*edited.get(ObjRef::new(10, 0)), Object::Stream(s) if &s.raw[..] == b"PDATA")); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { @@ -1039,15 +1340,18 @@ fn reads_classic_revision_and_edits_are_copy_on_write() { fn object_numbers_merge_source_and_overlay_in_order() { let mut doc = Document::new_empty(); doc.overlay.clear(); - doc.entries = Arc::new(XrefIndex::Materialized(Arc::new(BTreeMap::from([ - (0, XrefEntry::Free { next_generation: 65535 }), - (2, XrefEntry::InFile { offset: 10, generation: 7 }), - (4, XrefEntry::Free { next_generation: 9 }), - (6, XrefEntry::InStream { stream: 80, index: 1 }), - (8, XrefEntry::InFile { offset: 20, generation: 5 }), - (10, XrefEntry::InFile { offset: 30, generation: 3 }), - (u32::MAX, XrefEntry::InFile { offset: 40, generation: 1 }), - ])))); + doc.entries = Arc::new(XrefIndex::Materialized( + Arc::new(BTreeMap::from([ + (0, XrefEntry::Free { next_generation: 65535 }), + (2, XrefEntry::InFile { offset: 10, generation: 7 }), + (4, XrefEntry::Free { next_generation: 9 }), + (6, XrefEntry::InStream { stream: 80, index: 1 }), + (8, XrefEntry::InFile { offset: 20, generation: 5 }), + (10, XrefEntry::InFile { offset: 30, generation: 3 }), + (u32::MAX, XrefEntry::InFile { offset: 40, generation: 1 }), + ])), + Arc::default(), + )); doc.set(ObjRef::new(1, 2), Object::Int(1)); // before the first live source slot doc.set(ObjRef::new(2, 8), Object::Int(2)); // override, without duplication doc.set(ObjRef::new(4, 9), Object::Int(4)); // reuse a source free slot @@ -1110,7 +1414,7 @@ fn temporary_reader_keeps_file_views_and_evicts_data_of_its_own() { // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); let size = held_bytes(&owned, &reader.document.data, 0); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned), size); + reader.document.cache.lock().unwrap().insert(3, Ok(Arc::new(owned)), size); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } @@ -1323,8 +1627,14 @@ fn hybrid_reference_files_read_the_xref_stream_too() { /// stream puts it at position `member` of object stream 4, which holds `header` (the number /// and offset pairs) and then `members`. fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { + object_stream_file_with(header, members, member, false) +} + +/// [`object_stream_file`], with the stream's `/Length` in object 6 when `indirect_length`. +fn object_stream_file_with(header: &str, members: &str, member: u8, indirect_length: bool) -> Vec { let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 6]; + let size = if indirect_length { 7 } else { 6 }; + let mut offs = vec![0usize; size]; let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { offs[n] = out.len(); out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); @@ -1335,7 +1645,8 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); let count = header.split_whitespace().count() / 2; let inner = format!("{header}{members}"); - let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {} >>\nstream\n", header.len(), inner.len()).into_bytes(); + let length = if indirect_length { "6 0 R".to_string() } else { inner.len().to_string() }; + let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {length} >>\nstream\n", header.len()).into_bytes(); stm.extend_from_slice(inner.as_bytes()); stm.extend_from_slice(b"\nendstream"); obj(&mut out, 4, &stm); @@ -1344,13 +1655,16 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { xs.extend_from_slice(&data); xs.extend_from_slice(b"\nendstream"); obj(&mut out, 5, &xs); + if indirect_length { + obj(&mut out, 6, inner.len().to_string().as_bytes()); + } let table = out.len(); - out.extend_from_slice(b"xref\n0 6\n0000000000 65535 f \n"); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); for (n, off) in offs.iter().enumerate().skip(1) { let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; out.extend_from_slice(line.as_bytes()); } - out.extend_from_slice(format!("trailer << /Size 6 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); out } @@ -1704,6 +2018,131 @@ fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { rx.recv_timeout(std::time::Duration::from_secs(20)).expect("loading must not retry exponentially"); } +/// A file whose object 3 (its `/Info`) can't be parsed: the rest reads. +fn with_unreadable_info() -> Vec { + build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Author ) >>"], + "/Root 1 0 R /Info 3 0 R", + ) +} + +/// An object that can't be parsed reads as null and is listed once, by number and generation, +/// with why. Clones and scans share the list, an edit takes the object off it (in that document +/// only), and a sound file lists nothing. +#[test] +fn unreadable_objects_are_listed_with_the_reason() { + let doc = Document::open(Arc::new(with_unreadable_info())).unwrap(); + assert!(doc.unreadable_objects().is_empty(), "nothing has read object 3 yet"); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert!(doc.try_get(3).is_err()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "read twice, listed once: {lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); + let scanned = doc.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let unread = Document::open(Arc::new(with_unreadable_info())).unwrap(); + let scanned = unread.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(unread.unreadable_objects(), lines, "a clone's scan reaches the original"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); + assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); + assert_eq!(doc.unreadable_objects(), lines); + let sound = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) >>"], "/Root 1 0 R /Info 3 0 R"); + let sound = Document::open(Arc::new(sound)).unwrap(); + assert_eq!(sound.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 0); + assert!(sound.unreadable_objects().is_empty()); +} + +/// A file rebuilt from its object headers still lists an object the scan could not read. +#[test] +fn a_rebuilt_file_lists_the_objects_its_scan_could_not_read() { + let text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n3 0 obj\n<< /A ) >>\nendobj\n%%EOF\n"; + let doc = Document::open(Arc::new(text.as_bytes().to_vec())).unwrap(); + assert!(doc.repair_log().iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{:?}", doc.repair_log()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); +} + +/// The list keeps at most 100 objects and then says there were more; an object whose +/// cross-reference offset holds no object is listed with that as its reason. +#[test] +fn the_unreadable_object_list_is_bounded() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 120)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 120); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 101, "{:?}", lines.last()); + assert!(lines[0].starts_with("object 3 0 could not be read"), "{}", lines[0]); + assert_eq!(lines[100], "more objects could not be read; only the first 100 are listed"); + + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let text = String::from_utf8_lossy(&base).into_owned(); + let gone = text.replacen("3 0 obj\n(three)", "9 0 obj\n(three)", 1); + let doc = Document::open(Arc::new(gone.into_bytes())).unwrap(); + assert!(doc.try_get(3).is_err()); + assert_eq!( + doc.unreadable_objects(), + ["object 3 0 could not be read: no readable copy was found at its cross-reference offset or elsewhere in the file"] + ); +} + +/// The unreadable list is shared by clones, and each hides what it has edited. A clone that edited +/// every listed object still lists its own failures: hidden entries don't use up its allowance. +#[test] +fn objects_a_clone_edited_do_not_use_up_its_unreadable_allowance() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 110)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + for num in 3..103 { + assert!(doc.try_get(num).is_err()); + } + assert_eq!(doc.unreadable_objects().len(), 100); + let mut edited = doc.clone(); + for num in 3..103 { + edited.set(ObjRef::new(num, 0), Object::Null); + } + for num in 103..113 { + assert!(edited.try_get(num).is_err()); + } + let lines = edited.unreadable_objects(); + assert_eq!(lines.len(), 10, "{lines:?}"); + assert!(lines[0].starts_with("object 103 0 could not be read"), "{}", lines[0]); + let original = doc.unreadable_objects(); + assert_eq!(original.len(), 101, "the original still shows at most 100, then says there were more"); + assert_eq!(original[100], "more objects could not be read; only the first 100 are listed"); +} + +#[test] +fn a_restored_revision_keeps_what_the_saved_file_held_in_object_streams() { + // The earlier revision: an empty document read back from a file whose catalog and pages + // sit in an object stream. + let earlier = Document::open(Arc::new(crate::write_full(&Document::new_empty(), &crate::SaveOptions::default()).unwrap())).unwrap(); + // The saved revision adds an object the catalog links to, so a full save numbers its objects + // anew; an undo then returns to the earlier graph. + let mut later = earlier.clone(); + let extra = later.add(Object::Int(7)); + let root = later.root().unwrap(); + later.update_dict(root, |c| c.set(b"Extra".to_vec(), Object::Ref(extra))).unwrap(); + let saved = crate::write_full(&later, &crate::SaveOptions::default()).unwrap(); + let mut base = Document::open(Arc::new(saved)).unwrap(); + assert!( + base.object_numbers().iter().any(|&n| matches!(base.xref_entry(n), Some(XrefEntry::InStream { .. }))), + "the saved file holds objects in a stream" + ); + base.overlay_changes_from(&earlier).unwrap(); + let file = crate::write_incremental(&base, &crate::SaveOptions::default()).unwrap(); + let reread = Document::open(Arc::new(file)).unwrap(); + let catalog = reread.get(reread.root().unwrap()).as_dict().unwrap().clone(); + assert!(catalog.get(b"Extra").is_none(), "the restored catalog"); + let pages = reread.resolve(catalog.get(b"Pages").unwrap()); + assert_eq!(pages.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Pages"[..]), "the pages survive: {pages:?}"); +} + /// The cache keeps the sum of what its objects hold as they come and go, so neither reader has /// to measure the cache again on every read (#5873). #[test] @@ -1713,10 +2152,10 @@ fn the_object_cache_keeps_its_total_as_objects_come_and_go() { let two = Arc::new(Object::Array(vec![Object::Int(1); 10])); let (a, b) = (held_bytes(&one, &data, 0), held_bytes(&two, &data, 0)); let mut cache = super::state::ObjectCache::default(); - cache.insert(1, one.clone(), a); - cache.insert(2, two, b); + cache.insert(1, Ok(one.clone()), a); + cache.insert(2, Ok(two), b); assert_eq!(cache.owned(), a + b); - cache.insert(2, one, a); + cache.insert(2, Ok(one), a); assert_eq!(cache.owned(), a + a, "a replaced object no longer counts"); cache.retain(|n| *n == 1); assert_eq!((cache.len(), cache.owned()), (1, a)); diff --git a/crates/cos/src/document/xref.rs b/crates/cos/src/document/xref.rs index 02390ff7da..771df28b80 100644 --- a/crates/cos/src/document/xref.rs +++ b/crates/cos/src/document/xref.rs @@ -41,7 +41,7 @@ pub(super) struct XrefRevisionSection { #[derive(Clone, Debug)] pub(super) enum XrefIndex { Compact { revisions: Vec, max_num: u32, has_entries: bool }, - Materialized(Arc>), + Materialized(Arc>, Arc>), } pub(super) const MAX_COMPACT_XREF_ROWS: usize = 2_000_000; @@ -149,8 +149,8 @@ impl XrefSection { } } - pub(super) fn extend_numbers(&self, numbers: &mut Vec) { - for range in &self.ranges { + pub(super) fn numbers(&self) -> impl Iterator + '_ { + self.ranges.iter().flat_map(|range| { let available = match &self.data { XrefSectionData::Table => range.count as usize, XrefSectionData::Stream { bytes, widths } => { @@ -158,14 +158,12 @@ impl XrefSection { bytes.len().saturating_sub(range.offset).checked_div(row_width).unwrap_or(0) } }; - for i in 0..available.min(range.count as usize) { - if let Ok(i) = u32::try_from(i) - && let Some(num) = range.start.checked_add(i) - { - numbers.push(num); - } - } - } + (0..available.min(range.count as usize)).filter_map(move |i| u32::try_from(i).ok().and_then(|i| range.start.checked_add(i))) + }) + } + + pub(super) fn extend_numbers(&self, numbers: &mut Vec) { + numbers.extend(self.numbers()); } } @@ -189,20 +187,20 @@ impl XrefRevisionSection { impl XrefIndex { pub(super) fn empty() -> Self { - Self::Materialized(Arc::new(BTreeMap::new())) + Self::Materialized(Arc::new(BTreeMap::new()), Arc::default()) } pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { match self { Self::Compact { revisions, .. } => revisions.iter().find_map(|revision| revision.lookup(num, file)), - Self::Materialized(entries) => entries.get(&num).copied(), + Self::Materialized(entries, _) => entries.get(&num).copied(), } } pub(super) fn is_empty(&self) -> bool { match self { Self::Compact { has_entries, .. } => !has_entries, - Self::Materialized(entries) => entries.is_empty(), + Self::Materialized(entries, _) => entries.is_empty(), } } @@ -211,14 +209,14 @@ impl XrefIndex { pub(super) fn remove_exhausted(&mut self) -> bool { match self { Self::Compact { .. } => false, - Self::Materialized(entries) => Arc::make_mut(entries).remove(&u32::MAX).is_some(), + Self::Materialized(entries, _) => Arc::make_mut(entries).remove(&u32::MAX).is_some(), } } /// Every entry, newest section first, as the materialized reader keeps them. pub(super) fn to_map(&self, file: &[u8]) -> BTreeMap { match self { - Self::Materialized(entries) => entries.as_ref().clone(), + Self::Materialized(entries, _) => entries.as_ref().clone(), Self::Compact { revisions, .. } => { let mut numbers = Vec::new(); for revision in revisions { @@ -232,13 +230,13 @@ impl XrefIndex { pub(super) fn max_num(&self) -> u32 { match self { Self::Compact { max_num, .. } => *max_num, - Self::Materialized(entries) => entries.keys().next_back().copied().unwrap_or(0), + Self::Materialized(entries, _) => entries.keys().next_back().copied().unwrap_or(0), } } pub(super) fn object_numbers(&self, file: &[u8]) -> Vec { match self { - Self::Materialized(entries) => { + Self::Materialized(entries, _) => { entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect() } Self::Compact { revisions, .. } => { diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 6187acb846..32a6f6d8b3 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -22,7 +22,7 @@ mod parser; mod security; mod writer; -pub use budget::{DecodeBudget, ImageBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS}; +pub use budget::{DecodeBudget, ImageBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS, MAX_REDACTED_IMAGE_PIXELS}; pub use bytes::Bytes; pub use document::{Document, ReadState, Revision, SourceIdentity, XrefEntry}; pub use object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, Name, ObjRef, Object, PdfString, Stream}; @@ -30,7 +30,7 @@ pub use parser::{Lexer, is_whitespace, parse_indirect}; pub use pdfcraft_crypt::{ Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, }; -pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental}; +pub use writer::{SaveOptions, pdf_date, reachable_objects, serialize, write_full, write_incremental}; #[derive(Debug, thiserror::Error, Clone, PartialEq)] pub enum CosError { diff --git a/crates/cos/src/security.rs b/crates/cos/src/security.rs index 22c85c7f68..61c4f92a4c 100644 --- a/crates/cos/src/security.rs +++ b/crates/cos/src/security.rs @@ -7,10 +7,69 @@ //! false. Streams naming a crypt filter (`/Crypt` in `/Filter`) use that filter; embedded files //! use `/EFF`. -use pdfcraft_crypt::{EncryptDict, Method, SecurityHandler, StreamKind}; +use std::cell::Cell; +use pdfcraft_crypt::{EncryptDict, Method, SecurityHandler, StreamKind}; +use pdfcraft_filters::MAX_CHAIN; + +use crate::document::Held; use crate::{Dict, Document, Object, PdfString, Stream}; +/// How many references one read or write of an object may follow, with all that it loads on the +/// way. How an object is treated can be given by reference, and what that loads can do the same: a +/// file could make one read a tree of them, each object read again once the cache has let it go. +/// An object takes a few of these, and two dozen if every part of it is given by reference; past +/// this they read as absent. The parameters of an object stream, read together, count as one. +const MAX_FOLLOWED: u32 = 32; + +std::thread_local! { + /// How many [`FollowScope`]s this thread is inside. + static SCOPES: Cell = const { Cell::new(0) }; + /// The references followed in the outermost of them. + static FOLLOWED: Cell = const { Cell::new(0) }; + /// The references refused in the outermost of them, past [`MAX_FOLLOWED`]. + static REFUSED: Cell = const { Cell::new(0) }; +} + +/// One read or write of an object, with everything it loads on the way: what [`MAX_FOLLOWED`] counts. +pub(crate) struct FollowScope; + +impl FollowScope { + pub(crate) fn enter() -> Self { + if SCOPES.replace(SCOPES.get().saturating_add(1)) == 0 { + FOLLOWED.set(0); + REFUSED.set(0); + } + FollowScope + } +} + +impl Drop for FollowScope { + fn drop(&mut self) { + SCOPES.set(SCOPES.get().saturating_sub(1)); + } +} + +/// What `f` gives, counted against [`MAX_FOLLOWED`], or `spent` past it. Called inside a +/// [`FollowScope`], which is what starts the count again. +pub(crate) fn following(spent: T, f: impl FnOnce() -> T) -> T { + if FOLLOWED.get() >= MAX_FOLLOWED { + REFUSED.set(REFUSED.get().saturating_add(1)); + return spent; + } + FOLLOWED.set(FOLLOWED.get() + 1); + f() +} + +/// `o`, or the object it refers to, for what decides how a stream or dictionary is treated. Past +/// [`MAX_FOLLOWED`] a reference reads as absent. +fn followed<'o>(doc: &Document, o: &'o Object) -> Held<'o> { + match o { + Object::Ref(_) => following(Held::Direct(&Object::Null), || doc.held(o)), + o => Held::Direct(o), + } +} + fn bytes_of(doc: &Document, d: &Dict, key: &[u8]) -> Vec { d.get(key).map(|v| doc.resolve(v)).and_then(|v| v.as_string().map(|s| s.bytes.clone())).unwrap_or_default() } @@ -18,12 +77,12 @@ fn bytes_of(doc: &Document, d: &Dict, key: &[u8]) -> Vec { /// Read the handler parameters from an `/Encrypt` dictionary. pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { let int = |k: &[u8], default: i64| d.get(k).map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(default); - let name = |k: &[u8]| d.get(k).and_then(|v| v.as_name()).map(|n| n.to_vec()).unwrap_or_else(|| b"Identity".to_vec()); + let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec)); let mut crypt_filters = Vec::new(); if let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) { for (k, v) in cf.iter() { let Some(f) = doc.resolve(v).as_dict().cloned() else { continue }; - let method = match f.name(b"CFM") { + let method = match name(&f, b"CFM").as_deref() { Some(b"V2") => Method::Rc4, Some(b"AESV2") => Method::Aes128, Some(b"AESV3") => Method::Aes256, @@ -40,7 +99,8 @@ pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { .get(b"CF") .map(|c| doc.resolve(c)) .and_then(|c| c.as_dict().and_then(|c| c.get(b"StdCF").map(|f| doc.resolve(f)))) - .and_then(|f| f.as_dict().and_then(|f| f.int(b"Length"))) + .and_then(|f| f.as_dict().and_then(|f| f.get(b"Length").map(|n| doc.resolve(n)))) + .and_then(|n| n.as_int()) .filter(|n| *n > 0); let length_bits = match (v, cf_len) { (5.., _) => 256, @@ -50,7 +110,7 @@ pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { _ => int(b"Length", 40), }; EncryptDict { - filter: d.name(b"Filter").map(|n| n.to_vec()).unwrap_or_default(), + filter: name(d, b"Filter").unwrap_or_default(), v, r: int(b"R", 0), length_bits, @@ -60,28 +120,46 @@ pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { ue: bytes_of(doc, d, b"UE"), perms: bytes_of(doc, d, b"Perms"), p: int(b"P", -1) as i32, - encrypt_metadata: !matches!(d.get(b"EncryptMetadata"), Some(Object::Bool(false))), + encrypt_metadata: !matches!(d.get(b"EncryptMetadata").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(false))), crypt_filters, - stm_f: name(b"StmF"), - str_f: name(b"StrF"), - ef_f: d.get(b"EFF").and_then(|v| v.as_name()).map(|n| n.to_vec()).unwrap_or_default(), + stm_f: name(d, b"StmF").unwrap_or_else(|| b"Identity".to_vec()), + str_f: name(d, b"StrF").unwrap_or_else(|| b"Identity".to_vec()), + ef_f: name(d, b"EFF").unwrap_or_default(), } } -/// The crypt filter a stream names in its `/Filter` chain (`/Crypt` + `/DecodeParms /Name`). -fn named_crypt_filter(dict: &Dict) -> Option> { - let names: Vec<&[u8]> = match dict.get(b"Filter") { - Some(Object::Name(n)) => vec![n], - Some(Object::Array(a)) => a.iter().filter_map(Object::as_name).collect(), +/// Whether `f`, an entry of a `/Filter` chain or its only filter, is `/Crypt`. +fn is_crypt(doc: &Document, f: &Object) -> bool { + followed(doc, f).as_name() == Some(b"Crypt") +} + +/// The crypt filter a stream names in its `/Filter` chain (`/Crypt` + `/DecodeParms /Name`), each +/// given directly or by reference, in an array or not. A chain longer than any decoder takes names +/// none: nothing can read that stream anyway, and every stream of a file can share one long array. +/// A `/Name` that reads as absent means `/Identity`, unless a reference on the way to it was refused +/// (see [`MAX_FOLLOWED`]): that says nothing about the stream, so it names none, and the document's +/// own filter is used. +fn named_crypt_filter(doc: &Document, dict: &Dict) -> Option> { + let refused = REFUSED.get(); + let filter = followed(doc, dict.get(b"Filter")?); + let i = match &*filter { + Object::Array(a) if a.len() <= MAX_CHAIN + 1 => a.iter().position(|f| is_crypt(doc, f))?, + f if is_crypt(doc, f) => 0, _ => return None, }; - let i = names.iter().position(|n| *n == b"Crypt")?; - let parms = match dict.get(b"DecodeParms") { - Some(Object::Dict(d)) if i == 0 => Some(d.clone()), - Some(Object::Array(a)) => a.get(i).and_then(|p| p.as_dict().cloned()), + let parms = dict.get(b"DecodeParms").map(|p| followed(doc, p)); + let entry = match parms.as_deref() { + Some(Object::Array(a)) => a.get(i).map(|p| followed(doc, p)), + Some(p) if i == 0 => Some(Held::Direct(p)), _ => None, }; - Some(parms.and_then(|p| p.name(b"Name").map(|n| n.to_vec())).unwrap_or_else(|| b"Identity".to_vec())) + let name = entry.as_deref().and_then(Object::as_dict).and_then(|p| p.get(b"Name")).map(|n| followed(doc, n)); + match name.as_deref().and_then(Object::as_name) { + Some(name) => Some(name.to_vec()), + // /Identity here would read ciphertext as data, and write the stream in the clear. + None if REFUSED.get() > refused => None, + None => Some(b"Identity".to_vec()), + } } /// A signature dictionary (field signature, document timestamp or usage-rights signature), @@ -91,42 +169,67 @@ fn named_crypt_filter(dict: &Dict) -> Option> { /// `/Contents` alongside an array `/ByteRange` (`/Type` is optional). A signature dictionary /// has no `/Subtype`; a dictionary with one (an annotation, whose `/Contents` is the comment /// text) or with another `/Type` is never exempt, whatever else it carries. -fn is_signature_dict(d: &Dict) -> bool { - if d.get(b"Subtype").is_some() { +/// With `doc` its entries are read through references; a dictionary nested in another object +/// has none (see [`transform`]). +fn is_signature_dict(doc: Option<&Document>, d: &Dict) -> bool { + // All this decides is what happens to such a /Contents. + if !matches!(d.get(b"Contents"), Some(Object::String(_))) { return false; } - match d.get(b"Type") { - Some(_) => matches!(d.name(b"Type"), Some(b"Sig" | b"DocTimeStamp")), - None => matches!(d.get(b"Contents"), Some(Object::String(_))) && matches!(d.get(b"ByteRange"), Some(Object::Array(_))), + // A null entry, or one that refers to nothing, is an absent one (ISO 32000-2 §7.3.7, §7.3.10). + let entry = |key: &[u8]| { + let value = d.get(key)?; + let held = doc.map_or(Held::Direct(value), |doc| followed(doc, value)); + (!matches!(*held, Object::Null)).then_some(held) + }; + if entry(b"Subtype").is_some() { + return false; + } + match entry(b"Type") { + Some(t) => matches!(t.as_name(), Some(b"Sig" | b"DocTimeStamp")), + None => matches!(entry(b"ByteRange").as_deref(), Some(Object::Array(_))), } } -/// Remove the `/Crypt` entry from a stream's filter chain, with its decode parameters. -fn drop_crypt_filter(dict: &mut Dict) { - let i = match dict.get(b"Filter") { - Some(Object::Array(a)) => a.iter().position(|n| n.as_name() == Some(b"Crypt")), - Some(Object::Name(n)) if n.as_slice() == b"Crypt" => Some(0), - _ => None, +/// Longer than any filter's name. A name can be of any length, and a chain every stream shares can +/// hold a long one. +const MAX_FILTER_NAME: usize = 127; + +/// The dictionary of `source` without the `/Crypt` entries of its filter chain and their decode +/// parameters. The two come out direct when given by reference, which every stream of a file can +/// share, so of what was shared only small values are copied: filter names, and the parameters +/// decoders read ([`Document::with_direct_parms`]). The chain is short, as [`named_crypt_filter`] +/// found a filter in it. +fn drop_crypt_filter(doc: &Document, source: &Stream) -> Dict { + let entry = |f: &Object| match f { + f if is_crypt(doc, f) => Object::name("Crypt"), + Object::Name(n) if n.len() <= MAX_FILTER_NAME => f.clone(), + Object::Ref(_) => f.clone(), + _ => Object::Null, }; - let Some(i) = i else { return }; - let mut filters = match dict.get(b"Filter") { - Some(Object::Array(a)) => a.clone(), - _ => Vec::new(), + let filter = source.dict.get(b"Filter").map(|f| followed(doc, f)); + let chain: Vec = match filter.as_deref() { + Some(Object::Array(a)) => a.iter().map(entry).collect(), + Some(f) => vec![entry(f)], + None => Vec::new(), }; - if i < filters.len() { - filters.remove(i); - } + let crypt: Vec = chain.iter().map(|f| f.as_name() == Some(b"Crypt")).collect(); + let mut chained = Stream::from_raw(source.dict.clone(), Vec::new()); + chained.dict.set(b"Filter".to_vec(), Object::Array(chain.clone())); // A lone parameter dictionary belongs to the first filter (as `named_crypt_filter` reads it). - match dict.get(b"DecodeParms").cloned() { - Some(Object::Array(mut parms)) if i < parms.len() => { - parms.remove(i); - dict.set(b"DecodeParms".to_vec(), Object::Array(parms)); - } - Some(Object::Dict(_)) if i == 0 => { + let parms = match doc.with_direct_parms(&chained).dict.get(b"DecodeParms") { + Some(Object::Array(p)) => Some(Object::Array(p.iter().zip(&crypt).filter(|(_, c)| !**c).map(|(p, _)| p.clone()).collect())), + Some(Object::Dict(_)) if crypt.first() == Some(&true) => None, + _ => source.dict.get(b"DecodeParms").cloned(), + }; + let mut dict = source.dict.clone(); + match parms { + Some(p) => dict.set(b"DecodeParms".to_vec(), p), + None => { dict.remove(b"DecodeParms"); } - _ => {} } + let filters: Vec = chain.into_iter().zip(&crypt).filter(|(_, c)| !**c).map(|(f, _)| f).collect(); if filters.is_empty() { dict.remove(b"Filter"); if matches!(dict.get(b"DecodeParms"), Some(Object::Array(p)) if p.is_empty()) { @@ -135,6 +238,7 @@ fn drop_crypt_filter(dict: &mut Dict) { } else { dict.set(b"Filter".to_vec(), Object::Array(filters)); } + dict } /// What [`transform`] does to an object. @@ -152,51 +256,81 @@ pub(crate) enum Pass { /// Decrypt or encrypt every string and stream in an indirect object, except a signature /// dictionary's `/Contents` (see [`is_signature_dict`]). -pub(crate) fn transform(h: &SecurityHandler, o: &Object, num: u32, generation: u16, pass: Pass) -> Object { +/// +/// `o` is as `doc` holds it: what a stream's dictionary gives by reference is read from `doc`, so +/// its references have to be the document's, not a save's renumbered ones. +pub(crate) fn transform(doc: &Document, h: &SecurityHandler, o: &Object, num: u32, generation: u16, pass: Pass) -> Object { + let _scope = FollowScope::enter(); let decrypt = pass == Pass::Decrypt; let strings = |s: &PdfString| { let bytes = if decrypt { h.decrypt_string(num, generation, &s.bytes) } else { h.encrypt_string(num, generation, &s.bytes) }; PdfString { bytes, hex: s.hex || !decrypt } }; - walk(o, &strings, h, num, generation, pass) + // References are followed here only, for the object itself, never for what is nested in it. + // `walk` goes deep, and a load from down there adds that depth again for each object loaded, + // which overflowed the stack on a 10 KB file. A stream can't be nested, and a dictionary that + // is reads as it says directly. + match o { + Object::Stream(s) => stream(doc, s, &strings, h, num, generation, pass), + Object::Dict(d) if is_signature_dict(Some(doc), d) => signature_dict(d, &strings), + o => walk(o, &strings), + } } -fn walk(o: &Object, strings: &dyn Fn(&PdfString) -> PdfString, h: &SecurityHandler, num: u32, generation: u16, pass: Pass) -> Object { - let decrypt = pass == Pass::Decrypt; +/// `d`, a signature dictionary, with the strings of everything but its `/Contents` transformed. +fn signature_dict(d: &Dict, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { + Object::Dict( + d.iter() + .map(|(k, v)| match (k.as_slice(), v) { + (b"Contents", Object::String(_)) => (k.clone(), v.clone()), + _ => (k.clone(), walk(v, strings)), + }) + .collect(), + ) +} + +fn walk(o: &Object, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { match o { Object::String(s) => Object::String(strings(s)), - Object::Array(a) => Object::Array(a.iter().map(|x| walk(x, strings, h, num, generation, pass)).collect()), - Object::Dict(d) if is_signature_dict(d) => Object::Dict( - d.iter() - .map(|(k, v)| match (k.as_slice(), v) { - (b"Contents", Object::String(_)) => (k.clone(), v.clone()), - _ => (k.clone(), walk(v, strings, h, num, generation, pass)), - }) - .collect(), - ), - Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), walk(v, strings, h, num, generation, pass))).collect()), - Object::Stream(s) => { - let mut dict: Dict = s.dict.iter().map(|(k, v)| (k.clone(), walk(v, strings, h, num, generation, pass))).collect(); - let ty = s.dict.name(b"Type"); - // Cross-reference streams are never encrypted (§7.5.8), but only reading may trust the - // label: a stream a producer calls /XRef stays plaintext if writing believes it, and the - // writer builds its own cross-reference streams outside this transform. - if (decrypt && ty == Some(b"XRef")) || (ty == Some(b"Metadata") && !h.encrypts_metadata()) { - return Object::Stream(Stream { dict, raw: s.raw.clone() }); - } - let mut named = named_crypt_filter(&s.dict); - if pass == Pass::EncryptReplacing && named.as_deref().is_some_and(|n| !h.dict().crypt_filters.iter().any(|(f, _)| f == n)) { - drop_crypt_filter(&mut dict); - named = None; - } - let kind = match (&named, ty) { - (Some(n), _) => StreamKind::Named(n), - (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, - _ => StreamKind::Normal, - }; - let raw = if decrypt { h.decrypt_stream(num, generation, &s.raw, kind) } else { h.encrypt_stream(num, generation, &s.raw, kind) }; - Object::Stream(Stream { dict, raw: raw.into() }) - } + Object::Array(a) => Object::Array(a.iter().map(|x| walk(x, strings)).collect()), + Object::Dict(d) if is_signature_dict(None, d) => signature_dict(d, strings), + Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect()), other => other.clone(), } } + +fn stream( + doc: &Document, + s: &Stream, + strings: &dyn Fn(&PdfString) -> PdfString, + h: &SecurityHandler, + num: u32, + generation: u16, + pass: Pass, +) -> Object { + let decrypt = pass == Pass::Decrypt; + let walked = |d: &Dict| -> Dict { d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect() }; + let ty = s.dict.get(b"Type").map(|t| followed(doc, t)); + let ty = ty.as_deref().and_then(Object::as_name); + // Cross-reference streams are never encrypted (§7.5.8), but only reading may trust the + // label: a stream a producer calls /XRef stays plaintext if writing believes it, and the + // writer builds its own cross-reference streams outside this transform. + if (decrypt && ty == Some(b"XRef")) || (ty == Some(b"Metadata") && !h.encrypts_metadata()) { + return Object::Stream(Stream { dict: walked(&s.dict), raw: s.raw.clone() }); + } + let mut named = named_crypt_filter(doc, &s.dict); + // Before the walk, which then encrypts what is in the dictionary this leaves. + let foreign = pass == Pass::EncryptReplacing && named.as_deref().is_some_and(|n| !h.dict().crypt_filters.iter().any(|(f, _)| f == n)); + let dropped = foreign.then(|| drop_crypt_filter(doc, s)); + if dropped.is_some() { + named = None; + } + let dict = walked(dropped.as_ref().unwrap_or(&s.dict)); + let kind = match (&named, ty) { + (Some(n), _) => StreamKind::Named(n), + (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, + _ => StreamKind::Normal, + }; + let raw = if decrypt { h.decrypt_stream(num, generation, &s.raw, kind) } else { h.encrypt_stream(num, generation, &s.raw, kind) }; + Object::Stream(Stream { dict, raw: raw.into() }) +} diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index cd0137f452..0af74f7bab 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -133,11 +133,12 @@ fn write_stream(s: &Stream, out: &mut Vec) { /// The object as written: encrypted with its (output) number when the document is encrypted, /// except the `/Encrypt` dictionary itself (`source_num` is its number in the source document). +/// `o` has the document's numbers in its references ([`crate::security::transform`]). fn prepared<'a>(doc: &Document, source_num: u32, num: u32, generation: u16, o: &'a Object) -> Cow<'a, Object> { match doc.output_security() { - (Some(h), encrypt_num) if Some(source_num) != encrypt_num => { + (Some(h), encrypt_num) if Some(source_num) != encrypt_num && !doc.stored_as_is(source_num) => { let pass = if doc.encryption_changed() { crate::security::Pass::EncryptReplacing } else { crate::security::Pass::Encrypt }; - Cow::Owned(crate::security::transform(h, o, num, generation, pass)) + Cow::Owned(crate::security::transform(doc, h, o, num, generation, pass)) } _ => Cow::Borrowed(o), } @@ -166,11 +167,17 @@ pub struct SaveOptions { /// before adding it, with [`Document::free_unlinked_additions`]. Full saves always leave /// these out. pub drop_unlinked: bool, + /// Entropy for the file identifier of a rewrite that asks for a new one (see + /// [`Document::require_full_save_with_new_id`]). Given, the identifier depends only on this and + /// on the written bytes, so the same save gives the same file; tests use it to pin output. + /// Without it, fresh entropy from the host (the operating system on native targets, the + /// browser on WebAssembly) is mixed in. + pub id_entropy: Option<[u8; 16]>, } impl Default for SaveOptions { fn default() -> Self { - Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true, drop_unlinked: false } + Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true, drop_unlinked: false, id_entropy: None } } } @@ -266,16 +273,12 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, Ok(file) } -/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. -pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - // A full traversal must not fill the editor's shared caches (which undo snapshots also - // own). Parsed objects and decoded object streams are temporary working data here. - let mut reader = doc.object_reader(); - stamp_mod_date(&mut reader.document, opts); - let trailer_in = reader.document.trailer().clone(); +/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and +/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets +/// (aliases of one slot included) and, in visit order, whether it is a stream. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>); + +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -287,6 +290,10 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro queue.push_back(r); } } + // A direct /Encrypt can refer to objects, /CF or /StmF, say. + if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { + visit_refs(encrypt, &mut queue); + } // Unknown trailer entries are document data. Objects they alone reference must be written // too, or a full rewrite would keep the key and drop its value. for (key, value) in trailer_in.iter() { @@ -302,10 +309,43 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro if matches!(*o, Object::Null) { continue; // dangling reference: written as null by renumbering below } - map.insert(r, output_number(order.len())?); - order.push((r, matches!(*o, Object::Stream(_)))); + // Accepted xref/header aliases denote one slot, including page-tree cycles. + // Check for null first: an unrelated or freed generation must remain dangling. + let canonical = ObjRef::new(r.num, reader.document.generation(r.num)); + if let Some(&num) = map.get(&canonical) { + map.insert(r, num); + continue; + } + let num = output_number(order.len())?; + map.insert(r, num); + map.insert(canonical, num); + order.push((canonical, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); } + Ok((map, order)) +} + +/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] +/// reaches from the trailer, object streams included. For a check that something removed from a +/// document cannot reach the saved file through a reference nobody looked for. +pub fn reachable_objects(doc: &Document) -> Result, CosError> { + let mut reader = doc.object_reader(); + let trailer_in = reader.document.trailer().clone(); + let (_, order) = full_save_order(&mut reader, &trailer_in)?; + Ok(order.into_iter().map(|(r, _)| r.num).collect()) +} + +/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. +pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + // A full traversal must not fill the editor's shared caches (which undo snapshots also + // own). Parsed objects and decoded object streams are temporary working data here. + let mut reader = doc.object_reader(); + stamp_mod_date(&mut reader.document, opts); + let trailer_in = reader.document.trailer().clone(); + let (map, order) = full_save_order(&mut reader, &trailer_in)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); @@ -314,10 +354,15 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro out.extend_from_slice(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n']); // The /Encrypt dictionary and everything it references (e.g. indirect /CF crypt-filter // dictionaries) are needed before anything can be decrypted: never inside object streams. - let mut encryption_objects: std::collections::HashSet = std::collections::HashSet::new(); - let mut pending: Vec = trailer_in.reference(b"Encrypt").into_iter().collect(); + let mut encryption_objects: std::collections::HashSet = std::collections::HashSet::new(); + let mut pending: Vec = Vec::new(); + if let Some(encrypt) = trailer_in.get(b"Encrypt") { + collect_refs(encrypt, &mut |r| pending.push(r)); + } while let Some(r) = pending.pop() { - if encryption_objects.insert(r) { + if let Some(&num) = map.get(&r) + && encryption_objects.insert(num) + { collect_refs(&reader.get(r), &mut |x| pending.push(x)); } } @@ -325,20 +370,22 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro // damaged and could not be repaired", before it even asks for the password), while other // readers accept it. qpdf keeps the catalog out of object streams too. Verified with // Acrobat DC on macOS (#774). - let root_outside = if reader.document.output_security().0.is_some() { trailer_in.reference(b"Root") } else { None }; + let root_outside = + if reader.document.output_security().0.is_some() { trailer_in.reference(b"Root").and_then(|r| map.get(&r)).copied() } else { None }; let mut rows: BTreeMap = BTreeMap::new(); // Keep only references, not renamed copies of the entire object graph. Standalone // objects still precede object streams, preserving the established output order. let mut packed: Vec<(u32, ObjRef)> = Vec::new(); for (i, (r, is_stream)) in order.iter().enumerate() { let num = output_number(i)?; - let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(r) && root_outside != Some(*r); + let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(&num) && root_outside != Some(num); if in_stream { packed.push((num, *r)); } else { - let o = renumber(&*reread(&mut reader, *r, *is_stream)?, &map); + let o = reread(&mut reader, *r, *is_stream)?; + let o = renumber(&prepared(&reader.document, r.num, num, 0, &o), &map); rows.insert(num, Row::InFile(out.len() as u64, 0)); - write_indirect(num, 0, &prepared(&reader.document, r.num, num, 0, &o), &mut out); + write_indirect(num, 0, &o, &mut out); } } // Object streams take the numbers after the last object. Their contents are not encrypted @@ -362,26 +409,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro } } batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; - let mut trailer = Dict::new(); - for key in [&b"Root"[..], b"Info", b"Encrypt"] { - if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) { - trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0))); - } - } - if let Some(Object::Dict(d)) = trailer_in.get(b"Encrypt") { - trailer.set(b"Encrypt".to_vec(), Object::Dict(d.clone())); - } - if let Some(id) = trailer_in.get(b"ID") { - trailer.set(b"ID".to_vec(), id.clone()); - } - for (key, value) in trailer_in.iter() { - if !trailer_key_rewritten(key) { - trailer.set(key.clone(), renumber(value, &map)); - } - } - if reader.document.output_security().0.is_none() { - ensure_id(&mut trailer, opts, reader.document.bytes()); - } + let mut trailer = full_save_trailer(&reader.document, &trailer_in, &map, opts, &out); if opts.object_streams { write_xref_stream(&mut out, 0, &mut trailer, rows, next); } else { @@ -400,6 +428,34 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro Ok(out) } +/// Assemble the full rewrite's trailer and apply its identifier policy. +fn full_save_trailer(doc: &Document, trailer_in: &Dict, map: &HashMap, opts: &SaveOptions, out: &[u8]) -> Dict { + let mut trailer = Dict::new(); + for key in [&b"Root"[..], b"Info", b"Encrypt"] { + if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) { + trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0))); + } + } + if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { + trailer.set(b"Encrypt".to_vec(), renumber(encrypt, map)); + } + if let Some(id) = trailer_in.get(b"ID") { + trailer.set(b"ID".to_vec(), direct_id(doc, id)); + } + for (key, value) in trailer_in.iter() { + if !trailer_key_rewritten(key) { + trailer.set(key.clone(), renumber(value, map)); + } + } + if doc.output_security().0.is_none() { + ensure_id(&mut trailer, opts, doc.bytes()); + } + if doc.new_id_required() { + regenerate_id(&mut trailer, opts, out, doc.output_security().0.is_some()); + } + trailer +} + /// Object `r` read again for writing. The traversal read it once and only kept its number, so /// a second read that fails, finds nothing or finds a different kind of object must fail the /// save rather than write `null` in its place. @@ -442,6 +498,22 @@ impl ObjectStreamBatch { } } +/// `/ID` for the trailer, direct, as it has to be with `/Encrypt`: nothing else in the file holds +/// what it refers to now. Given by reference it is two entries at most, however many references a +/// file gave, so that one large string isn't copied for each. An entry that isn't a string stays in +/// its place as an empty one, which is how the key read it. +fn direct_id(doc: &Document, id: &Object) -> Object { + if matches!(id, Object::Array(a) if !a.iter().any(|s| matches!(s, Object::Ref(_)))) { + return id.clone(); + } + let entry = |s: &Object| match &*doc.resolve(s) { + s @ Object::String(_) => s.clone(), + _ => Object::String(crate::PdfString { bytes: Vec::new(), hex: true }), + }; + let held = doc.resolve(id); + Object::Array(held.as_array().into_iter().flatten().take(2).map(entry).collect()) +} + /// Trailer keys a full rewrite generates itself. Everything else is copied, with references /// renumbered, so a save does not drop metadata the reader accepted. fn trailer_key_rewritten(key: &[u8]) -> bool { @@ -486,7 +558,7 @@ pub(crate) fn unlinked_additions(doc: &Document) -> HashSet { pending.into_keys().collect() } -fn collect_refs(o: &Object, f: &mut impl FnMut(ObjRef)) { +pub(crate) fn collect_refs(o: &Object, f: &mut impl FnMut(ObjRef)) { match o { Object::Ref(r) => f(*r), Object::Array(a) => a.iter().for_each(|x| collect_refs(x, f)), @@ -632,6 +704,136 @@ fn ensure_id(trailer: &mut Dict, opts: &SaveOptions, content: &[u8]) { trailer.set(b"ID".to_vec(), Object::Array(vec![s.clone(), s])); } +/// Give a rewritten file a new identifier (§14.4), so that a redacted or sanitized copy does not +/// claim to be its source. An encrypted file keeps the first string, which its key is derived from, +/// and gets a new second one. When there is no such string (no `/ID`, one entry, or a first entry +/// that is not a string) the reader takes the empty string as the key input (`Document::first_id`), +/// so the empty string is written first and the key stays valid. +fn regenerate_id(trailer: &mut Dict, opts: &SaveOptions, written: &[u8], encrypted: bool) { + let fresh = Object::String(PdfString { bytes: new_id_bytes(opts, written), hex: true }); + let pair = if encrypted { + let first = match trailer.get(b"ID") { + Some(Object::Array(a)) => match a.first() { + Some(Object::String(s)) => Object::String(s.clone()), + _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), + }, + _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), + }; + vec![first, fresh] + } else { + vec![fresh.clone(), fresh] + }; + trailer.set(b"ID".to_vec(), Object::Array(pair)); +} + +/// FNV-1a over 64 bits, with its published offset basis and prime. The identifier is built from it +/// rather than from `std`'s `DefaultHasher`, whose algorithm the standard library does not specify. +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +fn fnv1a(state: u64, bytes: &[u8]) -> u64 { + bytes.iter().fold(state, |h, b| (h ^ u64::from(*b)).wrapping_mul(FNV_PRIME)) +} + +/// Sixteen bytes for a file identifier, one 8-byte half per lane. The bytes are a pure function of +/// `SaveOptions::id_entropy`, `id_seed` and the written file: the hash is FNV-1a (above) and every +/// integer is written as a fixed-width big-endian value, so the same input gives the same identifier +/// on every platform and toolchain. Without entropy only the salt changes (see `id_salt`). FNV is +/// not collision-resistant against chosen content; the identifier only has to be unique, not secret. +/// Every written byte is hashed, once, in a single traversal of the output. +fn new_id_bytes(opts: &SaveOptions, written: &[u8]) -> Vec { + let salt = id_salt(opts); + let length = u64::try_from(written.len()).unwrap_or(u64::MAX); + let mut halves = [0u8, 1].map(|lane| { + let mut h = fnv1a(FNV_OFFSET, &salt.to_be_bytes()); + h = fnv1a(h, &[lane]); + h = fnv1a(h, &opts.id_seed.to_be_bytes()); + fnv1a(h, &length.to_be_bytes()) + }); + for chunk in written.chunks(4096) { + for h in &mut halves { + *h = fnv1a(*h, chunk); + } + } + halves.iter().flat_map(|h| h.to_be_bytes()).collect() +} + +/// The salt of an identifier. With caller entropy it is a fixed hash of that entropy (deterministic, +/// for tests and callers who pin output). Without it the salt is fresh: on WebAssembly from the +/// browser ([`browser_salt`], so two fresh web apps that write the same bytes cannot share an +/// identifier), and elsewhere from the standard library's random hash keys — seeded by the operating +/// system, so separate processes differ — mixed with a process-wide counter, which separates +/// successive calls in one process. +fn id_salt(opts: &SaveOptions) -> u64 { + use std::hash::BuildHasher; + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + match &opts.id_entropy { + Some(entropy) => fnv1a(FNV_OFFSET, entropy), + None => { + #[cfg(target_arch = "wasm32")] + if let Some(salt) = browser_salt() { + return salt; + } + std::collections::hash_map::RandomState::new().hash_one(COUNTER.fetch_add(1, Ordering::Relaxed)) + } + } +} + +/// One byte from one `Math.random` draw: the draw (in [0, 1)) scaled to [0, 256), integer part +/// taken. The quotient is in [0, 256), so a saturating cast never wraps; a NaN or infinite draw +/// (which a conforming engine never returns) folds in as 0. No input can panic here. +#[cfg(any(target_arch = "wasm32", test))] +fn draw_byte(random: f64) -> u8 { + (random * 256.0).floor().rem_euclid(256.0) as u8 +} + +/// The `Math.random` fallback's mixing: one salt byte per draw (see [`draw_byte`]), folded through +/// FNV-1a exactly like the Web Crypto bytes, so all sixteen bytes of the fold carry entropy from +/// the draws. Pure and platform-independent, so the native test target can exercise it; on wasm +/// each draw comes from `Math.random`. +#[cfg(any(target_arch = "wasm32", test))] +fn salt_from_draws(draws: &[f64; 16]) -> u64 { + let mut bytes = [0u8; 16]; + for (byte, draw) in bytes.iter_mut().zip(draws) { + *byte = draw_byte(*draw); + } + fnv1a(FNV_OFFSET, &bytes) +} + +/// Fresh salt entropy from the browser, on WebAssembly. `crypto.getRandomValues` (Web Crypto, the +/// browser's secure generator, available in windows and workers alike) gives sixteen bytes, folded +/// into one salt. When that API is missing or refuses the call, `Math.random` is mixed instead +/// ([`salt_from_draws`]: sixteen draws, so the salt's full width carries entropy) — browsers seed +/// it afresh for every document, so a fresh instance still cannot reproduce another's salt. Only +/// where neither is reachable does the caller fall back to its `RandomState` salt, whose keys are +/// fixed at compile time on `wasm32-unknown-unknown` — the state before browser entropy was wired +/// in. +#[cfg(target_arch = "wasm32")] +fn browser_salt() -> Option { + use wasm_bindgen::JsCast; + let crypto = js_sys::Reflect::get(&js_sys::global(), &wasm_bindgen::JsValue::from_str("crypto")) + .ok() + .filter(|crypto| !crypto.is_undefined() && !crypto.is_null()); + if let Some(crypto) = crypto { + let fill = js_sys::Reflect::get(&crypto, &wasm_bindgen::JsValue::from_str("getRandomValues")) + .ok() + .and_then(|fill| fill.dyn_into::().ok()); + if let Some(fill) = fill { + let bytes = js_sys::Uint8Array::new_with_length(16); + if fill.call1(&crypto, &bytes).is_ok() { + return Some(fnv1a(FNV_OFFSET, &bytes.to_vec())); + } + } + } + // Web Crypto is missing or refused the call: degrade to `Math.random` rather than giving up. + let mut draws = [0.0; 16]; + for draw in &mut draws { + *draw = js_sys::Math::random(); + } + Some(salt_from_draws(&draws)) +} + /// A PDF date string for "now" given seconds since the Unix epoch (UTC). pub fn pdf_date(unix_secs: i64) -> String { let (days, secs) = (unix_secs.div_euclid(86_400), unix_secs.rem_euclid(86_400)); @@ -663,8 +865,67 @@ fn civil_from_days(days: i64) -> (i64, u32, u32) { #[cfg(test)] mod tests { + /// `/ID` by reference comes out direct, each entry where it was: the key is made from the first. + #[test] + fn a_file_identifier_keeps_its_entries_in_their_places() { + let doc = Document::new_empty(); + let string = |bytes: &[u8]| Object::String(crate::PdfString { bytes: bytes.to_vec(), hex: true }); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + assert_eq!(super::direct_id(&doc, &Object::Array(vec![nowhere, string(b"ab")])), Object::Array(vec![string(b""), string(b"ab")])); + } + use proptest::prelude::*; + #[test] + fn the_same_entropy_and_bytes_give_the_same_identifier_on_every_platform() { + // Computed from the formula outside this crate: fixed-width big-endian fields and FNV-1a. + let opts = super::SaveOptions { id_entropy: Some([0x42; 16]), id_seed: 7, ..super::SaveOptions::default() }; + let want = [0x96, 0xf8, 0xcb, 0x7d, 0x5b, 0xb0, 0x09, 0xc8, 0x92, 0xc9, 0xb7, 0x51, 0xdf, 0xe6, 0xab, 0xd1]; + assert_eq!(super::new_id_bytes(&opts, b"%PDF-1.7\n"), want); + } + + #[test] + fn fnv1a_is_the_published_64_bit_algorithm() { + // Published FNV-1a (64 bit) test vectors, so the identifier can be reproduced outside this crate. + assert_eq!(super::fnv1a(super::FNV_OFFSET, b""), 0xcbf2_9ce4_8422_2325); + assert_eq!(super::fnv1a(super::FNV_OFFSET, b"a"), 0xaf63_dc4c_8601_ec8c); + assert_eq!(super::fnv1a(super::FNV_OFFSET, b"foobar"), 0x8594_4171_f739_67e8); + } + + #[test] + fn math_random_draws_fill_the_whole_salt_width() { + // Draw i = (i + 1/2)/256 scales to the integer i, so the fold's byte i proves draw i landed + // there: all sixteen bytes carry their own draw's entropy (the fallback once shifted four + // draws into one word, leaving the other four bytes of the fold constant). + let draws: [f64; 16] = std::array::from_fn(|i| (i as f64 + 0.5) / 256.0); + let bytes: Vec = (0..16).collect(); + assert_eq!(super::salt_from_draws(&draws), super::fnv1a(super::FNV_OFFSET, &bytes)); + } + + #[test] + fn every_math_random_draw_reaches_the_salt() { + let base = [0.5; 16]; + let baseline = super::salt_from_draws(&base); + for i in 0..16 { + let mut draws = base; + draws[i] = 0.75; + assert_ne!(super::salt_from_draws(&draws), baseline, "draw {i} did not reach the salt"); + } + } + + #[test] + fn a_math_random_draw_outside_its_range_still_mixes_without_panicking() { + // A conforming `Math.random` stays in [0, 1), but the mixing must not panic outside it. + assert_eq!(super::draw_byte(0.0), 0); + assert_eq!(super::draw_byte(0.5), 128); + assert_eq!(super::draw_byte(255.5 / 256.0), 255); + assert_eq!(super::draw_byte(-0.5 / 256.0), 255); + assert_eq!(super::draw_byte(-1.0), 0); + assert_eq!(super::draw_byte(2.0), 0); + assert_eq!(super::draw_byte(f64::NAN), 0); + assert_eq!(super::draw_byte(f64::INFINITY), 0); + } + use super::*; use crate::Lexer; diff --git a/crates/cos/tests/file_identifier.rs b/crates/cos/tests/file_identifier.rs new file mode 100644 index 0000000000..313d56257b --- /dev/null +++ b/crates/cos/tests/file_identifier.rs @@ -0,0 +1,276 @@ +//! The file identifier (`/ID`, ISO 32000-2 §14.4) of a full save. An ordinary save keeps the +//! identifier the file has and writes the same bytes twice. A rewrite the document asked for (after +//! redaction or sanitizing) gives the file a new pair. An encrypted file keeps its first string, +//! because its file key is derived from it. + +use std::sync::Arc; + +use pdfcraft_cos::{Algorithm, Document, NewEncryption, Object, PdfString, SaveOptions, write_full, write_incremental}; +use pdfcraft_testkit::Pdf; + +const ID_HEX: &str = "/ID [<00112233445566778899AABBCCDDEEFF> <00112233445566778899AABBCCDDEEFF>]"; +const ID_BYTES: [u8; 16] = [0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]; + +/// One page whose content holds a secret, with `entry` added to the trailer (for example an `/ID`). +fn source_with(entry: Option<&str>) -> Vec { + let pdf = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>") + .stream("", "BT /F1 12 Tf 20 100 Td (TOPSECRET) Tj ET") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + match entry { + Some(entry) => pdf.trailer_entry(entry).build(), + None => pdf.build(), + } +} + +/// [`source_with`] with the standard identifier, or none. +fn source(with_id: bool) -> Vec { + source_with(with_id.then_some(ID_HEX)) +} + +/// A one-page file whose 70,000-byte content stream ends in `last`. Two of these built with +/// different last bytes have the same length and differ only past the first 64 KiB. +fn large_source(last: u8) -> Vec { + let mut content = vec![b' '; 70_000]; + content.push(last); + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>") + .stream("", content) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .build() +} + +fn reopen(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// The strings of the trailer's `/ID` (empty when there is none). +fn id_strings(doc: &Document) -> Vec> { + match doc.trailer().get(b"ID") { + Some(Object::Array(a)) => a.iter().map(|s| s.as_string().map(|s| s.bytes.clone()).unwrap_or_default()).collect(), + _ => Vec::new(), + } +} + +#[test] +fn an_ordinary_save_keeps_the_identifier_and_writes_the_same_bytes() { + let doc = reopen(source(true)); + let opts = SaveOptions::default(); + let saved = write_full(&doc, &opts).unwrap(); + assert_eq!(saved, write_full(&doc, &opts).unwrap(), "unchanged inputs produce identical bytes"); + assert_eq!(id_strings(&reopen(saved)), vec![ID_BYTES.to_vec(), ID_BYTES.to_vec()]); + let appended = reopen(write_incremental(&doc, &opts).unwrap()); + assert_eq!(id_strings(&appended), vec![ID_BYTES.to_vec(), ID_BYTES.to_vec()]); +} + +#[test] +fn an_ordinary_save_of_a_file_without_an_identifier_creates_the_same_one_every_time() { + let doc = reopen(source(false)); + let opts = SaveOptions::default(); + let saved = write_full(&doc, &opts).unwrap(); + assert_eq!(saved, write_full(&doc, &opts).unwrap(), "unchanged inputs produce identical bytes"); + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|s| s.len() == 16), "{ids:?}"); +} + +#[test] +fn a_rewrite_the_document_asked_for_gets_a_new_identifier() { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + assert!(doc.full_save_required() && doc.new_id_required()); + let opts = SaveOptions::default(); + for saved in [write_incremental(&doc, &opts).unwrap(), write_full(&doc, &opts).unwrap()] { + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2); + for id in &ids { + assert_eq!(id.len(), 16); + assert_ne!(id.as_slice(), ID_BYTES.as_slice(), "the copy must not claim the source's identifier"); + } + } + // Every save is a new identity, not the same one again. + let first = id_strings(&reopen(write_full(&doc, &opts).unwrap())); + let second = id_strings(&reopen(write_full(&doc, &opts).unwrap())); + assert_ne!(first, second); +} + +#[test] +fn a_rewrite_without_an_identifier_still_gets_one() { + let mut doc = reopen(source(false)); + doc.require_full_save_with_new_id(); + let ids = id_strings(&reopen(write_full(&doc, &SaveOptions::default()).unwrap())); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|s| s.len() == 16), "{ids:?}"); +} + +#[test] +fn caller_entropy_makes_the_new_identifier_reproducible() { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + let with = |entropy: u8| SaveOptions { id_entropy: Some([entropy; 16]), ..SaveOptions::default() }; + let first = write_full(&doc, &with(1)).unwrap(); + assert_eq!(first, write_full(&doc, &with(1)).unwrap(), "the same entropy gives the same bytes"); + let other = write_full(&doc, &with(2)).unwrap(); + assert_ne!(id_strings(&reopen(first)), id_strings(&reopen(other)), "different entropy gives a different identifier"); +} + +#[test] +fn an_encrypted_rewrite_keeps_the_first_identifier_string_its_key_depends_on() { + for algorithm in [Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = reopen(source(true)); + doc.set_encryption(&NewEncryption { + algorithm, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + let protected = write_full(&doc, &SaveOptions::default()).unwrap(); + let before = Document::open_with_password(Arc::new(protected), Some("user")).unwrap(); + + let mut doc = before.clone(); + doc.require_full_save_with_new_id(); + let out = write_incremental(&doc, &SaveOptions::default()).unwrap(); + // The output opens with the same password only if its first identifier string is unchanged. + let back = Document::open_with_password(Arc::new(out), Some("user")).unwrap(); + let (old, new) = (id_strings(&before), id_strings(&back)); + assert_eq!(new.len(), 2, "{algorithm:?}"); + assert_eq!(new.first(), old.first(), "{algorithm:?}: the key depends on the first string"); + assert_ne!(new.get(1), old.get(1), "{algorithm:?}: the second string is new"); + } +} + +/// The `/ID` shapes whose key the reader takes as the empty string: none at all, one entry, and a +/// first entry that is not a string. +fn unusable_ids() -> Vec<(&'static str, Option)> { + let empty = Object::String(PdfString::literal(Vec::new())); + vec![ + ("absent", None), + ("one entry", Some(Object::Array(vec![empty.clone()]))), + ("first entry not a string", Some(Object::Array(vec![Object::Int(7), empty]))), + ] +} + +/// An encrypted file whose key is derived from the empty string, with its trailer `/ID` replaced by `id`. +fn protected_with_id(algorithm: Algorithm, id: Option) -> Vec { + let mut doc = reopen(source_with(Some("/ID [<> <>]"))); + doc.set_encryption(&NewEncryption { + algorithm, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + match id { + Some(id) => doc.trailer_mut().set(b"ID".to_vec(), id), + None => { + doc.trailer_mut().remove(b"ID"); + } + } + write_full(&doc, &SaveOptions::default()).unwrap() +} + +#[test] +fn an_encrypted_rewrite_without_a_usable_identifier_still_gets_a_new_one() { + // The key is derived from the first string of /ID, and a missing /ID, a one-entry /ID and a + // first entry that is not a string all count as the empty string. The rewrite keeps that + // (empty) first string, so the file still opens with its password, and writes a new second one. + for algorithm in [Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + for (shape, id) in unusable_ids() { + let before = Document::open_with_password(Arc::new(protected_with_id(algorithm, id)), Some("user")).unwrap(); + let mut doc = before.clone(); + doc.require_full_save_with_new_id(); + let out = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open_with_password(Arc::new(out), Some("user")).unwrap(); + let ids = id_strings(&back); + let previous = id_strings(&before); + assert_eq!(ids.len(), 2, "{algorithm:?}, {shape}: the rewrite writes a pair"); + assert!(ids.first().is_some_and(|s| s.is_empty()), "{algorithm:?}, {shape}: the first string is the one the key uses"); + assert_eq!(ids.get(1).map(Vec::len), Some(16), "{algorithm:?}, {shape}"); + assert_ne!(ids.get(1), previous.get(1), "{algorithm:?}, {shape}: the second string is new"); + } + } +} + +/// The standard fixture rewritten for a new identifier, with `id_entropy` of 0x42. +fn fixed_entropy_rewrite() -> Vec { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + write_full(&doc, &SaveOptions { id_entropy: Some([0x42; 16]), ..SaveOptions::default() }).unwrap() +} + +#[test] +fn a_fixed_entropy_gives_the_same_identifier_every_time() { + // The formula itself is pinned on fixed bytes in the writer's tests. The written file is not the + // same on every platform, so no identifier of a rewrite can be pinned here. + let first = id_strings(&reopen(fixed_entropy_rewrite())); + assert_eq!(first.len(), 2); + assert!(first.iter().all(|s| s.len() == 16), "{first:?}"); + assert_eq!(first, id_strings(&reopen(fixed_entropy_rewrite()))); +} + +#[test] +fn the_identifier_depends_on_the_whole_rewritten_file() { + // Same caller entropy and same length; the two rewrites differ only in one byte of a stream + // that lies past the first 64 KiB of the output. Their identifiers must still differ. + let opts = SaveOptions { id_entropy: Some([9; 16]), ..SaveOptions::default() }; + let rewrite = |last: u8| { + let mut doc = reopen(large_source(last)); + doc.require_full_save_with_new_id(); + write_full(&doc, &opts).unwrap() + }; + let (a, b) = (rewrite(b'A'), rewrite(b'B')); + assert_eq!(a.len(), b.len(), "the two rewrites have the same length"); + let first_difference = a.iter().zip(&b).position(|(x, y)| x != y).unwrap_or(usize::MAX); + assert!(first_difference > 64 * 1024, "the rewrites first differ at byte {first_difference}, inside the first 64 KiB"); + assert_ne!(id_strings(&reopen(a)), id_strings(&reopen(b)), "content past 64 KiB must change the identifier"); +} + +/// Where [`stand_in_for_a_fresh_process`] writes its rewrite. +const ID_PROBE: &str = "PDFCRAFT_TEST_ID_PROBE"; + +/// The child half of [`separate_fresh_processes_get_different_identifiers`]: it runs alone, in its +/// own process, and writes a default-path rewrite of the standard fixture to the file the parent +/// named. Without the variable it is a plain no-op. +#[test] +fn stand_in_for_a_fresh_process() { + let Some(path) = std::env::var_os(ID_PROBE) else { return }; + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + std::fs::write(path, write_full(&doc, &SaveOptions::default()).unwrap()).unwrap(); +} + +#[test] +fn separate_fresh_processes_get_different_identifiers() { + // The default identifier is meant to differ between fresh instances of the whole save pipeline, + // not only between two calls in one process (which the counter already separates). On the native + // test target a fresh instance is a fresh process, whose hash keys the operating system seeds; + // the fix under test runs on WebAssembly, where no browser test harness exists — there the + // browser's `crypto.getRandomValues` stands in for this seeding, and this test cannot see it. + let exe = std::env::current_exe().unwrap(); + let probe = |name: &str| { + let path = std::env::temp_dir().join(format!("pdfcraft-id-probe-{}-{name}.pdf", std::process::id())); + let out = std::process::Command::new(&exe) + .args(["--exact", "stand_in_for_a_fresh_process", "--nocapture", "--"]) + .env(ID_PROBE, &path) + .output() + .expect("run this test's own binary as a child"); + assert!(out.status.success(), "{name}: the child failed: {}", String::from_utf8_lossy(&out.stderr)); + let saved = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: the child wrote its file: {e}")); + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2, "{name}"); + assert!(ids.iter().all(|s| s.len() == 16), "{name}: {ids:?}"); + let _ = std::fs::remove_file(&path); + ids + }; + let (first, second) = (probe("a"), probe("b")); + assert_ne!(first, second, "two fresh processes must not share an identifier"); +} diff --git a/crates/cos/tests/it/encryption.rs b/crates/cos/tests/it/encryption.rs index 2ee33afc12..b19d618f5c 100644 --- a/crates/cos/tests/it/encryption.rs +++ b/crates/cos/tests/it/encryption.rs @@ -494,3 +494,826 @@ fn a_stream_labelled_xref_is_encrypted_when_saved() { let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); assert!(!bytes.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the labelled stream was written in the clear"); } + +/// The content stream's reference. +fn contents_of(doc: &Document) -> ObjRef { + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let pages = doc.get(catalog.reference(b"Pages").unwrap()).as_dict().cloned().unwrap(); + let page = doc.resolve(&pages.get(b"Kids").unwrap().as_array().unwrap()[0]).as_dict().cloned().unwrap(); + page.reference(b"Contents").unwrap() +} + +/// The content stream's data as loaded, whatever its filters say. +fn raw_content(doc: &Document) -> String { + match &*doc.get(contents_of(doc)) { + Object::Stream(s) => String::from_utf8_lossy(&s.raw).trim().to_string(), + _ => panic!("no content stream"), + } +} + +/// Gives the content stream the `/Filter` and `/DecodeParms` of `selector`. +fn select(doc: &mut Document, (filter, parms): (Object, Object)) { + let contents = contents_of(doc); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Filter".to_vec(), filter); + s.dict.set(b"DecodeParms".to_vec(), parms); + doc.set(contents, Object::Stream(s)); +} + +/// `/Crypt` naming `name`, in the `form`th spelling: all direct, then each part by reference. +/// `None` past the last. +fn selector(doc: &mut Document, form: usize, name: &str) -> Option<(Object, Object)> { + let crypt = Object::name("Crypt"); + let parms = |n: Object| { + let mut d = pdfcraft_cos::Dict::new(); + d.set(b"Name".to_vec(), n); + Object::Dict(d) + }; + let named = parms(Object::name(name)); + Some(match form { + 0 => (crypt, named), + 1 => (crypt, Object::Ref(doc.add(named))), + 2 => (Object::Array(vec![crypt]), Object::Array(vec![Object::Ref(doc.add(named))])), + 3 => (Object::Array(vec![crypt]), Object::Ref(doc.add(Object::Array(vec![named])))), + 4 => (crypt, parms(Object::Ref(doc.add(Object::name(name))))), + 5 => (Object::Ref(doc.add(crypt)), named), + 6 => (Object::Ref(doc.add(Object::Array(vec![crypt]))), Object::Array(vec![named])), + 7 => (Object::Array(vec![Object::Ref(doc.add(crypt))]), Object::Array(vec![named])), + // Behind a filter given by reference: places in the two arrays pair up all the same. + 8 => (Object::Array(vec![Object::Ref(doc.add(Object::name("ASCIIHexDecode"))), crypt]), Object::Array(vec![Object::Null, named])), + // The two entries in different forms, which is how a lone filter is usually spelled. + 9 => (Object::Array(vec![crypt]), named), + 10 => (crypt, Object::Array(vec![named])), + // The filter twice, which names the first's. + 11 => (Object::Array(vec![crypt.clone(), crypt]), Object::Array(vec![named.clone(), named])), + // A reference to a reference. + 12 => { + let name = doc.add(Object::name(name)); + let hop = doc.add(Object::Ref(name)); + (crypt, parms(Object::Ref(hop))) + } + _ => return None, + }) +} + +fn holds(bytes: &[u8], text: &str) -> bool { + String::from_utf8_lossy(bytes).contains(text) +} + +/// An object `/Encrypt` names is written as stored only while it is as stored. A file whose +/// `/Encrypt` names its `/Info` reads that dictionary as stored; once the title is edited, both +/// saves encrypt it rather than writing the new title in the clear. +#[test] +fn an_edited_object_that_encrypt_names_is_encrypted() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + let mut dict = doc.get(doc.trailer().reference(b"Encrypt").unwrap()).as_dict().cloned().unwrap(); + dict.set(b"Junk".to_vec(), Object::Ref(info)); + doc.trailer_mut().set(b"Encrypt".to_vec(), Object::Dict(dict)); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d| d.set(b"Title".to_vec(), PdfString::text("Merger plan"))).unwrap(); + for (save, saved) in [("incremental", write_incremental(&doc, &SaveOptions::default())), ("full", write_full(&doc, &SaveOptions::default()))] { + assert!(!holds(&saved.unwrap(), "Merger plan"), "{save}: the edited title was written in the clear"); + } +} + +/// A stream can select its crypt filter by reference at every step: the chain, an entry of it, the +/// parameters, an entry of those, the name (ISO 32000-2 §7.3.10, §7.4.10). Read as absent, the +/// filter was `/Identity`, so such a stream was saved in the clear and, in a file from elsewhere, +/// its ciphertext read as data. A full save renumbers: what it follows has the document's numbers. +#[test] +fn a_crypt_filter_selected_by_reference_is_used() { + for alg in [Algorithm::Aes128, Algorithm::Aes256] { + for form in 0.. { + let mut doc = Document::open(Arc::new(protect(alg, "", "owner", -1))).unwrap(); + let Some(selected) = selector(&mut doc, form, "StdCF") else { break }; + select(&mut doc, selected); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&saved, "Secret page"), "{alg:?} form {form}: saved encrypted"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "{alg:?} form {form}: read back"); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(!holds(&full, "Secret page"), "{alg:?} form {form}: renumbered, encrypted"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "{alg:?} form {form}: renumbered"); + } + } +} + +/// A reference to a stream that takes `follows` references to read cold, this one included: its +/// nine filters are all read, the last leading to the next stream. +fn spends(doc: &mut Document, follows: usize) -> Object { + let left = follows - 1; + let mut filters: Vec = (0..left.min(8)).map(|i| Object::Ref(doc.add(Object::Int(i as i64)))).collect(); + if left > 8 { + filters.push(spends(doc, left - 8)); + } + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(filters)); + Object::Ref(doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(dict, Vec::new())))) +} + +/// Gives the content stream a `/Type` that takes `follows` references to read, ahead of its filter. +fn spend_first(doc: &mut Document, follows: usize) { + let spent = spends(doc, follows); + let contents = contents_of(doc); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Type".to_vec(), spent); + doc.set(contents, Object::Stream(s)); +} + +/// Past what one read may follow a reference reads as absent, and a crypt filter with no `/Name` is +/// `/Identity`. Together those would read this stream's ciphertext as data, so what can't be told +/// is the document's own filter, as a `/Filter` that can't be read is. +#[test] +fn a_crypt_filter_past_what_a_read_may_follow_is_the_documents_own() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let selected = selector(&mut doc, 1, "StdCF").unwrap(); + select(&mut doc, selected); + spend_first(&mut doc, 40); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&saved, "Secret page"), "saved encrypted"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(!holds(&full, "Secret page"), "renumbered, encrypted"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "renumbered"); +} + +/// A read that takes all it may follow has refused nothing: a crypt filter with no `/Name` after it +/// is still `/Identity`, and the stream is in the clear. +#[test] +fn a_crypt_filter_with_no_name_after_all_a_read_may_follow_is_identity() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + select(&mut doc, (Object::name("Crypt"), Object::Dict(pdfcraft_cos::Dict::new()))); + spend_first(&mut doc, 32); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&saved, "Secret page"), "written as the stream asks"); + assert_eq!(raw_content(&Document::open(Arc::new(saved)).unwrap()), SECRET); +} + +/// `/Identity` selected by reference keeps a stream in the clear in a document that encrypts the +/// rest, in every spelling, on both saves. +#[test] +fn an_identity_crypt_filter_selected_by_reference_leaves_a_stream_alone() { + for form in 0.. { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let Some(selected) = selector(&mut doc, form, "Identity") else { break }; + select(&mut doc, selected); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&saved, "Secret page"), "form {form}: written as the stream asks"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "form {form}"); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(holds(&full, "Secret page"), "form {form}: renumbered"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "form {form}: renumbered"); + } +} + +/// New protection drops a crypt filter it doesn't define, whichever way the stream selected it. The +/// chain comes out direct. Parameters in the stream's own dictionary stay, encrypted with it; of an +/// array in an object of its own, which every stream can share, only what decoders read is copied. +#[test] +fn protecting_drops_a_crypt_filter_selected_by_reference() { + let protected = |doc: &mut Document| { + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + write_full(doc, &SaveOptions::default()).unwrap() + }; + for form in 0.. { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let Some(selected) = selector(&mut doc, form, "MyCF") else { break }; + select(&mut doc, selected); + let bytes = protected(&mut doc); + assert!(!holds(&bytes, "Secret page"), "form {form}: encrypted"); + assert!(!holds(&bytes, "MyCF"), "form {form}: no selector left in the stream"); + let back = Document::open(Arc::new(bytes.clone())).unwrap(); + assert_eq!(raw_content(&back), SECRET, "form {form}"); + let contents = back.get(contents_of(&back)); + let filter = contents.as_dict().unwrap().get(b"Filter"); + match form { + 8 => assert!( + matches!(filter, Some(Object::Array(a)) if matches!(a.as_slice(), [Object::Ref(_)])), + "form 8: the other filter stays: {filter:?}" + ), + _ => assert_eq!(filter, None, "form {form}"), + } + assert!(hayro_syntax::Pdf::new(bytes).is_ok(), "form {form}"); + } + for shared in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut named = pdfcraft_cos::Dict::new(); + named.set(b"Name".to_vec(), Object::name("MyCF")); + let mut other = pdfcraft_cos::Dict::new(); + other.set(b"Note".to_vec(), Object::String(PdfString::text("Private note"))); + other.set(b"Predictor".to_vec(), Object::Int(1)); + let parms = Object::Array(vec![Object::Dict(named), Object::Dict(other)]); + let parms = if shared { Object::Ref(doc.add(parms)) } else { parms }; + select(&mut doc, (Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")]), parms)); + let bytes = protected(&mut doc); + assert!(!holds(&bytes, "Private note"), "shared: {shared}"); + let back = Document::open(Arc::new(bytes)).unwrap(); + let contents = back.get(contents_of(&back)); + let kept = contents.as_dict().unwrap().get(b"DecodeParms").unwrap().as_array().unwrap(); + assert_eq!(kept.len(), 1, "one filter left, one entry"); + let kept = kept[0].as_dict().unwrap(); + assert_eq!(kept.get(b"Predictor"), Some(&Object::Int(1)), "shared: {shared}"); + assert_eq!(kept.get(b"Note").is_some(), !shared, "shared: {shared}"); + } +} + +/// Every stream of a file can share one large object as its `/DecodeParms` or `/Filter`, which +/// costs nothing until new protection drops a crypt filter from each. Copied whole, a 1 MB file +/// came out at 1 GB. +#[test] +fn protecting_copies_no_large_object_into_every_stream() { + let pad = Object::Array(vec![Object::Int(1); 100_000]); + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut large = pdfcraft_cos::Dict::new(); + large.set(b"Pad".to_vec(), pad.clone()); + let parms = doc.add(Object::Array(vec![Object::Null, Object::Dict(large)])); + let filter = doc.add(Object::Array(vec![Object::name("Crypt"), Object::Name(vec![b'A'; 100_000]), pad])); + let mut streams = Vec::new(); + for _ in 0..200 { + let mut by_parms = pdfcraft_cos::Dict::new(); + by_parms.set(b"Filter".to_vec(), Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")])); + by_parms.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + streams.push(Object::Ref(doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(by_parms, b"41>".to_vec()))))); + let mut by_filter = pdfcraft_cos::Dict::new(); + by_filter.set(b"Filter".to_vec(), Object::Ref(filter)); + streams.push(Object::Ref(doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(by_filter, b"41>".to_vec()))))); + } + doc.update_dict(doc.root().unwrap(), |c| c.set(b"Test".to_vec(), Object::Array(streams))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(bytes.len() < 1 << 20, "{} bytes", bytes.len()); +} + +/// A file can name one large string thousands of times in `/ID`, which holds two. An encrypted file +/// keeps its `/ID`, so each copy would be in the file a save writes. +#[test] +fn a_file_identifier_of_many_references_is_two_strings() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let large = doc.add(Object::String(PdfString { bytes: vec![0xAB; 100_000], hex: true })); + doc.trailer_mut().set(b"ID".to_vec(), Object::Array(vec![Object::Ref(large); 500])); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(bytes.len() < 1 << 20, "{} bytes", bytes.len()); +} + +/// What a file can do with those references: point a stream's selector at the stream itself, or at +/// nothing. They read as absent, so a stream with `/Crypt` is `/Identity` (ISO 32000-2 §7.4.10), +/// and one that names no filter follows the document's. Loading ends either way. +#[test] +fn a_crypt_filter_reference_to_itself_or_nothing_reads_as_absent() { + let named = |n: Object| { + let mut d = pdfcraft_cos::Dict::new(); + d.set(b"Name".to_vec(), n); + Object::Dict(d) + }; + for (what, encrypted) in [ + ("/Filter to itself", true), + ("/Filter nowhere", true), + ("/DecodeParms to itself", false), + ("/DecodeParms nowhere", false), + ("/Name to itself", false), + ("/Name nowhere", false), + ] { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let me = Object::Ref(contents_of(&doc)); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + let crypt = Object::name("Crypt"); + select( + &mut doc, + match what { + "/Filter to itself" => (me, Object::Null), + "/Filter nowhere" => (nowhere, Object::Null), + "/DecodeParms to itself" => (crypt, me), + "/DecodeParms nowhere" => (crypt, nowhere), + "/Name to itself" => (crypt, named(me)), + _ => (crypt, named(nowhere)), + }, + ); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert_eq!(!holds(&saved, "Secret page"), encrypted, "{what}"); + assert_eq!(raw_content(&Document::open(Arc::new(saved)).unwrap()), SECRET, "{what}"); + } +} + +/// A lone parameter dictionary belongs to the first filter, so `/Crypt` behind another filter has +/// none of its own and is `/Identity` (ISO 32000-2 §7.4.1), whatever that dictionary says. +#[test] +fn a_lone_parameter_dictionary_is_the_first_filters() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let mut parms = pdfcraft_cos::Dict::new(); + parms.set(b"Name".to_vec(), Object::name("StdCF")); + let first = Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")]); + let second = Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("Crypt")]); + select(&mut doc, (first, Object::Dict(parms.clone()))); + assert!(!holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "first: it names StdCF"); + select(&mut doc, (second, Object::Dict(parms))); + assert!(holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "second: it names nothing"); +} + +/// A lone parameter dictionary is the first filter's, so dropping a `/Crypt` behind that filter +/// leaves it, and dropping one in front of it takes it away. +#[test] +fn protecting_keeps_the_parameters_of_the_filter_before_a_crypt_filter() { + let protected = |filters: Vec| { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut parms = pdfcraft_cos::Dict::new(); + parms.set(b"Name".to_vec(), Object::name("MyCF")); + select(&mut doc, (Object::Array(filters), Object::Dict(parms))); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let back = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + back.get(contents_of(&back)).as_dict().unwrap().contains(b"DecodeParms") + }; + let (hex, crypt) = (Object::name("ASCIIHexDecode"), Object::name("Crypt")); + assert!(protected(vec![hex.clone(), crypt.clone()]), "the first filter's, which stays"); + assert!(!protected(vec![crypt, hex]), "/Crypt's, which goes"); +} + +/// Dropping `/Crypt` takes its entry out of `/DecodeParms`, and an array too short to have one is +/// left as it is, given directly or by reference. +#[test] +fn protecting_leaves_a_parameter_array_too_short_for_the_crypt_filter() { + for by_reference in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let short = Object::Array(vec![Object::Null]); + let parms = if by_reference { Object::Ref(doc.add(short)) } else { short }; + select(&mut doc, (Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("Crypt")]), parms)); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&bytes, "Secret page"), "by reference: {by_reference}"); + let back = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "by reference: {by_reference}"); + let contents = back.get(contents_of(&back)); + let filter = contents.as_dict().unwrap().get(b"Filter").unwrap().as_array().unwrap(); + assert_eq!(filter.len(), 1, "only /Crypt goes"); + assert_eq!(filter[0].as_name(), Some(b"ASCIIHexDecode".as_slice()), "the other filter stays as it was"); + } +} + +/// A chain of up to [`pdfcraft_filters::MAX_CHAIN`] filters and `/Crypt` is read; a longer one +/// names no crypt filter, so its entries aren't followed for every stream sharing it. +#[test] +fn a_chain_longer_than_any_decoder_takes_names_no_crypt_filter() { + let chain = |others: usize| { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let mut filters = vec![Object::name("ASCIIHexDecode"); others]; + filters.push(Object::name("Crypt")); + let array = doc.add(Object::Array(filters)); + select(&mut doc, (Object::Ref(array), Object::Null)); + holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page") + }; + assert!(chain(pdfcraft_filters::MAX_CHAIN), "the longest chain read: /Identity, in the clear"); + assert!(!chain(pdfcraft_filters::MAX_CHAIN + 1), "one more: the document's filter"); +} + +/// A stream's `/Type` by reference picks its handling like one given directly: metadata left +/// readable under `/EncryptMetadata false`, an embedded file under `/EFF`. +#[test] +fn a_stream_type_given_by_reference_picks_its_crypt_filter() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let meta = catalog.reference(b"Metadata").unwrap(); + let ty = doc.add(Object::name("Metadata")); + let Object::Stream(mut s) = doc.get(meta).as_ref().clone() else { panic!("no metadata stream") }; + s.dict.set(b"Type".to_vec(), Object::Ref(ty)); + doc.set(meta, Object::Stream(s)); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "o", + permissions: -1, + encrypt_metadata: false, + seed: [5; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&bytes, "meta"), "search engines can read XMP"); + + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + doc.update_dict(encrypt, |e| e.set(b"EFF".to_vec(), Object::name("Identity"))).unwrap(); + let mut doc = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let contents = contents_of(&doc); + let ty = doc.add(Object::name("EmbeddedFile")); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Type".to_vec(), Object::Ref(ty)); + doc.set(contents, Object::Stream(s)); + assert!(holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "/EFF /Identity leaves it alone"); +} + +/// What makes a dictionary a signature's can be given by reference, and what is null or refers to +/// nothing is absent: `/Contents` stays as written in each of these. +#[test] +fn a_signature_dictionary_is_known_in_every_form() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let ty = doc.add(Object::name("Sig")); + let range = Object::Array([0, 10, 20, 30].into_iter().map(Object::Int).collect()); + let by_reference = doc.add(range.clone()); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + let forms: [(Vec<(&str, Object)>, &str); 5] = [ + (vec![("Type", Object::Ref(ty))], "DEAD"), + (vec![("ByteRange", Object::Ref(by_reference))], "BEEF"), + (vec![("Type", Object::Null), ("ByteRange", range.clone())], "CAFE"), + (vec![("Type", nowhere.clone()), ("ByteRange", range.clone())], "F00D"), + (vec![("Subtype", nowhere), ("ByteRange", range)], "D00D"), + ]; + let mut signatures = Vec::new(); + for (entries, contents) in forms { + let mut sig = pdfcraft_cos::Dict::new(); + for (key, value) in entries { + sig.set(key.as_bytes().to_vec(), value); + } + let bytes = (0..contents.len()).step_by(2).map(|i| u8::from_str_radix(&contents[i..i + 2], 16).unwrap()).collect(); + sig.set(b"Contents".to_vec(), Object::String(PdfString { bytes, hex: true })); + signatures.push(Object::Ref(doc.add(Object::Dict(sig)))); + } + // Nested in another object, it follows nothing, but known directly it is one all the same. + let mut nested = pdfcraft_cos::Dict::new(); + nested.set(b"Type".to_vec(), Object::name("Sig")); + nested.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0xAB, 0xBA], hex: true })); + let mut field = pdfcraft_cos::Dict::new(); + field.set(b"V".to_vec(), Object::Dict(nested)); + signatures.push(Object::Ref(doc.add(Object::Dict(field)))); + // Neither of what makes one: a string /Contents alone is anybody's, so it's encrypted. + let mut other = pdfcraft_cos::Dict::new(); + other.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0xAB, 0xCD], hex: true })); + signatures.push(Object::Ref(doc.add(Object::Dict(other)))); + doc.update_dict(doc.root().unwrap(), |c| c.set(b"Test".to_vec(), Object::Array(signatures))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + for contents in ["DEAD", "BEEF", "CAFE", "F00D", "D00D", "ABBA"] { + assert!(holds(&bytes, &format!("<{contents}>")), "{contents}: written as is"); + } + assert!(!holds(&bytes, ""), "without a /ByteRange it is no signature"); +} + +/// What a test needs to write an update of a classic file protected with AES-128. +struct Classic { + bytes: Vec, + root: ObjRef, + pages: ObjRef, + info: ObjRef, + encrypt: ObjRef, + id: Vec, + dict: pdfcraft_cos::Dict, +} + +fn classic_file() -> Classic { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + let id = doc.trailer().get(b"ID").unwrap().as_array().unwrap()[0].as_string().unwrap().bytes.clone(); + let dict = doc.get(encrypt).as_dict().cloned().unwrap(); + let pages = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Pages").unwrap(); + Classic { root: doc.root().unwrap(), pages, info: doc.trailer().reference(b"Info").unwrap(), bytes, encrypt, id, dict } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02X}")).collect() +} + +/// `bytes` with an update appended: `objects`, and a trailer of `trailer` entries, in which `@n` +/// stands for the offset of object `n`. An object with nothing in it gets an entry of the +/// cross-reference table that points at nothing. +fn updated(bytes: &[u8], objects: &[(u32, Vec)], trailer: &str) -> Vec { + let at = bytes.windows(9).rposition(|w| w == b"startxref").unwrap(); + let prev: usize = String::from_utf8_lossy(&bytes[at + 9..]).split_whitespace().next().unwrap().parse().unwrap(); + let mut out = bytes.to_vec(); + let mut table = String::from("xref\n"); + let mut trailer = trailer.to_string(); + for (num, body) in objects { + if body.is_empty() { + table.push_str(&format!("{num} 1\n0000000010 00000 n \n")); + continue; + } + trailer = trailer.replace(&format!("@{num}"), &out.len().to_string()); + table.push_str(&format!("{num} 1\n{:010} 00000 n \n", out.len())); + out.extend_from_slice(format!("{num} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(table.as_bytes()); + out.extend_from_slice(format!("trailer\n<< {trailer} /Size 2000 /Prev {prev} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + out +} + +/// The `/Encrypt` dictionary of `file`, with `/O`, `/StmF` and `/StrF` as given. +fn encrypt_with(file: &Classic, o: &str, filter: &str) -> Vec { + let string = |key: &[u8]| hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes); + format!( + "<< /Filter /Standard /V 4 /R 4 /Length 128 /P {} /O {o} /U <{}> /CF << /StdCF << /CFM /AESV2 /Length 16 /AuthEvent /DocOpen >> >> /StmF /{filter} /StrF /{filter} >>", + file.dict.int(b"P").unwrap(), + string(b"U") + ) + .into_bytes() +} + +/// An object stream can have its parameters by reference, and a stream's crypt filter can be +/// selected by a member of it: legal together, though each is rare. What reads the stream first +/// goes through the object stream, and members must come out of it as they do read directly. They +/// didn't, while a read made for a selector wasn't allowed to follow its own: the object stream was +/// kept undecoded, its members lost for good, and a full save wrote nulls without saying so. +#[test] +fn an_object_stream_reached_from_a_selector_keeps_its_members() { + let file = classic_file(); + let parms = "[<< /Name /Identity >> null]"; + let header = format!("902 0 903 {} ", parms.len() + 1); + let members = format!("{header}{parms} << /Keep (yes) >>"); + let (mut rows, mut before) = (vec![1u8], 0u8); + for b in members.bytes() { + rows.push(b.wrapping_sub(before)); + before = b; + } + let coded = pdfcraft_cos::Stream::flate(pdfcraft_cos::Dict::new(), &rows).raw; + let mut object_stream = + format!("<< /Type /ObjStm /N 2 /First {} /Filter /FlateDecode /DecodeParms 911 0 R /Length {} >>\nstream\n", header.len(), coded.len()) + .into_bytes(); + object_stream.extend_from_slice(&coded); + object_stream.extend_from_slice(b"\nendstream"); + let data = [2u8, 0x03, 0x8E, 0, 2, 0x03, 0x8E, 1]; + let mut xref = format!("<< /Type /XRef /Size 2000 /W [1 2 1] /Index [902 2] /Length {} >>\nstream\n", data.len()).into_bytes(); + xref.extend_from_slice(&data); + xref.extend_from_slice(b"\nendstream"); + let string = |key: &[u8]| format!("<{}>", hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes)); + let objects = [ + (file.root.num, format!("<< /Type /Catalog /Pages {} 0 R /A 900 0 R /B 903 0 R >>", file.pages.num).into_bytes()), + (file.encrypt.num, encrypt_with(&file, &string(b"O"), "Identity")), + (900, b"<< /Filter [/Crypt /ASCIIHexDecode] /DecodeParms 902 0 R /Length 1 >>\nstream\n>\nendstream".to_vec()), + (910, object_stream), + (911, format!("<< /Predictor 11 /Columns {} >>", members.len()).into_bytes()), + (912, xref), + ]; + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>] /XRefStm @912", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let keep = + |doc: &Document, r: &Object| doc.resolve(r).as_dict().and_then(|d| d.get(b"Keep")).and_then(Object::as_string).map(|s| s.bytes.clone()); + let member = Object::Ref(ObjRef::new(903, 0)); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert_eq!(keep(&doc, &member).as_deref(), Some(b"yes".as_slice()), "read first"); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + let _ = doc.get(ObjRef::new(900, 0)); + assert_eq!(keep(&doc, &member).as_deref(), Some(b"yes".as_slice()), "read after the stream selecting from it"); + let saved = write_full(&Document::open(Arc::new(bytes)).unwrap(), &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + let catalog = back.get(back.root().unwrap()); + assert_eq!(keep(&back, catalog.as_dict().unwrap().get(b"B").unwrap()).as_deref(), Some(b"yes".as_slice()), "after a full save"); +} + +/// `walk` goes as deep as an object nests, so following a reference from down there adds that depth +/// again for each object it loads: this file, under 10 KB, overflowed a 2 MiB stack and took the +/// process with it. Only the object itself follows references, and it does so in very little stack. +#[test] +fn nested_dictionaries_load_nothing_on_a_small_stack() { + let file = classic_file(); + let first = 960u32; + let objects: Vec<(u32, Vec)> = (first..first + 40) + .map(|num| { + let open = "[".repeat(120); + let close = "]".repeat(120); + (num, format!("{open}<< /Contents () /Type {} 0 R >>{close}", num + 1).into_bytes()) + }) + .collect(); + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let reading = std::thread::Builder::new().stack_size(1 << 20).spawn(move || { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.get(ObjRef::new(first, 0)).as_array().is_some() + }); + assert_eq!(reading.unwrap().join().ok(), Some(true)); +} + +/// The object itself can follow references, a ladder of them as long as loading may nest, and that +/// fits a small stack too. +#[test] +fn a_ladder_of_signature_checks_fits_a_small_stack() { + let file = classic_file(); + let first = 960u32; + let objects: Vec<(u32, Vec)> = + (first..first + 40).map(|num| (num, format!("<< /Contents () /Type {} 0 R >>", num + 1).into_bytes())).collect(); + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let reading = std::thread::Builder::new().stack_size(1 << 20).spawn(move || { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.get(ObjRef::new(first, 0)).as_dict().is_some() + }); + assert_eq!(reading.unwrap().join().ok(), Some(true)); +} + +/// The strings of `/Encrypt` are never encrypted (ISO 32000-2 §7.6.2), and can be objects of their +/// own. Read as ordinary strings they were decrypted, and a full save encrypted them again under +/// another number, so the file it wrote no longer opened with its own password. +#[test] +fn encrypt_strings_given_by_reference_stay_as_stored() { + let file = classic_file(); + let string = |key: &[u8]| hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes); + let replaced = format!( + "<< /Filter /Standard /V 4 /R 4 /Length 128 /P {} /O 900 0 R /U 901 0 R /CF << /StdCF << /CFM /AESV2 /Length 16 /AuthEvent /DocOpen >> >> /StmF /StdCF /StrF /StdCF >>", + file.dict.int(b"P").unwrap() + ); + let objects = [ + (file.encrypt.num, replaced.into_bytes()), + (900, format!("<{}>", string(b"O")).into_bytes()), + (901, format!("<{}>", string(b"U")).into_bytes()), + ]; + let trailer = + format!("/Root {} 0 R /Info {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.info.num, file.encrypt.num, id = hex(&file.id)); + let source = Document::open(Arc::new(updated(&file.bytes, &objects, &trailer))).unwrap(); + assert_eq!(content(&source), SECRET, "the file reads"); + assert_eq!(title(&source).as_deref(), Some("Quarterly numbers")); + for object_streams in [false, true] { + let saved = write_full(&source, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(content(&back), SECRET, "object streams: {object_streams}"); + assert_eq!(title(&back).as_deref(), Some("Quarterly numbers"), "object streams: {object_streams}"); + } +} + +/// A save keeps what a file is protected with: its `/Encrypt`, which holds the handler and the +/// permissions, its `/ID`, and so its file key, under every algorithm and by either kind of save. +#[test] +fn a_save_keeps_the_handler_the_key_and_the_permissions() { + for alg in ALL { + let doc = Document::open_with_password(Arc::new(protect(alg, "user", "owner", -1 ^ 0b1_0000)), Some("user")).unwrap(); + let held = |d: &Document| { + let encrypt = d.resolve(d.trailer().get(b"Encrypt").unwrap()).as_dict().unwrap().clone(); + (encrypt, d.trailer().get(b"ID").cloned(), d.security().unwrap().file_key().to_vec(), format!("{:?}", d.permissions())) + }; + let before = held(&doc); + let mut edited = doc.clone(); + let _ = edited.add(Object::Int(1)); + for (save, saved) in [ + ("incremental", write_incremental(&edited, &SaveOptions::default()).unwrap()), + ("full", write_full(&doc, &SaveOptions::default()).unwrap()), + ] { + let back = Document::open_with_password(Arc::new(saved), Some("user")).unwrap(); + assert!(held(&back) == before, "{alg:?}: a {save} save"); + assert_eq!(content(&back), SECRET, "{alg:?}: a {save} save"); + } + } +} + +/// `/ID` has to be direct in an encrypted file, but one that isn't has its key made from the same +/// bytes. A full save writes it direct, and an incremental one leaves it as it was. +#[test] +fn a_file_identifier_given_by_reference_is_read() { + let file = classic_file(); + let id = hex(&file.id); + let reference = |entries: &str, objects: &[(u32, Vec)]| { + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID {entries}", file.root.num, file.encrypt.num); + updated(&file.bytes, objects, &trailer) + }; + let whole = reference("900 0 R", &[(900, format!("[<{id}> <{id}>]").into_bytes())]); + let entries = reference("[900 0 R 901 0 R]", &[(900, format!("<{id}>").into_bytes()), (901, format!("<{id}>").into_bytes())]); + for file in [whole, entries] { + let source = Document::open(Arc::new(file)).unwrap(); + assert_eq!(content(&source), SECRET); + let back = Document::open(Arc::new(write_full(&source, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(content(&back), SECRET); + assert!(matches!(back.trailer().get(b"ID"), Some(Object::Array(_))), "written direct"); + let mut edited = source.clone(); + let _ = edited.add(Object::Int(1)); + let back = Document::open(Arc::new(write_incremental(&edited, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(content(&back), SECRET, "an incremental save"); + assert_eq!(back.trailer().get(b"ID"), edited.trailer().get(b"ID"), "and leaves it as it was"); + } +} + +/// A direct `/Encrypt` can refer to objects. A full save dropped those it found only there, and kept +/// its references as they were under the old numbers, so nothing in the file read afterwards. +#[test] +fn a_direct_encrypt_dictionary_keeps_what_it_refers_to() { + for object_streams in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let mut dict = doc.get(doc.trailer().reference(b"Encrypt").unwrap()).as_dict().cloned().unwrap(); + let filter = doc.add(Object::name("StdCF")); + let filters = doc.add(dict.get(b"CF").unwrap().clone()); + dict.set(b"StmF".to_vec(), Object::Ref(filter)); + dict.set(b"CF".to_vec(), Object::Ref(filters)); + doc.trailer_mut().set(b"Encrypt".to_vec(), Object::Dict(dict)); + let options = SaveOptions { object_streams, ..SaveOptions::default() }; + let saved = write_full(&doc, &options).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(content(&back), SECRET, "object streams: {object_streams}"); + let again = write_full(&back, &options).unwrap(); + assert_eq!(content(&Document::open(Arc::new(again)).unwrap()), SECRET, "object streams: {object_streams}, saved again"); + } +} + +/// How a stream is treated can be given by reference, and what that loads can be a stream too. A +/// file can make a ladder of those, where each rung reads the rung below, then two trees wider than +/// the object cache, which push that rung out of it, then the rung below again. Each rung would +/// take twice what the one below does, if one read could follow any number of references. +#[test] +fn selectors_that_lead_to_selectors_load_at_once() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let stream_of = |doc: &mut Document, filters: Vec| { + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(filters)); + doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(dict, Vec::new()))) + }; + let tree = |doc: &mut Document| { + let mut level: Vec = (0..6_561).map(|i| doc.add(Object::Int(i))).collect(); + while level.len() > 1 { + let chunks: Vec> = level.chunks(9).map(|c| c.iter().map(|r| Object::Ref(*r)).collect()).collect(); + level = chunks.into_iter().map(|c| stream_of(doc, c)).collect(); + } + Object::Ref(level[0]) + }; + let (first, second) = (tree(&mut doc), tree(&mut doc)); + let mut top = doc.add(Object::Int(0)); + // Twelve rungs of two, and the trees' five levels, stay inside the 32 that loading may nest. + for _ in 0..12 { + let (a, b) = (stream_of(&mut doc, vec![Object::Ref(top)]), stream_of(&mut doc, vec![Object::Ref(top)])); + top = stream_of(&mut doc, vec![Object::Ref(a), first.clone(), second.clone(), Object::Ref(b)]); + } + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let doc = Document::open(Arc::new(saved)).unwrap(); + let _ = done.send(doc.get(top).as_dict().is_some()); + }); + // With the budget it takes milliseconds, and without it longer than anyone would wait, so a + // margin this wide can't fail on a busy machine. + assert_eq!(finished.recv_timeout(std::time::Duration::from_secs(120)), Ok(true), "loading must not fan out"); +} diff --git a/crates/cos/tests/it/generations.rs b/crates/cos/tests/it/generations.rs new file mode 100644 index 0000000000..bbe2ff0b33 --- /dev/null +++ b/crates/cos/tests/it/generations.rs @@ -0,0 +1,510 @@ +//! An indirect reference identifies an object number AND its generation (ISO 32000-2 §7.3.10). + +use std::sync::Arc; + +use hayro_syntax::object::{Dict as HayroDict, ObjectIdentifier}; +use pdfcraft_cos::{Document, ObjRef, Object, SaveOptions, XrefEntry, write_full, write_incremental}; + +/// Object 3 was reused: only 3 1 R refers to the current value, never 3 0 R. +fn reused_object() -> Arc> { + object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 1, "42"), + ]) +} + +fn object_file(objects: &[(u32, u16, &str)]) -> Arc> { + object_file_with_entry(objects, None) +} + +fn object_file_with_entry(objects: &[(u32, u16, &str)], changed: Option<(u32, u16, bool)>) -> Arc> { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, generation, value) in objects { + offsets.push((*num, bytes.len(), *generation)); + bytes.extend_from_slice(format!("{num} {generation} obj\n{value}\nendobj\n").as_bytes()); + } + let xref = bytes.len(); + let size = objects.len() + 1; + bytes.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); + for (num, offset, generation) in offsets { + let (generation, free) = changed.filter(|(n, _, _)| *n == num).map_or((generation, false), |(_, g, free)| (g, free)); + let (offset, kind) = if free { (0, 'f') } else { (offset, 'n') }; + bytes.extend_from_slice(format!("{offset:010} {generation:05} {kind} \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size {size} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + Arc::new(bytes) +} + +fn contradictory_entry(free: bool) -> Arc> { + object_file_with_entry( + &[(1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "42")], + Some((3, if free { 2 } else { 0 }, free)), + ) +} + +#[test] +fn contradictory_xref_generation_recovers_the_matching_header_and_records_repair() { + for warm in [false, true] { + let doc = Document::open(contradictory_entry(false)).unwrap(); + if warm { + assert_eq!(*doc.try_get(3).unwrap(), Object::Int(42)); + } + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 1))), Object::Int(42)); + // Preserve the existing xref-matching behavior, but never accept an unrelated generation. + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); + assert!(!doc.xref_entry_is_exact(3), "a contradictory generation is not an exact xref entry"); + assert!( + doc.repair_log() + .iter() + .any(|line| { line.contains("object 3") && line.contains("xref generation 0") && line.contains("header generation 1") }), + "{:?}", + doc.repair_log() + ); + let repairs = doc.repair_log().to_vec(); + for _ in 0..3 { + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + } + assert_eq!(doc.repair_log(), repairs); + + let mut edited = doc.clone(); + edited.free(ObjRef::new(3, 1)); + for generation in [0, 1, 2] { + assert_eq!(*edited.get(ObjRef::new(3, generation)), Object::Null); + } + edited.set(ObjRef::new(3, 2), Object::Int(99)); + assert_eq!(*edited.get(ObjRef::new(3, 1)), Object::Null); + assert_eq!(*edited.get(ObjRef::new(3, 2)), Object::Int(99)); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + } +} + +/// A damaged row may name generation zero while the retained object and catalog use one. +fn contradictory_dictionary() -> Arc> { + object_file_with_entry( + &[(1, 0, "<< /Type /Catalog /Pages 2 0 R /Current 3 1 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "<< /Value 42 >>")], + Some((3, 0, false)), + ) +} + +fn assert_repaired_dictionary(bytes: Vec, value: i32) { + let bytes = Arc::new(bytes); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::>(b"Current").unwrap().get::(b"Value"), Some(value)); + let back = Document::open(bytes).unwrap(); + let catalog = back.get(back.root().unwrap()); + let current = back.resolve(catalog.as_dict().unwrap().get(b"Current").unwrap()); + assert_eq!(current.as_dict().unwrap().int(b"Value"), Some(i64::from(value))); +} + +#[test] +fn repaired_header_references_survive_full_saves_and_dictionary_edits() { + let original = contradictory_dictionary(); + let doc = Document::open(original.clone()).unwrap(); + for object_streams in [false, true] { + assert_repaired_dictionary(write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(), 42); + } + let mut edited = doc.clone(); + edited.update_dict(ObjRef::new(3, 1), |dict| dict.set(b"Value".to_vec(), Object::Int(77))).unwrap(); + assert_eq!(edited.generation(3), 1); + let incremental = write_incremental(&edited, &SaveOptions::default()).unwrap(); + assert!(edited.full_save_required(), "repair-aware dictionary edits normalize on full rewrite"); + assert_repaired_dictionary(incremental, 77); + for object_streams in [false, true] { + assert_repaired_dictionary(write_full(&edited, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(), 77); + } + let mut restored = doc.clone(); + restored.overlay_changes_from(&edited).unwrap(); + assert_repaired_dictionary(write_incremental(&restored, &SaveOptions::default()).unwrap(), 77); +} + +#[test] +fn free_xref_slots_never_revive_retained_object_headers() { + for warm in [false, true] { + let doc = Document::open(contradictory_entry(true)).unwrap(); + assert!(matches!(doc.xref_entry(3), Some(XrefEntry::Free { next_generation: 2 }))); + if warm { + assert_eq!(*doc.try_get(3).unwrap(), Object::Null); + } + for generation in [0, 1, 2] { + assert_eq!(*doc.get(ObjRef::new(3, generation)), Object::Null); + } + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 1))), Object::Null); + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + } + // Control: the retained bytes really are a readable object when its slot is live. + let live = Document::open(contradictory_entry(false)).unwrap(); + assert_eq!(*live.try_get(3).unwrap(), Object::Int(42)); +} + +#[test] +fn incremental_generation_identity_is_visible_to_an_independent_reader() { + let original = reused_object(); + let mut doc = Document::open(original.clone()).unwrap(); + doc.set(ObjRef::new(3, 1), Object::Int(77)); + let bytes = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(bytes.starts_with(&original)); + let oracle = hayro_syntax::Pdf::new(Arc::new(bytes)).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(77)); + assert_eq!(catalog.get::(b"Stale"), None); +} + +#[test] +fn generation_repair_diagnostics_are_bounded_without_disabling_recovery() { + let mut objects = vec![(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")]; + objects.extend((3..303).map(|num| (num, 1, "42"))); + let bytes = object_file(&objects); + let bytes = String::from_utf8(bytes.as_ref().clone()).unwrap().replace("00001 n \n", "00000 n \n"); + for materialized in [false, true] { + let bytes = if materialized { bytes.replace("/Size 303", "/Size 302") } else { bytes.clone() }; + let doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert_eq!(doc.repair_log().len(), 257); + assert!(doc.repair_log().last().unwrap().contains("omitted after 256 objects")); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(302, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(302, 2)), Object::Null); + assert_eq!(doc.repair_log().len(), 257); + } +} + +#[test] +fn references_require_the_current_generation_even_with_a_warm_cache() { + let bytes = reused_object(); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + assert_eq!(oracle.xref().get::(ObjectIdentifier::new(3, 0)), None); + assert_eq!(oracle.xref().get::(ObjectIdentifier::new(3, 1)), Some(42)); + + let doc = Document::open(bytes).unwrap(); + assert!(doc.repair_log().is_empty()); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.try_get(3).unwrap(), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 0))), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); +} + +#[test] +fn overlay_generation_changes_do_not_retarget_old_references_or_clones() { + let mut doc = Document::open(reused_object()).unwrap(); + let snapshot = doc.clone(); + let old = ObjRef::new(3, 1); + let new = ObjRef::new(3, 2); + doc.free(old); + assert_eq!(*doc.get(old), Object::Null); + doc.set(new, Object::Int(99)); + assert_eq!(*doc.get(old), Object::Null); + assert_eq!(*doc.get(new), Object::Int(99)); + assert_eq!(*snapshot.get(old), Object::Int(42)); + assert_eq!(*snapshot.get(new), Object::Null); + + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert!(back.repair_log().is_empty()); + assert_eq!(*back.get(old), Object::Null); + assert_eq!(*back.get(new), Object::Int(99)); +} + +#[test] +fn restoring_snapshot_preserves_changes_to_nonzero_generation_objects() { + for generation in [1, 2] { + let mut saved = Document::open(reused_object()).unwrap(); + let mut current = saved.clone(); + let reference = ObjRef::new(3, generation); + current.set(reference, Object::Int(99)); + current.update_dict(current.root().unwrap(), |catalog| catalog.set(b"Current".to_vec(), Object::Ref(reference))).unwrap(); + + saved.overlay_changes_from(¤t).unwrap(); + assert_eq!(*saved.get(reference), Object::Int(99), "generation {generation}"); + let back = Document::open(Arc::new(write_incremental(&saved, &SaveOptions::default()).unwrap())).unwrap(); + assert!(back.repair_log().is_empty()); + assert_eq!(*back.get(reference), Object::Int(99), "generation {generation}"); + let catalog = back.get(back.root().unwrap()); + assert_eq!(*back.resolve(catalog.as_dict().unwrap().get(b"Current").unwrap()), Object::Int(99)); + } +} + +#[test] +fn full_save_does_not_turn_stale_references_into_live_ones() { + let doc = Document::open(reused_object()).unwrap(); + for object_streams in [false, true] { + let saved = write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(); + let saved = Arc::new(saved); + let oracle = hayro_syntax::Pdf::new(saved.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(42)); + assert_eq!(catalog.get::(b"Stale"), None); + let back = Document::open(saved).unwrap(); + assert!(back.repair_log().is_empty()); + let catalog = back.get(back.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + assert_eq!(catalog.get(b"Stale"), Some(&Object::Null)); + assert_eq!(*back.resolve(catalog.get(b"Current").unwrap()), Object::Int(42)); + } +} + +#[test] +fn compressed_objects_only_accept_generation_zero() { + let doc = Document::open(reused_object()).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let current = catalog.as_dict().unwrap().reference(b"Current").unwrap(); + assert_eq!(current.generation, 0); + assert!(matches!(back.xref_entry(current.num), Some(XrefEntry::InStream { .. }))); + assert_eq!(*back.get(current), Object::Int(42)); + assert_eq!(*back.get(ObjRef::new(current.num, 1)), Object::Null); +} + +#[test] +fn indirect_stream_lengths_require_the_referenced_generation() { + for generation in [0, 1] { + // The embedded endstream is data when /Length resolves, but the existing missing- + // length recovery stops there when the reference is stale. + let stream = format!("<< /Length 4 {generation} R >>\nstream\nA\nendstream\nB\nendstream"); + let doc = Document::open(object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 0, &stream), + (4, 1, "13"), + ])) + .unwrap(); + let object = doc.get(ObjRef::new(3, 0)); + let Object::Stream(stream) = &*object else { panic!("expected stream") }; + assert_eq!(&stream.raw[..], if generation == 1 { b"A\nendstream\nB".as_slice() } else { b"A".as_slice() }); + } +} + +/// A newer valid row must not hide damage in a retained older revision. +fn shadowed_generation_disagreement(materialized: bool, shifted: bool) -> Arc> { + let original = contradictory_dictionary(); + let text = std::str::from_utf8(&original).unwrap(); + let previous: usize = text.rsplit("startxref\n").next().unwrap().lines().next().unwrap().parse().unwrap(); + let mut bytes = (*original).clone(); + let offset = bytes.len(); + bytes.extend_from_slice(b"3 1 obj\n<< /Value 77 >>\nendobj\n"); + let xref = bytes.len(); + let size = if materialized { 3 } else { 4 }; + bytes.extend_from_slice( + format!("xref\n3 1\n{offset:010} 00001 n \ntrailer\n<< /Size {size} /Root 1 0 R /Prev {previous} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(), + ); + if shifted { + bytes.splice(..0, b"junk before the header\n".iter().copied()); + } + Arc::new(bytes) +} + +#[test] +fn retained_generation_disagreements_are_reported_in_materialized_mode_too() { + for materialized in [false, true] { + for shifted in [false, true] { + let doc = Document::open(shadowed_generation_disagreement(materialized, shifted)).unwrap(); + assert_eq!(doc.revisions().len(), 2, "the xref chain is retained"); + assert!(!doc.reconstructed()); + assert_eq!(doc.get(ObjRef::new(3, 1)).as_dict().unwrap().int(b"Value"), Some(77)); + assert!( + doc.repair_log().iter().any(|line| line.contains("object 3: xref generation 0 disagrees with object header generation 1")), + "materialized={materialized}, shifted={shifted}: {:?}", + doc.repair_log() + ); + } + } +} + +#[test] +fn repair_aware_edits_keep_both_aliases_and_full_save_normalizes_them() { + let mut doc = Document::open(contradictory_entry(false)).unwrap(); + let snapshot = doc.clone(); + doc.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + for generation in [0, 1] { + assert_eq!(*doc.get(ObjRef::new(3, generation)), Object::Int(77)); + assert_eq!(*snapshot.get(ObjRef::new(3, generation)), Object::Int(42)); + } + let state = doc.read_state(); + assert!(doc.set_existing(ObjRef::new(3, 2), Object::Int(99)).is_err()); + assert!(state.matches(&doc)); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); + let mut restored = snapshot.clone(); + restored.overlay_changes_from(&doc).unwrap(); + for changed in [&doc, &restored] { + assert!(changed.full_save_required()); + for bytes in [ + write_incremental(changed, &SaveOptions::default()).unwrap(), + write_full(changed, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(), + ] { + let bytes = Arc::new(bytes); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(77)); + assert_eq!(catalog.get::(b"Stale"), Some(77), "both source aliases stay live"); + let back = Document::open(bytes).unwrap(); + assert!(back.repair_log().is_empty(), "normalized output has no generation repairs"); + } + } + // Explicit replacement keeps its narrower, caller-supplied generation contract. + doc.set(ObjRef::new(3, 2), Object::Int(99)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Int(99)); +} + +#[test] +fn hybrid_generation_diagnostics_observe_rows_before_the_table_hides_them() { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, generation, body) in [(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "42")] { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{num} {generation} obj\n{body}\nendobj\n").as_bytes()); + } + let hybrid = bytes.len(); + offsets.push(hybrid); + bytes.extend_from_slice(b"4 0 obj\n<< /Type /XRef /Size 5 /W [1 4 2] /Index [3 1] /Length 7 >>\nstream\n"); + bytes.push(1); + bytes.extend_from_slice(&(offsets[2] as u32).to_be_bytes()); + bytes.extend_from_slice(&0u16.to_be_bytes()); + bytes.extend_from_slice(b"\nendstream\nendobj\n"); + let xref = bytes.len(); + bytes.extend_from_slice(b"xref\n0 5\n0000000000 65535 f \n"); + for (i, offset) in offsets.into_iter().enumerate() { + let generation = u16::from(i == 2); + bytes.extend_from_slice(format!("{offset:010} {generation:05} n \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size 5 /Root 1 0 R /XRefStm {hybrid} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(!doc.reconstructed()); + assert!(doc.xref_oddities().iter().any(|(_, num)| *num == 3), "the conflicting hybrid row is set aside"); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null, "the table still determines accepted references"); + assert!(doc.repair_log().iter().any(|line| line.contains("object 3: xref generation 0 disagrees with object header generation 1"))); +} + +#[test] +fn discarded_xref_repair_notes_do_not_survive_object_header_reconstruction() { + let bytes = + String::from_utf8((*contradictory_dictionary()).clone()).unwrap().replace("/Root 1 0 R", "/Root 99 0 R").replace("/Size 4", "/Size 3"); + let doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert!(doc.reconstructed()); + assert!(doc.repair_log().iter().all(|line| !line.contains("xref generation")), "{:?}", doc.repair_log()); +} + +#[test] +fn restoring_equal_values_still_restores_the_accepted_generation_aliases() { + for edited in [false, true] { + for value in [42, 77] { + let mut current = Document::open(contradictory_entry(false)).unwrap(); + if edited { + current.set_existing(ObjRef::new(3, 0), Object::Int(42)).unwrap(); + } + let mut saved = Document::open(object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 0, &value.to_string()), + ])) + .unwrap(); + assert_eq!(*saved.get(ObjRef::new(3, 1)), Object::Null); + saved.overlay_changes_from(¤t).unwrap(); + for generation in [0, 1] { + assert_eq!(*saved.get(ObjRef::new(3, generation)), Object::Int(42), "edited={edited}, value={value}"); + } + assert!(saved.full_save_required()); + } + } +} + +#[test] +fn repaired_page_tree_aliases_keep_one_identity_in_full_saves() { + let source = object_file_with_entry( + &[ + (1, 0, "<< /Type /Catalog /Pages 2 1 R /Stale 2 2 R >>"), + (2, 1, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + (3, 0, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 10 10] >>"), + ], + Some((2, 0, false)), + ); + for edited in [false, true] { + let mut doc = Document::open(source.clone()).unwrap(); + if edited { + doc.update_dict(ObjRef::new(2, 0), |dict| dict.set(b"Edited".to_vec(), Object::Bool(true))).unwrap(); + } + let mut restored = Document::open(source.clone()).unwrap(); + restored.overlay_changes_from(&doc).unwrap(); + for current in [&doc, &restored] { + for object_streams in [false, true] { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + let mut saves = vec![write_full(current, &opts).unwrap()]; + if edited { + saves.push(write_incremental(current, &opts).unwrap()); + } + for bytes in saves { + let back = Document::open(Arc::new(bytes)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + let pages_ref = catalog.reference(b"Pages").unwrap(); + let pages = back.get(pages_ref); + let child_ref = pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap()[0].as_ref().unwrap(); + let child = back.get(child_ref); + assert_eq!(child.as_dict().unwrap().reference(b"Parent"), Some(pages_ref), "edited={edited}, packed={object_streams}"); + assert!(matches!(catalog.get(b"Stale"), Some(Object::Null)), "unrelated generations stay null"); + assert!(back.repair_log().is_empty()); + } + } + } + } +} + +#[test] +fn restoring_explicit_sets_removes_old_generation_aliases() { + for value in [42, 77] { + let mut current = Document::open(contradictory_entry(false)).unwrap(); + current.set(ObjRef::new(3, 0), Object::Int(42)); + for repaired_overlay in [false, true] { + let mut saved = Document::open(contradictory_entry(false)).unwrap(); + if repaired_overlay { + saved.set_existing(ObjRef::new(3, 0), Object::Int(value)).unwrap(); + } + saved.overlay_changes_from(¤t).unwrap(); + assert_eq!(*saved.get(ObjRef::new(3, 0)), Object::Int(42)); + assert_eq!(*saved.get(ObjRef::new(3, 1)), Object::Null, "value={value}, overlay={repaired_overlay}"); + } + } +} + +/// `accepted_generations` lists exactly the generations `get` resolves, for every kind of slot: +/// a clean file slot, a damaged one (row and header disagree), a repair-aware edit, a plain +/// edit, a freed slot, a free row, an object stream member and an unknown number. +#[test] +fn accepted_generations_are_the_generations_get_resolves() { + let agrees = |doc: &Document, num: u32, expected: &[u16]| { + assert_eq!(doc.accepted_generations(num), expected, "object {num}"); + for generation in 0..8 { + let live = !matches!(*doc.get(ObjRef::new(num, generation)), Object::Null); + assert_eq!(live, expected.contains(&generation), "object {num} generation {generation}"); + } + }; + let doc = Document::open(contradictory_entry(false)).unwrap(); + agrees(&doc, 1, &[0]); + agrees(&doc, 3, &[0, 1]); + agrees(&doc, 9, &[]); + let mut repaired = doc.clone(); + repaired.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + agrees(&repaired, 3, &[0, 1]); + let mut narrowed = doc.clone(); + narrowed.set(ObjRef::new(3, 2), Object::Int(99)); + agrees(&narrowed, 3, &[2]); + let mut freed = doc.clone(); + freed.free(ObjRef::new(3, 0)); + agrees(&freed, 3, &[]); + agrees(&Document::open(contradictory_entry(true)).unwrap(), 3, &[]); + let mut doc = + Document::open(object_file(&[(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")])).unwrap(); + let member = doc.add(Object::Dict(Default::default())); + doc.update_dict(ObjRef::new(1, 0), |c| c.set(b"Extra".to_vec(), Object::Ref(member))).unwrap(); + let packed = Document::open(Arc::new(write_full(&doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap(); + assert!(matches!(packed.xref_entry(member.num), Some(XrefEntry::InStream { .. })), "the member is in an object stream"); + agrees(&packed, member.num, &[0]); +} diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..1bd7ee1fd6 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -3,6 +3,7 @@ mod bounded_writer; mod corpus; mod encryption; +mod generations; mod proptests; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/writer.rs b/crates/cos/tests/it/writer.rs index eb03523189..2716b9242e 100644 --- a/crates/cos/tests/it/writer.rs +++ b/crates/cos/tests/it/writer.rs @@ -358,11 +358,32 @@ fn objects_referenced_by_the_encrypt_dictionary_stay_outside_object_streams() { let cf = doc.get(enc_ref).as_dict().unwrap().get(b"CF").cloned().unwrap(); let cf_ref = doc.add(cf); doc.update_dict(enc_ref, |d| d.set(b"CF".to_vec(), Object::Ref(cf_ref))).unwrap(); - let packed = write_full(&doc, &SaveOptions::default()).unwrap(); - let back = Document::open(Arc::new(packed.clone())).expect("reopens"); - assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); - assert_eq!(title(&back), b"Caf\xe9 (draft)"); - assert_eq!(hayro_pages(&packed), 4); + // Visit this alias from Root before the encryption walk reaches /CF's other alias. + doc.update_dict(doc.root().unwrap(), |d| d.set(b"CFProbe".to_vec(), Object::Ref(cf_ref))).unwrap(); + let classic = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let saved = Document::open(Arc::new(classic.clone())).unwrap(); + let cf_num = saved.get(saved.root().unwrap()).as_dict().unwrap().reference(b"CFProbe").unwrap().num; + for damaged in [false, true] { + let mut bytes = classic.clone(); + if damaged { + // Equal-length edits preserve the xref offsets and its generation-zero row. + for (from, to) in [(format!("\n{cf_num} 0 obj"), format!("\n{cf_num} 1 obj")), (format!("/CF {cf_num} 0 R"), format!("/CF {cf_num} 1 R"))] + { + let start = bytes.windows(from.len()).position(|w| w == from.as_bytes()).unwrap(); + bytes[start..start + from.len()].copy_from_slice(to.as_bytes()); + } + } + let doc = Document::open(Arc::new(bytes)).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(packed.clone())).expect("reopens"); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); + assert_eq!(hayro_pages(&packed), 4); + let probe = back.get(back.root().unwrap()).as_dict().unwrap().reference(b"CFProbe").unwrap(); + let encrypt = back.get(back.trailer().reference(b"Encrypt").unwrap()); + assert_eq!(encrypt.as_dict().unwrap().reference(b"CF"), Some(probe)); + assert!(matches!(back.xref_entry(probe.num), Some(pdfcraft_cos::XrefEntry::InFile { .. })), "crypt filters must stay outside object streams"); + } } #[test] diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index 6346e85120..149bd99d97 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -92,6 +92,26 @@ pub fn blank(width: f64, height: f64, pages: usize) -> Result)]) -> Result { + if pages.is_empty() || pages.len() > 10_000 { + return Err(CreateError::Invalid("invalid page count".into())); + } + if pages.iter().any(|(w, h, _, _)| !(w.is_finite() && h.is_finite() && (3.0..=MAX_SIDE).contains(w) && (3.0..=MAX_SIDE).contains(h))) { + return Err(CreateError::Invalid("invalid page size".into())); + } + let mut doc = Document::new_empty(); + let mut tree = Pages::of(&doc)?; + for (w, h, resources, content) in pages { + let content = if content.is_empty() { None } else { Some(content.clone()) }; + tree.add(&mut doc, *w, *h, resources.clone(), content); + } + tree.finish(&mut doc)?; + Ok(doc) +} + // ── images ────────────────────────────────────────────────────────────────────────────────── /// An image ready to embed: its XObject dictionary, encoded data, optional soft mask, the ICC @@ -496,6 +516,129 @@ fn decoded(name: &str, bytes: &[u8], format: image::ImageFormat) -> Result Option { + b.get(at..at + 2).map(|x| u16::from_le_bytes([x[0], x[1]])) +} + +/// How a PCX image holds its pixels: bits per plane and planes. +#[derive(Clone, Copy, PartialEq)] +enum PcxLayout { + /// One byte per pixel per colour: each scanline has a red, a green and a blue row. + Rgb, + /// One byte per pixel, an index into the palette after the image (grey without one). + Indexed, + /// One bit per pixel in each of 1 to 4 planes, an index into the header's palette. + Bits(usize), +} + +impl PcxLayout { + fn of(bits: u8, planes: usize) -> Option { + match (bits, planes) { + (8, 3) => Some(PcxLayout::Rgb), + (8, 1) => Some(PcxLayout::Indexed), + (1, 1..=4) => Some(PcxLayout::Bits(planes)), + _ => None, + } + } +} + +/// The palette of a PCX image, and where its run-length coded scanlines end. An 8-bit image ends +/// with a 0x0C byte and 256 colours; a one-plane bit image is black and white whatever its header +/// holds; other bit images use the 16 colours at byte 16. +fn pcx_palette(bytes: &[u8], layout: PcxLayout) -> (Vec, usize) { + match layout { + PcxLayout::Rgb => (Vec::new(), bytes.len()), + PcxLayout::Indexed => match bytes.len().checked_sub(769).filter(|at| *at >= 128 && bytes.get(*at) == Some(&0x0C)) { + Some(at) => (bytes.get(at + 1..).unwrap_or_default().to_vec(), at), + None => ((0..=255u8).flat_map(|v| [v, v, v]).collect(), bytes.len()), + }, + PcxLayout::Bits(1) => (vec![0, 0, 0, 255, 255, 255], bytes.len()), + PcxLayout::Bits(_) => (bytes.get(16..64).unwrap_or_default().to_vec(), bytes.len()), + } +} + +/// The `rows` scanlines of `line` bytes each, decoded. A run may cross from one plane's row into the +/// next, but not out of its scanline. +fn pcx_scanlines(mut body: &[u8], rows: usize, line: usize) -> Option> { + let mut out = Vec::new(); + out.try_reserve_exact(rows.checked_mul(line)?).ok()?; + for row in 1..=rows { + let end = row * line; + while out.len() < end { + let (&c, rest) = body.split_first()?; + body = rest; + if c < 0xC0 { + out.push(c); + continue; + } + let (&value, rest) = body.split_first()?; + body = rest; + let count = usize::from(c & 0x3F); + if count > end - out.len() { + return None; + } + out.resize(out.len() + count, value); + } + } + Some(out) +} + +/// The RGB samples of the `w` pixels of scanline `row` (planes `stride` bytes apart). +fn pcx_row(out: &mut Vec, row: &[u8], w: usize, stride: usize, layout: PcxLayout, palette: &[u8]) { + let at = |i: usize| row.get(i).copied().unwrap_or(0); + for x in 0..w { + let index = match layout { + PcxLayout::Rgb => { + out.extend([at(x), at(stride + x), at(2 * stride + x)]); + continue; + } + PcxLayout::Indexed => usize::from(at(x)), + PcxLayout::Bits(planes) => (0..planes).fold(0, |index, k| index | (usize::from(at(k * stride + x / 8) >> (7 - x % 8) & 1) << k)), + }; + out.extend((0..3).map(|c| palette.get(index * 3 + c).copied().unwrap_or(0))); + } +} + +/// A PCX image: a 128-byte header (size at 4-11, resolution at 12 and 14, plane count at 65, bytes +/// per plane row at 66), then run-length coded scanlines, each holding one row of every plane. +fn pcx(name: &str, bytes: &[u8]) -> Result { + let bad = |m: &str| CreateError::Image(name.into(), m.into()); + let header = bytes.get(..128).ok_or_else(|| bad("a damaged PCX header"))?; + let word = |at: usize| le16(header, at).unwrap_or(0); + let byte = |at: usize| header.get(at).copied().unwrap_or(0); + let (bits, planes, stride) = (byte(3), usize::from(byte(65)), usize::from(word(66))); + let (Some(w), Some(h)) = (word(8).checked_sub(word(4)), word(10).checked_sub(word(6))) else { + return Err(bad("a damaged PCX header")); + }; + let (w, h) = (usize::from(w) + 1, usize::from(h) + 1); + if planes == 0 || stride == 0 { + return Err(bad("a damaged PCX header")); + } + let layout = PcxLayout::of(bits, planes).ok_or_else(|| bad("this PCX plane layout is not supported"))?; + if stride < (w * usize::from(bits)).div_ceil(8) { + return Err(bad("a damaged PCX header")); + } + let line = planes * stride; + if line.saturating_mul(h) > MAX_DECODED_BYTES || w.saturating_mul(h).saturating_mul(3) > MAX_DECODED_BYTES { + return Err(bad("unusual PCX image size")); + } + let (palette, end) = pcx_palette(bytes, layout); + let body = bytes.get(128..end).unwrap_or_default(); + let scanlines = pcx_scanlines(body, h, line).ok_or_else(|| bad("a damaged PCX body"))?; + let mut data = Vec::new(); + data.try_reserve_exact(w * h * 3).map_err(|_| bad("not enough memory for this image"))?; + for row in scanlines.chunks_exact(line) { + pcx_row(&mut data, row, w, stride, layout, &palette); + } + let dpi = (f64::from(word(12)), f64::from(word(14))); + let dpi = if dpi.0 > 0.0 && dpi.1 > 0.0 { dpi } else { (72.0, 72.0) }; + let mut d = Dict::new(); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let px = (u32::try_from(w).unwrap_or(0), u32::try_from(h).unwrap_or(0)); + Ok(Embedded { dict: d, data, filtered: false, smask: None, icc: None, px, dpi }) +} + /// Largest GIF logical screen, in pixels (4096×4096). const MAX_GIF_PIXELS: u64 = 1 << 24; @@ -603,7 +746,13 @@ pub enum SourceKind { } /// File extensions Create converts to PDF (images and plain text). -pub const CONVERTIBLE: [&str; 12] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "txt", "text"]; +pub const CONVERTIBLE: [&str; 13] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx", "txt", "text"]; + +/// A PCX file: its manufacturer byte, a version (1 was never used), run-length coding and 1 or 8 +/// bits per plane. `pcx` says when the rest of the header isn't one it can read. +fn is_pcx(bytes: &[u8]) -> bool { + bytes.first() == Some(&0x0A) && matches!(bytes.get(1), Some(0 | 2..=5)) && bytes.get(2) == Some(&1) && matches!(bytes.get(3), Some(1 | 8)) +} /// A classic TIFF or a BigTIFF (version 43), in either byte order. fn is_tiff(bytes: &[u8]) -> bool { @@ -620,6 +769,7 @@ fn is_image(bytes: &[u8]) -> bool { || bytes.starts_with(b"\x89PNG\r\n\x1a\n") || is_tiff(bytes) || is_gif(bytes) + || is_pcx(bytes) // JPEG 2000: a JP2 file or a raw codestream. || bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) @@ -655,6 +805,8 @@ fn embed(name: &str, bytes: &[u8], wanted: usize) -> Result, Creat Ok(vec![decoded(name, &pdfcraft_xfa::image::bmp_with_pixel_offset(bytes), image::ImageFormat::Bmp)?]) } else if is_gif(bytes) { Ok(vec![decoded(name, bytes, image::ImageFormat::Gif)?]) + } else if is_pcx(bytes) { + Ok(vec![pcx(name, bytes)?]) } else if bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) { Ok(vec![jpx(name, bytes)?]) } else { diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 9e620c19c3..bea96c887d 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -485,6 +485,82 @@ fn bigtiff_images_become_pages() { assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 1.0], "4×2 px at 144 dpi"); } +/// A PCX header for a `w` by `h` image at 72 dpi. +fn pcx_header(version: u8, bits: u8, planes: u8, stride: u16, w: u16, h: u16) -> Vec { + let mut pcx = vec![0u8; 128]; + pcx[..4].copy_from_slice(&[0x0A, version, 1, bits]); + pcx[8..10].copy_from_slice(&(w - 1).to_le_bytes()); + pcx[10..12].copy_from_slice(&(h - 1).to_le_bytes()); + pcx[12..14].copy_from_slice(&72u16.to_le_bytes()); + pcx[14..16].copy_from_slice(&72u16.to_le_bytes()); + pcx[65] = planes; + pcx[66..68].copy_from_slice(&stride.to_le_bytes()); + pcx +} + +/// A 24-bit PCX: each scanline holds a red, a green and a blue row. +#[test] +fn pcx_images_become_pages() { + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[10, 11, 20, 21, 30, 31]); + pcx.extend_from_slice(&[40, 41, 50, 51, 60, 61]); + let embedded = super::pcx("a.pcx", &pcx).unwrap(); + assert_eq!(embedded.data, [10, 20, 30, 11, 21, 31, 40, 50, 60, 41, 51, 61]); + let doc = reopen(&from_images(&[("a.pcx".into(), pcx)]).unwrap()); + assert_eq!(pages(&doc).len(), 1); + assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 2.0]); + assert_eq!(image_of(&doc, 0).name(b"ColorSpace"), Some(&b"DeviceRGB"[..])); +} + +#[test] +fn a_pcx_run_may_cross_planes_but_not_scanlines() { + // 0xC4 repeats 7 four times: the red row and half of the green one + let mut pcx = pcx_header(5, 8, 3, 2, 2, 1); + pcx.extend_from_slice(&[0xC4, 7, 8, 9]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [7, 7, 8, 7, 7, 9]); + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[0xC8, 7, 1, 2, 3, 4]); + assert!(super::pcx("a.pcx", &pcx).is_err(), "eight bytes in a six-byte scanline"); +} + +#[test] +fn an_8_bit_pcx_uses_its_palette_and_is_grey_without_one() { + let mut pcx = pcx_header(5, 8, 1, 2, 2, 1); + pcx.extend_from_slice(&[1, 2]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 1, 1, 2, 2, 2], "no palette: grey"); + pcx.push(0x0C); + let mut palette = vec![0u8; 768]; + palette[3..9].copy_from_slice(&[9, 8, 7, 6, 5, 4]); + pcx.extend_from_slice(&palette); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [9, 8, 7, 6, 5, 4]); +} + +#[test] +fn bit_plane_pcx_images_read_their_palette() { + // version 2, one plane: black and white + let mut pcx = pcx_header(2, 1, 1, 2, 2, 1); + pcx.extend_from_slice(&[0b1000_0000, 0]); + assert!(from_images(&[("a.pcx".into(), pcx.clone())]).is_ok(), "an old version is still a PCX"); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [255, 255, 255, 0, 0, 0]); + // two planes: colour 3 is at byte 16 + 9 + let mut pcx = pcx_header(5, 1, 2, 1, 1, 1); + pcx[25..28].copy_from_slice(&[1, 2, 3]); + pcx.extend_from_slice(&[0x80, 0x80]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 2, 3]); +} + +#[test] +fn damaged_pcx_headers_and_bodies_are_refused() { + let no_planes = pcx_header(5, 8, 0, 2, 2, 1); + let no_stride = pcx_header(5, 8, 3, 0, 2, 1); + let short_stride = pcx_header(5, 8, 3, 1, 2, 1); + let short_body = pcx_header(5, 8, 3, 2, 2, 1); + for bytes in [no_planes, no_stride, short_stride, short_body] { + assert!(super::pcx("a.pcx", &bytes).is_err()); + } + assert!(super::pcx("a.pcx", &[0x0A, 5, 1, 8]).is_err(), "no header"); +} + /// A synthetic ICC profile: a 128-byte header (size, device class, colour space, `acsp`) and /// some payload. Only the header is read. fn icc_profile(space: &[u8; 4]) -> Vec { diff --git a/crates/crypt/src/lib.rs b/crates/crypt/src/lib.rs index ff92d9af66..e697bc8553 100644 --- a/crates/crypt/src/lib.rs +++ b/crates/crypt/src/lib.rs @@ -82,17 +82,24 @@ pub enum Auth { User, } -/// Permission bits from `/P` (§7.6.4.2, Table 22). Owners have every permission. +/// Permission flags from `/P`, read the way the handler's revision reads them (ISO 32000-1:2008, +/// Table 22). Owners have every permission. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Permissions { pub bits: i32, pub owner: bool, + /// The `/R` value of the handler that read `bits`. + pub revision: i64, } impl Permissions { fn bit(&self, n: u32) -> bool { self.owner || (self.bits >> (n - 1)) & 1 == 1 } + /// Revision 3 and later read the full flag list of Table 22; revision 2 has its own. + fn is_revision_3_or_later(&self) -> bool { + self.revision >= 3 + } pub fn print(&self) -> bool { self.bit(3) } @@ -107,17 +114,22 @@ impl Permissions { self.bit(6) } pub fn fill_forms(&self) -> bool { - self.bit(9) || self.bit(6) + if self.is_revision_3_or_later() { self.bit(9) || self.bit(6) } else { self.bit(6) } } + /// Revision 2 covers accessibility with the copy flag (bit 5); revision 3 and later with bit 10. pub fn extract_for_accessibility(&self) -> bool { - self.bit(10) + if self.is_revision_3_or_later() { self.bit(10) } else { self.bit(5) } } - /// Insert, rotate or delete pages; create bookmarks and thumbnails. + /// Insert, rotate or delete pages; create bookmarks and thumbnails. Revision 3 and later grant + /// it by bit 11 alone. Table 22 defines bit 11 for those revisions only, so at revision 2 the + /// modify flag (bit 4) covers it. pub fn assemble(&self) -> bool { - self.bit(11) || self.bit(4) + if self.is_revision_3_or_later() { self.bit(11) } else { self.bit(4) } } + /// Revision 3 and later need bit 3 as well as bit 12. Table 22 defines bit 12 for those + /// revisions only: at revision 2 the print flag (bit 3) is all there is. pub fn print_high_quality(&self) -> bool { - self.bit(12) + if self.is_revision_3_or_later() { self.print() && self.bit(12) } else { self.print() } } } @@ -250,7 +262,7 @@ impl SecurityHandler { /// edited `/P` grants nothing extra. Without a readable `/Perms`, `/P` applies as before. pub fn permissions(&self) -> Permissions { let bits = self.perms_p().map_or(self.dict.p, |p| self.dict.p & p); - Permissions { bits, owner: self.auth == Auth::Owner } + Permissions { bits, owner: self.auth == Auth::Owner, revision: self.dict.r } } /// `/P` as `/Perms` holds it: AES-256 ECB of one block (= CBC with a zero IV), marked `adb`. diff --git a/crates/crypt/src/tests.rs b/crates/crypt/src/tests.rs index 6064176d78..09a88ed129 100644 --- a/crates/crypt/src/tests.rs +++ b/crates/crypt/src/tests.rs @@ -240,11 +240,11 @@ fn unsupported_handlers_are_reported() { #[test] fn permission_bits_follow_table_22() { - let p = Permissions { bits: -3904, owner: false }; // 0xFFFFF0C0: only reserved bits set + let p = Permissions { bits: -3904, owner: false, revision: 3 }; // 0xFFFFF0C0: only reserved bits set assert!(!p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.assemble()); - let p = Permissions { bits: -1, owner: false }; + let p = Permissions { bits: -1, owner: false, revision: 3 }; assert!(p.print() && p.modify() && p.copy() && p.annotate() && p.assemble() && p.fill_forms()); - let p = Permissions { bits: 0, owner: true }; + let p = Permissions { bits: 0, owner: true, revision: 3 }; assert!(p.modify(), "owners ignore /P"); } diff --git a/crates/crypt/tests/it/main.rs b/crates/crypt/tests/it/main.rs index 8b39bf1956..e7742bbd75 100644 --- a/crates/crypt/tests/it/main.rs +++ b/crates/crypt/tests/it/main.rs @@ -1,4 +1,5 @@ //! The crate's integration tests, linked into one test binary. +mod permissions_revision; mod proptests; mod qpdf_literals; diff --git a/crates/crypt/tests/it/permissions_revision.rs b/crates/crypt/tests/it/permissions_revision.rs new file mode 100644 index 0000000000..bad709cd51 --- /dev/null +++ b/crates/crypt/tests/it/permissions_revision.rs @@ -0,0 +1,126 @@ +//! The permission accessors of `pdfcraft_crypt::Permissions`, checked against the flag table of +//! ISO 32000-1:2008 (Table 22). Expected values come from that table only. +//! +//! Bit n has the mask `1 << (n - 1)`. Readers use bits 3 to 6 and 9 to 12 and ignore the rest. + +use pdfcraft_crypt::{Algorithm, Auth, NewEncryption, Permissions, SecurityHandler, create}; + +const ID: &[u8] = b"0123456789abcdef"; +const T: bool = true; +const F: bool = false; + +/// The eight accessors in this order: print, high-quality print, modify, copy, accessibility, +/// annotate, fill forms, assemble. +fn cells(p: &Permissions) -> [bool; 8] { + [p.print(), p.print_high_quality(), p.modify(), p.copy(), p.extract_for_accessibility(), p.annotate(), p.fill_forms(), p.assemble()] +} + +/// Creates a file with `flags` at revision `revision`, opens it with the user password, and +/// returns the user's permissions. +fn user_view(alg: Algorithm, revision: i64, flags: i32) -> Permissions { + let made = create( + &NewEncryption { algorithm: alg, user_password: "user", owner_password: "owner", permissions: flags, encrypt_metadata: true, seed: [42; 32] }, + ID, + ) + .expect("the handler is created"); + assert_eq!(made.dict().r, revision, "the algorithm writes revision {revision}"); + let opened = SecurityHandler::open(made.dict().clone(), ID, Some("user")).expect("the user password opens the file"); + assert_eq!(opened.auth(), Auth::User, "the user password gives user access"); + opened.permissions() +} + +/// Revision 3 and later: each row is a flag word and the expected eight cells. +const REVISION_3_ROWS: &[(&str, i32, [bool; 8])] = &[ + ("every meaningful flag set, ignored flags clear", -196, [T, T, T, T, T, T, T, T]), + ("every flag set, ignored flags set too", -1, [T, T, T, T, T, T, T, T]), + ("reserved flags only", -4096, [F, F, F, F, F, F, F, F]), + ("ignored flags set, meaningful flags clear", -3901, [F, F, F, F, F, F, F, F]), + ("modify flag clear", -9, [T, T, F, T, T, T, T, T]), + ("copy flag clear", -17, [T, T, T, F, T, T, T, T]), + ("annotate flag clear", -33, [T, T, T, T, T, F, T, T]), + ("annotate and fill-existing flags clear", -289, [T, T, T, T, T, F, F, T]), + ("annotate flag set, fill-existing flag clear", -257, [T, T, T, T, T, T, T, T]), + ("accessibility flag clear", -513, [T, T, T, T, F, T, T, T]), + ("high-quality flag clear, print flag set", -2049, [T, F, T, T, T, T, T, T]), + ("bits 3 to 12 set, reserved bits clear", 4092, [T, T, T, T, T, T, T, T]), + ("positive flag word, bit 32 clear", 2_147_483_647, [T, T, T, T, T, T, T, T]), + ("revision 2 example read at revision 3", -44, [T, T, F, T, T, F, T, T]), +]; + +#[test] +fn revision_three_rows_hold_through_a_created_and_reopened_file() { + for (name, bits, want) in REVISION_3_ROWS { + assert_eq!(cells(&user_view(Algorithm::Rc4_128, 3, *bits)), *want, "{name}: flag word {bits:#010x}"); + } +} + +#[test] +fn owner_access_gives_every_flag() { + let made = create( + &NewEncryption { + algorithm: Algorithm::Rc4_128, + user_password: "user", + owner_password: "owner", + permissions: 0, + encrypt_metadata: true, + seed: [42; 32], + }, + ID, + ) + .expect("the handler is created"); + assert_eq!(made.auth(), Auth::Owner, "the owner password gives owner access"); + assert_eq!(cells(&made.permissions()), [T; 8]); +} + +#[test] +fn revision_two_print_modify_copy_and_annotate_follow_the_table() { + for (bits, want) in [(-4, [T, T, T, T]), (-44, [T, F, T, F]), (-17, [T, T, F, T])] { + let p = user_view(Algorithm::Rc4_40, 2, bits); + assert_eq!([p.print(), p.modify(), p.copy(), p.annotate()], want, "revision 2, flag word {bits}"); + } +} + +#[test] +fn revision_two_with_every_flag_set_grants_accessibility_and_form_filling() { + let p = user_view(Algorithm::Rc4_40, 2, -4); + assert!(p.extract_for_accessibility(), "bit 5 is set, and at revision 2 it covers accessibility"); + assert!(p.fill_forms(), "bit 6 is set"); +} + +#[test] +fn revision_three_assembly_needs_its_own_flag() { + let p = user_view(Algorithm::Rc4_128, 3, -1025); + assert!(p.modify(), "bit 4 is set"); + assert!(!p.assemble(), "assembly needs bit 11; bit 4 alone does not grant it"); +} + +#[test] +fn revision_three_high_quality_print_needs_the_print_flag() { + let p = user_view(Algorithm::Rc4_128, 3, -5); + assert!(!p.print(), "bit 3 is clear"); + assert!(!p.print_high_quality(), "high-quality print needs bit 3 and bit 12"); +} + +#[test] +fn revision_two_accessibility_follows_the_copy_flag() { + let p = user_view(Algorithm::Rc4_40, 2, -17); + assert!(!p.copy(), "bit 5 is clear"); + assert!(!p.extract_for_accessibility(), "at revision 2 accessibility is bit 5"); +} + +#[test] +fn revision_two_form_filling_ignores_bit_nine() { + let p = user_view(Algorithm::Rc4_40, 2, -44); + assert!(!p.annotate(), "bit 6 is clear"); + assert!(!p.fill_forms(), "at revision 2 form filling is bit 6 alone"); +} + +#[test] +fn revision_two_ignores_the_flags_only_later_revisions_define() { + // Bit 4 is clear and bit 11 set: bit 11 means nothing at revision 2, so assembly stays refused. + let p = user_view(Algorithm::Rc4_40, 2, -44); + assert!(!p.assemble(), "at revision 2 assembly is bit 4 alone"); + // Bit 3 is set and bit 12 clear: printing at revision 2 has one quality. + let p = user_view(Algorithm::Rc4_40, 2, -2049); + assert!(p.print_high_quality(), "at revision 2 high-quality print is bit 3 alone"); +} diff --git a/crates/edit/src/added.rs b/crates/edit/src/added.rs index a603a37060..0fc2aa7359 100644 --- a/crates/edit/src/added.rs +++ b/crates/edit/src/added.rs @@ -31,7 +31,7 @@ use layout::Piece; pub(crate) use layout::WRAP_CALLS; pub use layout::{PASTE_STEP, first_undrawable, lines, paste_place, paste_rect, text_rect, text_width}; pub use store::{add_content, delete_content, isolate_content, list_added, scan_markers, update_content}; -pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, MarkerScan}; +pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan}; #[derive(Clone, Debug, PartialEq)] pub enum Content { diff --git a/crates/edit/src/added/draw.rs b/crates/edit/src/added/draw.rs index 9404f939b1..f3f9b55859 100644 --- a/crates/edit/src/added/draw.rs +++ b/crates/edit/src/added/draw.rs @@ -1,7 +1,7 @@ use super::layout::{ Piece, arabic_error, hebrew_error, is_arabic, is_hebrew, is_thai, latin_width, norm, shaped_layout, text_rect, thai_error, wrapped, }; -use super::types::{AddedText, Align, Family}; +use super::types::{AddedText, Align, Family, Ink}; use super::{ Content, Dict, Document, EditError, GlyphError, GlyphOutline, HashMap, HashSet, ObjRef, Object, PdfString, ShapedCluster, ShapedScript, Stream, align_code, literal, matrix_number, n, shaped_glyph, win_ansi, @@ -105,8 +105,7 @@ pub(super) fn draw_laid_shaped( fonts.set(name.clone().into_bytes(), Object::Ref(font)); names.push(name); } - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT {} {} {} rg\n", n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT {}\n", t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -293,8 +292,7 @@ pub(super) fn draw(doc: &mut Document, c: &Content, view: [f64; 6], taken: &Dict } res.set(b"Font".to_vec(), Object::Dict(fonts)); let r = text_rect(t); - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT /{name} {} Tf {} {} {} rg\n", n(t.size), n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT /{name} {} Tf {}\n", n(t.size), t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -355,6 +353,12 @@ pub(super) fn params(c: &Content) -> Dict { d.set(b"Font".to_vec(), Object::name(t.family.base_font(t.bold, t.italic))); d.set(b"Size".to_vec(), Object::Real(t.size)); d.set(b"Color".to_vec(), arr(&t.color)); + // The ink beside its RGB look, which a reader that knows no ink keeps showing. + match t.painted_ink() { + Some(Ink::Gray(g)) => d.set(b"Gray".to_vec(), Object::Real(g)), + Some(Ink::Cmyk(c)) => d.set(b"CMYK".to_vec(), arr(&c)), + None => {} + } d.set(b"Align".to_vec(), Object::Int(align_code(t.align))); // Each paragraph's own alignment, or null where it follows /Align. let aligns = crate::paragraph_align::tidy(t.aligns.clone(), t.align); diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index 7dae55596f..3555910f04 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -1,8 +1,33 @@ use super::draw::{draw, params}; use super::layout::{first_undrawable, is_thai, nums, shaped_arabic, thai_error}; -use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, MarkerScan, TAG}; +use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan, TAG}; use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Object, Stream, align_from, check, contents, page_list, place_tagged}; +/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range or not +/// made only of numbers (a `/Gray` name, a `/CMYK` entry that isn't a number), have none: the +/// text paints in its RGB `/Color`. +fn ink(doc: &Document, d: &Dict) -> Option { + let ink = if let Some(g) = d.get(b"Gray") { + Ink::Gray(read_value(doc, g, Object::as_f64)?) + } else { + // Four entries or none: a longer array is never copied or read through. + read_value(doc, d.get(b"CMYK")?, |cmyk| { + let [c, m, y, k] = cmyk.as_array()?.as_slice() else { return None }; + Some(Ink::Cmyk([c.as_f64()?, m.as_f64()?, y.as_f64()?, k.as_f64()?])) + })? + }; + Some(ink).filter(|i| i.valid()) +} + +/// `f` of `o`, a reference followed. A direct value is read where it is, never copied, so a +/// large one a file puts there costs nothing to look at. +fn read_value(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> T) -> T { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + direct => f(direct), + } +} + fn parse(doc: &Document, d: &Dict) -> Option { let r = nums(doc, d.get(b"Rect")); let rect: [f64; 4] = r.try_into().ok()?; @@ -21,7 +46,7 @@ fn parse(doc: &Document, d: &Dict) -> Option { .map(|a| a.iter().take(crate::paragraph_align::count(&text)).map(|v| v.as_int().map(|c| align_from(Some(c)))).collect()) }) .unwrap_or_default(); - Some(Content::Text(AddedText { + let mut t = AddedText { rect, text, family, @@ -29,11 +54,19 @@ fn parse(doc: &Document, d: &Dict) -> Option { italic, size: d.get(b"Size").and_then(Object::as_f64).unwrap_or(12.0), color: if c.len() == 3 { [c[0], c[1], c[2]] } else { [0.0; 3] }, + ink: ink(doc, d), align, aligns: crate::paragraph_align::tidy(aligns, align), // Items written before character spacing have none. char_spacing: d.get(b"CharSpacing").and_then(Object::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0), - })) + }; + // An ink whose look isn't the stored colour (a file edited by hand) is none. + t.drop_stale_ink(); + // One that stands is the ink as painted, finer digits dropped, and its look. + if let Some(i) = t.ink { + t.set_ink(Some(i)); + } + Some(Content::Text(t)) } b"Image" => { let crop = nums(doc, d.get(b"Crop")); @@ -138,6 +171,9 @@ fn validate(c: &Content) -> Result<(), EditError> { "the character spacing must be between -{CHAR_SPACING_LIMIT} and {CHAR_SPACING_LIMIT} points" ))); } + if t.ink.is_some_and(|i| !i.valid()) { + return Err(EditError::Invalid("a gray level and each CMYK value must be from 0 to 1".into())); + } if (r[2] - r[0]).abs() < 1.0 { return Err(EditError::Invalid("the text box is too narrow".into())); } diff --git a/crates/edit/src/added/types.rs b/crates/edit/src/added/types.rs index 0caaaa863b..24101b2eea 100644 --- a/crates/edit/src/added/types.rs +++ b/crates/edit/src/added/types.rs @@ -1,5 +1,5 @@ use super::layout::norm; -use super::{Content, MarkKind, ObjRef, Std14}; +use super::{Content, MarkKind, ObjRef, Std14, n}; pub(super) const TAG: &str = "Added"; @@ -90,6 +90,60 @@ pub enum Align { Justify, } +/// A device colour added text is painted in instead of RGB, for print files: a text's black in +/// CMYK stays 100 K rather than a DeviceRGB black the printer separates into rich black. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Ink { + /// A DeviceGray level, from 0 (black) to 1 (white): painted with `g`. + Gray(f64), + /// DeviceCMYK `[c, m, y, k]`, each from 0 to 1 (`[0, 0, 0, 1]` is 100 K): painted with `k`. + Cmyk([f64; 4]), +} + +impl Ink { + /// Whether every component is a number from 0 to 1. + pub fn valid(self) -> bool { + self.components().iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) + } + + /// The ink as its operator paints it: each component to the thousandth that `g` and `k` are + /// written with (as RGB's `rg`), so the item, what it reads back as and the page agree. + pub fn painted(self) -> Ink { + let round = |v: f64| (v * 1000.0).round() / 1000.0; + match self { + Ink::Gray(g) => Ink::Gray(round(g)), + Ink::Cmyk(c) => Ink::Cmyk(c.map(round)), + } + } + + fn components(&self) -> &[f64] { + match self { + Ink::Gray(g) => std::slice::from_ref(g), + Ink::Cmyk(c) => c, + } + } + + /// How the ink looks on screen: gray is the same level in each of red, green and blue + /// (ISO 32000-2 §10.4.2.2), CMYK takes each colour plus black from 1 (§10.4.2.5). + pub fn rgb(self) -> [f64; 3] { + match self { + Ink::Gray(g) => [g.clamp(0.0, 1.0); 3], + Ink::Cmyk([c, m, y, k]) => [c, m, y].map(|v| 1.0 - (v + k).clamp(0.0, 1.0)), + } + } + + /// The operator that makes this ink the fill colour, without a line end. + pub(super) fn fill_op(self) -> String { + match self { + Ink::Gray(g) => format!("{} g", n(g.clamp(0.0, 1.0))), + Ink::Cmyk(c) => { + let [c, m, y, k] = c.map(|v| v.clamp(0.0, 1.0)); + format!("{} {} {} {} k", n(c), n(m), n(y), n(k)) + } + } + } +} + #[derive(Clone, Debug, PartialEq)] pub struct AddedText { /// The text box in display space; its width wraps the text and its top is the first line's @@ -100,7 +154,14 @@ pub struct AddedText { pub bold: bool, pub italic: bool, pub size: f64, + /// The RGB colour: what the text is painted in, unless it has an [`AddedText::ink`] this is + /// the look of. pub color: [f64; 3], + /// A gray or CMYK ink the text is painted in, with `color` its look ([`Ink::rgb`]). Set it + /// with [`AddedText::set_ink`] and the colour with [`AddedText::set_color`], which drops an + /// ink the colour no longer shows; one left behind by setting `color` directly paints nothing + /// ([`AddedText::painted_ink`]) until [`AddedText::drop_stale_ink`] forgets it. + pub ink: Option, pub align: Align, /// Extra space after every character, in points: the PDF's character spacing (`Tc`). pub char_spacing: f64, @@ -114,6 +175,44 @@ impl AddedText { pub fn paragraph_align(&self, i: usize) -> Align { crate::paragraph_align::align_of(&self.aligns, i, self.align) } + + /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. An + /// ink in range is kept as painted ([`Ink::painted`]); one out of range stays as given, for + /// saving to refuse. + pub fn set_ink(&mut self, ink: Option) { + let ink = ink.map(|i| if i.valid() { i.painted() } else { i }); + if let Some(i) = ink { + self.color = i.rgb(); + } + self.ink = ink; + } + + /// Paint the text in RGB `rgb`. Its ink stays only when `rgb` is the ink's own look; once + /// gone it stays gone, so picking that look again later paints in RGB. + pub fn set_color(&mut self, rgb: [f64; 3]) { + self.color = rgb; + self.drop_stale_ink(); + } + + /// Forget an ink `color` no longer shows, so two styles that paint the same compare equal. + pub fn drop_stale_ink(&mut self) { + self.ink = self.painted_ink(); + } + + /// The ink the text is painted in, or `None` for RGB: the one it has while `color` is still + /// its look. + pub fn painted_ink(&self) -> Option { + // Within rounding: a look read back from a file, or from a colour picker, is the same. + self.ink.filter(|i| i.rgb().iter().zip(self.color).all(|(a, b)| (a - b).abs() < 1e-6)) + } + + /// The operator that sets the text's fill colour, without a line end. + pub(super) fn fill_op(&self) -> String { + match self.painted_ink() { + Some(ink) => ink.fill_op(), + None => crate::marks::rgb(self.color), + } + } } impl Default for AddedText { @@ -126,6 +225,7 @@ impl Default for AddedText { italic: false, size: 12.0, color: [0.0; 3], + ink: None, align: Align::Left, aligns: Vec::new(), char_spacing: 0.0, diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index f4680bcc5c..72656fa1bb 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -1691,6 +1691,12 @@ pub(crate) fn selectable_images( .collect()) } +/// Whether an XObject drawn at or after operator `from` of the page's joined operators shows +/// inside `rect` (user space). +pub(crate) fn painted_over(doc: &Document, page: usize, rect: [f64; 4], from: usize) -> Result { + Ok(page_images(doc, page)?.iter().any(|i| i.op >= from && i.visible_rect.and_then(|v| intersection(v, rect)).is_some())) +} + fn transform_image( doc: &mut Document, page: &pdfcraft_model::Page, diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index 44a6730acc..0cf1886afd 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -61,8 +61,8 @@ pub use text::{ }; pub mod added; pub use added::{ - Added, AddedImage, AddedText, Align, Content, Family, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, list_added, - scan_markers, update_content, + Added, AddedImage, AddedText, Align, Content, Family, Ink, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, + list_added, scan_markers, update_content, }; mod apply; @@ -77,8 +77,8 @@ use marks::{matrix_number, n}; pub(crate) use place::undrawable; #[cfg(test)] use place::{ISOLATION_MAX_DEPTH, OLD_WRAP_CLOSE, WRAP_CLOSE, WRAP_OPEN, is_wrap_close, page_fonts, stream_bytes, stream_tag, tagged, wrapper_role}; -use place::{check, check_on_top, contents, page_list, place_tagged}; -pub use place::{stamp, undrawable_message}; +use place::{Reach, check, check_on_top, contents, draw_on_top, lift_on_top, page_list, place_tagged, revealed_under_paint}; +pub use place::{add_type3_fonts, stamp, stamp_with_fonts, undrawable_message}; #[cfg(test)] mod marked_isolation_tests; diff --git a/crates/edit/src/place.rs b/crates/edit/src/place.rs index b535d60563..6460aab066 100644 --- a/crates/edit/src/place.rs +++ b/crates/edit/src/place.rs @@ -371,6 +371,76 @@ pub(crate) fn place_tagged(doc: &mut Document, page: &pdfcraft_model::Page, tag: Ok(mark_ref) } +/// Draw `ops`, written for the space whose CTM is `ctm`, on top of everything on `page`, tagged +/// `Edited`. A rewritten invisible paragraph goes here when [`revealed_under_paint`]: an OCR +/// layer written before the scan image it lies under would keep the revealed text hidden (#3973). +pub(crate) fn draw_on_top(doc: &mut Document, page: &pdfcraft_model::Page, ctm: [f64; 6], ops: &[pdfcraft_content::Op]) -> Result<(), EditError> { + if ops.is_empty() { + return Ok(()); + } + use pdfcraft_content::Op; + let mut all = vec![Op::new("q", vec![]), Op::new("cm", ctm.iter().map(|v| pdfcraft_content::num(*v)).collect())]; + all.extend_from_slice(ops); + all.push(Op::new("Q", vec![])); + place_tagged(doc, page, "Edited", pdfcraft_content::serialize_ops(&all), false).map(drop) +} + +/// Where a rewritten paragraph can be drawn, in user space ([`revealed_under_paint`]). +pub(crate) struct Reach { + /// The paragraph's old box, where its cover goes. + pub rect: [f64; 4], + /// The move asked for (`BlockStyle::offset`). + pub offset: [f64; 2], + /// The widest line the rewrite may draw, how many lines it has, and how far apart they are. + pub width: f64, + pub rows: usize, + pub pitch: f64, + pub upright: bool, +} + +impl Reach { + /// The old box and the moved one, grown to `width` either way (alignment may grow a line left + /// or right), down to `rows` lines and up one `pitch` above its top (a larger size on the first line, however many lines the old box had); turned text grows by the lines' diagonal every way. + /// Unbounded when a number is not finite. + pub(crate) fn bounds(&self) -> [f64; 4] { + const UNBOUNDED: [f64; 4] = [f64::NEG_INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::INFINITY]; + let ([dx, dy], r) = (self.offset, self.rect); + if !r.iter().chain(&self.offset).chain([&self.width, &self.pitch]).all(|v| v.is_finite()) { + return UNBOUNDED; + } + let moved = [r[0] + dx, r[1] + dy, r[2] + dx, r[3] + dy]; + let rows = f64::from(u32::try_from(self.rows).unwrap_or(u32::MAX)); + let (width, height) = (self.width.abs(), self.pitch.abs() * rows); + let grown = if self.upright { + [moved[0].min(moved[2] - width), moved[1].min(moved[3] - height), moved[2].max(moved[0] + width), moved[3] + self.pitch.abs()] + } else { + let d = width.hypot(height); + [moved[0] - d, moved[1] - d, moved[2] + d, moved[3] + d] + }; + let b = [r[0].min(grown[0]), r[1].min(grown[1]), r[2].max(grown[2]), r[3].max(grown[3])]; + if b.iter().all(|v| v.is_finite()) { b } else { UNBOUNDED } + } +} + +/// Whether a rewritten paragraph that was `invisible` goes on top of page `page`: something after +/// operator `at` of the page's joined `ops` paints over `reach` (user space, [`Reach::bounds`]), as the scan image +/// does over an OCR layer written before it (#3973). XObjects count where they are drawn; an inline +/// image or a shading after it counts anywhere, and so does any XObject when the page's images +/// cannot be listed. +pub(crate) fn revealed_under_paint(doc: &Document, page: usize, invisible: bool, reach: [f64; 4], ops: &[pdfcraft_content::Op], at: usize) -> bool { + let later = ops.get(at..).unwrap_or_default(); + if !invisible || later.iter().any(|op| op.is("BI") || op.is("sh")) { + return invisible; + } + later.iter().any(|op| op.is("Do")) && crate::images::painted_over(doc, page, reach, at).unwrap_or(true) +} + +/// The operators to draw on top of the page ([`draw_on_top`]): all of `paragraph` when `over`, +/// none otherwise. The page was checked to take them ([`check_on_top`]) before the edit began. +pub(crate) fn lift_on_top(over: bool, paragraph: &mut Vec) -> Vec { + if over { std::mem::take(paragraph) } else { Vec::new() } +} + fn too_deep_on_top() -> EditError { EditError::Invalid("the page's marked content is nested too deeply to add anything on top of it".into()) } @@ -408,14 +478,79 @@ pub(crate) fn place_inside_wrapper(doc: &mut Document, page: &pdfcraft_model::Pa /// it away); the original content is wrapped in q/Q first. Used for applied redaction boxes. /// Returns the new content stream. pub fn stamp(doc: &mut Document, page: usize, tag: &str, content: Vec) -> Result { + stamp_with_fonts(doc, page, tag, &mut pdfcraft_fonts::TextPlan::default(), |_| content) +} + +/// Draw the content `make` builds on top of a page, with standard Helvetica as `/PCHelv` and +/// the plan's Type3 fonts added to the page's resources first (so the content can reference +/// them by the names the plan carries). The original content is wrapped in q/Q first; the +/// stream is tagged `tag` (not a mark kind, so Remove never takes it away). This is how the +/// OCR text layers get their Unicode faces (see [`add_type3_fonts`]). Returns the new content +/// stream. +pub fn stamp_with_fonts( + doc: &mut Document, + page: usize, + tag: &str, + plan: &mut pdfcraft_fonts::TextPlan, + make: impl FnOnce(&pdfcraft_fonts::TextPlan) -> Vec, +) -> Result { let all = page_list(doc); check(&[page], all.len())?; check_on_top(doc, &all[page])?; + add_type3_fonts(doc, page, plan)?; + let content = make(plan); + let all = page_list(doc); add_resources(doc, &all[page], None, Some(&content))?; let p = page_list(doc).swap_remove(page); place_tagged(doc, &p, tag, content, false) } +/// Add the plan's Type3 fonts to a page's (0-based) resources, building each font once and +/// sharing the object with every page that uses it. Fills in each planned font's resource name +/// (`PCOcr`, `PCOcr1`, … — a name free on the page being registered: a font built for an +/// earlier page keeps its name only where that name is free or already that font) and object; +/// the content generators read the names from the plan, so this must run before they are +/// called for each page. +pub fn add_type3_fonts(doc: &mut Document, page: usize, plan: &mut pdfcraft_fonts::TextPlan) -> Result<(), EditError> { + if plan.fonts.is_empty() { + return Ok(()); + } + let all = page_list(doc); + check(&[page], all.len())?; + let p = &all[page]; + let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut fonts = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); + for i in 0..plan.fonts.len() { + let object = match plan.fonts[i].object { + Some(r) => { + // Built for an earlier page under a name that was free THERE. On this page the + // name may already be another font (the page's own, or an earlier OCR layer's): + // then this page gets a free name of its own, and the plan carries it for the + // content written next. + let taken = fonts.get(plan.fonts[i].name.as_bytes()).is_some_and(|o| o.as_ref() != Some(r)); + if taken { + plan.fonts[i].name = super::text::unique_font_name(&fonts, "PCOcr"); + } + r + } + None => { + let glyphs = plan.fonts[i].glyphs.iter().map(|(ch, g)| (*ch, super::text::glyph_path(g), g.width)).collect::>(); + let dict = super::text::type3_font_dict(doc, &glyphs)?; + let r = doc.add(Object::Dict(dict)); + plan.fonts[i].name = super::text::unique_font_name(&fonts, "PCOcr"); + plan.fonts[i].object = Some(r); + r + } + }; + fonts.set(plan.fonts[i].name.as_bytes().to_vec(), Object::Ref(object)); + } + res.set(b"Font".to_vec(), Object::Dict(fonts)); + // The page gets its own (possibly copied) resources; shared dictionaries stay untouched. + let obj = p.obj; + doc.update_dict(obj, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + Ok(()) +} + pub(crate) fn begin(kind: MarkKind, subtype: &str, matrix: [f64; 6]) -> String { format!( "q\n/Artifact <> BDC\n{} {} {} {} {} {} cm\n", diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index e78da842b8..49ab8abbc7 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1048,6 +1048,164 @@ fn added_text_and_images_are_page_content_that_stays_editable() { assert!(marks_present(&doc).is_empty()); } +/// #6440: added text in a print file can be gray or CMYK, painted with `g` / `k`, so its black +/// stays 100 K. The ink survives save and reopen; another colour drops it for good. +#[test] +fn gray_and_cmyk_added_text_writes_g_and_k_until_it_is_recoloured() { + use crate::added::Ink; + let item_stream = |doc: &Document, i: usize| { + let obj = list_added(doc)[i].obj; + let Object::Stream(s) = &*doc.get(obj) else { panic!("an item is a stream") }; + String::from_utf8_lossy(&s.decoded().unwrap()).into_owned() + }; + let mut doc = fixture(); + let base = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + let mut black = base.clone(); + black.set_ink(Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(black.color, [0.0; 3], "100 K looks black on screen"); + let mut gray = AddedText { rect: [72.0, 400.0, 300.0, 500.0], ..base.clone() }; + gray.set_ink(Some(Ink::Gray(0.25))); + add_content(&mut doc, 0, &Content::Text(black)).unwrap(); + add_content(&mut doc, 0, &Content::Text(gray)).unwrap(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 200.0, 300.0, 300.0], ..base.clone() })).unwrap(); + + let doc = reopen(&doc); + let (k, g, rgb) = (item_stream(&doc, 0), item_stream(&doc, 1), item_stream(&doc, 2)); + assert!(k.contains("/PCFHelvetica 14 Tf 0 0 0 1 k\n") && !k.contains("rg"), "{k}"); + assert!(g.contains("/PCFHelvetica 14 Tf 0.25 g\n") && !g.contains("rg"), "{g}"); + assert!(rgb.contains("/PCFHelvetica 14 Tf 0 0 0 rg\n"), "RGB text paints as before: {rgb}"); + let inks: Vec> = list_added(&doc).iter().map(|a| if let Content::Text(t) = &a.content { t.painted_ink() } else { None }).collect(); + assert_eq!(inks, [Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), Some(Ink::Gray(0.25)), None]); + + // Another colour drops the ink, and picking the ink's look again later doesn't bring it back; + // picking that look while the ink is there keeps it. + let Content::Text(cmyk) = list_added(&doc)[0].content.clone() else { panic!() }; + let mut same = cmyk.clone(); + same.set_color([0.0; 3]); + assert_eq!(same, cmyk, "black on 100 K stays 100 K"); + let mut recoloured = cmyk.clone(); + recoloured.set_color([1.0, 0.0, 0.0]); + assert_eq!(recoloured.ink, None); + recoloured.set_color([0.0; 3]); + assert_eq!((recoloured.ink, recoloured.painted_ink()), (None, None), "the ink stays gone"); + + // Set directly, a colour paints in RGB too, and the item reads back without the ink: what a + // draft with its stale ink dropped compares equal to. + let mut doc = doc; + let red = AddedText { color: [1.0, 0.0, 0.0], ..cmyk }; + update_content(&mut doc, 0, 0, &Content::Text(red.clone())).unwrap(); + let stream = item_stream(&doc, 0); + assert!(stream.contains("1 0 0 rg") && !stream.contains(" k\n"), "{stream}"); + let mut settled = red; + settled.drop_stale_ink(); + for doc in [&doc, &reopen(&doc)] { + let Content::Text(back) = list_added(doc)[0].content.clone() else { panic!() }; + // The stored box is the text's own height. + assert_eq!(back, AddedText { rect: back.rect, ..settled.clone() }); + } + + // CMYK looks as ISO 32000-2 §10.4.2.5 maps it to RGB; an ink out of range is refused. + assert_eq!(Ink::Cmyk([0.5, 0.0, 1.0, 0.25]).rgb(), [0.25, 0.75, 0.0]); + let mut wrong = base; + wrong.ink = Some(Ink::Gray(1.5)); + assert!(add_content(&mut doc, 0, &Content::Text(wrong)).is_err()); +} + +/// #6440: a file edited by hand paints its added text in RGB unless the stored gray or CMYK is +/// in range, the right length and still the look of `/Color`; nothing in it panics. +#[test] +fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { + use crate::added::Ink; + let mut doc = fixture(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); + let obj = list_added(&doc)[0].obj; + let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); + // A `/CMYK` far longer than four numbers, as a hostile file might store it: an ink first. + let long = real(&[0.0, 0.0, 0.0, 1.0].repeat(50_000)); + let long_ref = Object::Ref(doc.add(long.clone())); + let ink_ref = Object::Ref(doc.add(real(&[0.0, 0.0, 0.0, 1.0]))); + let read_text = |entries: &[(&[u8], Object)]| { + let mut edited = doc.clone(); + let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; + let mut params = s.dict.get(b"PCAdded").and_then(|p| p.as_dict().cloned()).unwrap(); + for (key, value) in entries { + params.set(key.to_vec(), value.clone()); + } + s.dict.set(b"PCAdded".to_vec(), Object::Dict(params)); + edited.set(obj, Object::Stream(s)); + let Content::Text(t) = list_added(&edited)[0].content.clone() else { panic!() }; + t + }; + let read = |entries: &[(&[u8], Object)]| read_text(entries).ink; + // As PDFCraft writes it: the ink stands. + assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", ink_ref)]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), "by reference"); + // Finer than it paints, it reads back as painted (`0.5 g`), its look with it. + let fine = read_text(&[(b"Color", real(&[0.50049; 3])), (b"Gray", Object::Real(0.50049))]); + assert_eq!((fine.ink, fine.color), (Some(Ink::Gray(0.5)), [0.5; 3])); + for (what, entries) in [ + ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), + ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), + ("five CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0, 0.0]))]), + ("a long CMYK", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long.clone())]), + ("a long CMYK by reference", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long_ref.clone())]), + ("gray a long array", vec![(b"Color", real(&[0.0; 3])), (b"Gray", long.clone())]), + ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), + ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), + ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), + ( + "a CMYK entry not a number", + vec![ + (b"Color", real(&[0.0; 3])), + (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(1.0)])), + ], + ), + ( + "gray not a number beside a CMYK", + vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black")), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))], + ), + ("gray NaN", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::Real(f64::NAN))]), + ("CMYK infinite", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, f64::INFINITY]))]), + ] { + assert_eq!(read(&entries), None, "{what}"); + } +} + +/// #6440: an ink finer than the thousandth `g` and `k` are written to is kept as painted, so the +/// item, its stored `/CMYK` and what it reads back as are what the page shows. +#[test] +fn an_ink_is_kept_as_it_is_painted() { + use crate::added::Ink; + let mut doc = fixture(); + let mut t = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + t.set_ink(Some(Ink::Cmyk([0.12345, 0.00049, 0.0, 0.99951]))); + let painted = Ink::Cmyk([0.123, 0.0, 0.0, 1.0]); + assert_eq!((t.ink, t.color), (Some(painted), painted.rgb())); + add_content(&mut doc, 0, &Content::Text(t)).unwrap(); + let doc = reopen(&doc); + let item = &list_added(&doc)[0]; + let Object::Stream(s) = &*doc.get(item.obj) else { panic!("an item is a stream") }; + let stream = String::from_utf8_lossy(&s.decoded().unwrap()).into_owned(); + assert!( + stream.contains( + "/PCFHelvetica 14 Tf 0.123 0 0 1 k +" + ), + "{stream}" + ); + let stored = s + .dict + .get(b"PCAdded") + .and_then(|p| p.as_dict()) + .and_then(|p| p.get(b"CMYK")) + .and_then(Object::as_array) + .map(|a| a.iter().filter_map(Object::as_f64).collect::>()); + assert_eq!(stored, Some(vec![0.123, 0.0, 0.0, 1.0])); + let Content::Text(back) = &item.content else { panic!() }; + assert_eq!(back.painted_ink(), Some(painted)); +} + /// #4464: a page left with one added item draws that item and nothing else. #[test] fn an_isolated_item_is_all_that_is_left_on_its_page() { @@ -3514,7 +3672,10 @@ fn page_content_bytes(doc: &Document, page: usize) -> Vec { } } -/// A page drawing image XObject /Im0 at 100,100 size 200 × 100 (pixels 4 × 2). +/// A page drawing image XObject /Im0 at 100,100 size 200 × 100 (pixels 4 × 2), on a real optional +/// content layer: the catalog's `/OCProperties` declares object 7 as the only OCG and the page +/// resources' `/Properties` maps `/L0` to it, so `/OC /L0 BDC` marks a valid optional-content +/// member rather than a dangling name. fn image_page() -> Document { image_page_with("q 200 0 0 100 100 100 cm /Im0 Do Q BT /F1 12 Tf 72 700 Td (Caption) Tj ET") } @@ -3534,12 +3695,13 @@ fn remove_crop(doc: &mut Document, page: usize, index: usize) -> Result<(), Edit fn image_page_with(content: &str) -> Document { let objs: Vec = vec![ - "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Catalog /Pages 2 0 R /OCProperties << /OCGs [7 0 R] /D << /Order [7 0 R] >> >> >>".into(), "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), - "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /XObject << /Im0 6 0 R >> /ExtGState << /GS0 << /Type /ExtGState /ca 0.5 /CA 0.5 >> /GSalpha << /Type /ExtGState /ca 0.6 >> /GSblend << /Type /ExtGState /BM /Multiply >> /GS_translucent << /Type /ExtGState /ca 0.5 >> >> >> >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /XObject << /Im0 6 0 R >> /ExtGState << /GS0 << /Type /ExtGState /ca 0.5 /CA 0.5 >> /GSalpha << /Type /ExtGState /ca 0.6 >> /GSblend << /Type /ExtGState /BM /Multiply >> /GS_translucent << /Type /ExtGState /ca 0.5 >> >> /Properties << /L0 7 0 R >> >> >>".into(), format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into(), "<< /Type /XObject /Subtype /Image /Width 4 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 8 >>\nstream\n\u{0}\u{0}\u{0}\u{0}\u{0}\u{0}\u{0}\u{0}\nendstream".into(), + "<< /Type /OCG /Name (Layer 0) >>".into(), ]; let mut out = b"%PDF-1.7\n".to_vec(); let mut offs = Vec::new(); @@ -3753,6 +3915,30 @@ fn repeated_group_moves_of_a_cropped_image_change_one_clip_transform() { assert_eq!(caption, ["Caption"]); } +#[test] +fn moving_an_image_inside_marked_content_leaves_every_sequence_intact() { + // `translate_images` changes the image's own `cm`; its `Do` stays where it was, so a moved + // image stays in the sequence that tags it (an image on a layer sits in `/OC … BDC`). + for content in [ + "/P <> BDC q 100 0 0 100 50 50 cm /Im0 Do Q EMC /P <> BDC BT /F1 12 Tf 72 700 Td (Text) Tj ET EMC", + "/OC /L0 BDC q 100 0 0 100 50 50 cm /Im0 Do Q EMC /P <> BDC BT /F1 12 Tf 72 700 Td (Text) Tj ET EMC", + ] { + let mut doc = image_page_with(content); + let before = images::page_images(&doc, 0).unwrap()[0].rect; + images::translate_images(&mut doc, 0, &[0], [10.0, 0.0]).unwrap(); + let after = images::page_images(&doc, 0).unwrap()[0].rect; + assert!(close(after, [before[0] + 10.0, before[1], before[2] + 10.0, before[3]]), "{content}: {after:?}"); + let ops = pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops; + let count = |name: &str| ops.iter().filter(|op| op.is(name)).count(); + assert_eq!((count("BDC"), count("EMC")), (2, 2), "{content}: both sequences remain"); + let first = ops.iter().position(|op| op.is("BDC")).unwrap(); + let end = ops.iter().position(|op| op.is("EMC")).unwrap(); + let image = ops.iter().position(|op| op.is("Do")).unwrap(); + assert!(first < image && image < end, "{content}: the image is still inside its own sequence"); + assert!(!ops[end..].iter().any(|op| op.is("Do")), "{content}: the other sequence gained no image"); + } +} + #[test] fn page_images_move_turn_replace_and_delete() { let mut doc = image_page(); @@ -6213,6 +6399,126 @@ fn group_selection_refuses_incomplete_content_and_resource_budget_overruns() { assert!(editable_objects(&doc, 0).is_err(), "an undecodable stream is refused"); } +/// Invisible text moves on top only when something painted after it covers it: an XObject drawn +/// over its box (an image, or a form holding one), or any inline image or shading. An image +/// clipped away entirely, or drawn elsewhere, does not move it. +#[test] +fn revealed_text_moves_on_top_only_when_painted_over() { + let on_top_of = |doc: Document| { + let block = text::text_blocks(&doc, 0).unwrap().remove(0); + let ops = pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops; + let at = ops.iter().position(|o| o.is("BT")).unwrap(); + revealed_under_paint(&doc, 0, block.invisible, block.rect, &ops, at) + }; + let on_top = |content: &str| on_top_of(image_page_with(content)); + let scan = "q 200 0 0 100 100 100 cm /Im0 Do Q"; + assert!(on_top(&format!("BT /F1 12 Tf 3 Tr 110 150 Td (Under) Tj ET {scan}")), "the scan covers it"); + assert!(!on_top(&format!("BT /F1 12 Tf 3 Tr 400 600 Td (Away) Tj ET {scan}")), "an image elsewhere"); + assert!(!on_top(&format!("{scan} BT /F1 12 Tf 3 Tr 110 150 Td (Over) Tj ET")), "the scan is drawn first"); + assert!(!on_top(&format!("BT /F1 12 Tf 0 Tr 110 150 Td (Shown) Tj ET {scan}")), "visible text stays where it is"); + assert!(!on_top("BT /F1 12 Tf 3 Tr 110 150 Td (Path) Tj ET 0 0 600 800 re f"), "a filled path does not move it"); + assert!(on_top("BT /F1 12 Tf 3 Tr 400 600 Td (Shade) Tj ET /Sh sh"), "a shading after it"); + assert!(on_top("BT /F1 12 Tf 3 Tr 400 600 Td (Inline) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 ID x EI"), "an inline image after it"); + let under = "BT /F1 12 Tf 3 Tr 110 150 Td (Under) Tj ET"; + let elsewhere = "q 50 0 0 50 400 600 cm /Im0 Do Q"; + assert!(!on_top(&format!("{under} q 0 0 10 10 re W n {scan} Q")), "the scan is clipped away entirely"); + assert!(on_top(&format!("{under} {elsewhere} {scan}")), "two images, the second over it"); + assert!(on_top(&format!("{under} {scan} {elsewhere}")), "two images, the first over it"); + assert!(!on_top(&format!("{under} {elsewhere} {elsewhere}")), "two images, neither over it"); + let in_form = |content: &str| { + let page = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /XObject << /Fm0 7 0 R >> >> >>"; + let image = stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", "A"); + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources << /XObject << /Im0 6 0 R >> >>", "/Im0 Do"); + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + page.into(), + stream("", content), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into(), + image, + form, + ]; + on_top_of(build(&objs.iter().map(String::as_str).collect::>())) + }; + assert!(in_form(&format!("{under} q 200 0 0 100 100 100 cm /Fm0 Do Q")), "a form holding the scan, over it"); + assert!(!in_form(&format!("{under} q 50 0 0 50 400 600 cm /Fm0 Do Q")), "a form holding an image elsewhere"); +} + +/// Revealed OCR text is checked where the rewrite lands, not only where it was: an image drawn +/// after it over just the moved or grown text still lifts the text on top, on both rewrite paths. +#[test] +fn an_ocr_rewrite_goes_on_top_of_paint_over_where_it_lands() { + let scan = "q 200 0 0 100 100 100 cm /Im0 Do Q"; + let moved = text::BlockStyle { offset: Some([-290.0, -450.0]), ..Default::default() }; + let sized = text::BlockStyle { size: Some(24.0), ..Default::default() }; + let spaced = text::BlockStyle { line_spacing: Some(3.0), ..Default::default() }; + let range = text::RangeStyle { + range: 0..2, + family: None, + resource: None, + size: Some(120.0), + color: None, + underline: None, + char_spacing: None, + scale: None, + }; + let enlarged = text::BlockStyle { ranges: vec![range], ..Default::default() }; + let above = "q 200 0 0 60 100 200 cm /Im0 Do Q"; + let cases = [ + ("BT /F1 12 Tf 3 Tr 400 600 Td (Away) Tj ET", "Moved", &moved), + ("BT /F1 12 Tf 3 Tr 400 600 Td (Away) Tj ET", "Moved\nTwice", &moved), + ("BT /F1 12 Tf 3 Tr 40 150 Td (Hi) Tj ET", "Hello there, this line now runs well past the scan's left edge", &text::BlockStyle::default()), + ("BT /F1 12 Tf 3 Tr 40 150 Td (Hi) Tj ET", "Hello there,\nthis line now runs well past the scan's left edge", &text::BlockStyle::default()), + ("BT /F1 12 Tf 3 Tr 110 230 Td (Above) Tj ET", "One\nTwo\nThree\nFour\nFive\nSix", &text::BlockStyle::default()), + ]; + let low = "q 200 0 0 30 100 300 cm /Im0 Do Q"; + let cases = cases.into_iter().map(|(c, e, s)| (c, e, s, scan)).chain([ + ("BT /F1 12 Tf 3 Tr 110 500 Td (Above) Tj ET", "One\nTwo\nThree\nFour\nFive\nSix", &sized, low), + ("BT /F1 12 Tf 3 Tr 110 500 Td (Above) Tj ET", "One\nTwo\nThree\nFour\nFive\nSix", &spaced, low), + ("BT /F1 12 Tf 3 Tr 110 150 Td (Hi there) Tj ET", "Hi there", &enlarged, above), + ]); + for (content, edit, style, scan) in cases { + let mut doc = image_page_with(&format!("{content} {scan}")); + let before = text::text_blocks(&doc, 0).unwrap().remove(0); + let ops = pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops; + let at = ops.iter().position(|o| o.is("BT")).unwrap(); + assert!(!revealed_under_paint(&doc, 0, true, before.rect, &ops, at), "{edit:?}: the old box alone is clear of the scan"); + text::rewrite_block(&mut doc, 0, 0, Some(edit), style).unwrap(); + let ops = pdfcraft_content::parse(&page_content_bytes(&doc, 0)).ops; + let scan_at = ops.iter().position(|o| o.is("Do")).unwrap(); + let shows = ["Tj", "TJ", "'", "\""]; + assert!(ops.iter().skip(scan_at).any(|o| shows.iter().any(|s| o.is(s))), "{edit:?}: the rewrite is drawn over the scan"); + } +} + +/// [`crate::Reach`] grows the old box to the move, the width either way and the lines below. +#[test] +fn a_rewrites_reach_covers_its_move_width_and_lines() { + let reach = |offset, width, rows, upright| crate::Reach { rect: [10.0, 90.0, 30.0, 100.0], offset, width, rows, pitch: 10.0, upright }.bounds(); + assert_eq!(reach([0.0, 0.0], 20.0, 1, true), [10.0, 90.0, 30.0, 110.0]); + assert_eq!(reach([5.0, -50.0], 20.0, 1, true), [10.0, 40.0, 35.0, 100.0]); + assert_eq!(reach([0.0, 0.0], 50.0, 3, true), [-20.0, 70.0, 60.0, 110.0]); + assert_eq!(reach([0.0, 0.0], 30.0, 4, false), [-40.0, 40.0, 80.0, 150.0]); + assert_eq!(reach([f64::NAN, 0.0], 20.0, 1, true)[0], f64::NEG_INFINITY); + let larger = crate::Reach { rect: [10.0, 90.0, 30.0, 100.0], offset: [0.0, 0.0], width: 20.0, rows: 1, pitch: 24.0, upright: true }; + assert_eq!(larger.bounds(), [10.0, 76.0, 30.0, 124.0], "a larger size grows the line up as well as down"); +} + +/// Revealed OCR text may have to go on top of the page; a page whose marked content nests too deep +/// for that is refused before either rewrite path changes anything. +#[test] +fn an_ocr_rewrite_refused_on_top_leaves_the_document_as_it_was() { + let content = format!("{} BT /F1 20 Tf 3 Tr 42 44 Td (John) Tj ET /Im Do {}", "/X BMC ".repeat(10_001), "EMC ".repeat(10_001)); + let mut doc = text_page(&content); + assert!(text::text_blocks(&doc, 0).unwrap()[0].invisible); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + for edit in ["Jane", "Jane\nDoe"] { + let error = text::rewrite_block(&mut doc, 0, 0, Some(edit), &text::BlockStyle::default()).unwrap_err(); + assert!(error.to_string().contains("nested too deeply"), "{edit:?}: {error}"); + assert!(write_full(&doc, &SaveOptions::default()).unwrap() == before, "{edit:?}: a refused rewrite changes nothing"); + } +} + #[test] fn group_move_budget_refusals_keep_the_original_page_selectable() { let text = text_page(&format!("{} BT /F1 12 Tf 40 400 Td (Target) Tj ET", "n ".repeat(99_995))); @@ -10109,6 +10415,86 @@ fn styled_existing_justification_is_refused_instead_of_silently_left_aligning() assert_eq!(page_content_bytes(&reopen(&doc), 0), before); } +/// The OCR stamp path registers the plan's Type3 faces (a free `PCOcr…` name per stamp, so a +/// page that is OCR'd twice keeps both layers' fonts), and the stamped codes extract back +/// through the font's ToUnicode CMap. +#[test] +fn stamp_with_fonts_registers_the_planned_type3_faces() { + if without_craft_fonts("stamp_with_fonts_registers_the_planned_type3_faces") { + return; + } + let mut doc = text_page("BT /F2 12 Tf 72 700 Td (ab) Tj ET"); + let mut plan = pdfcraft_fonts::TextPlan::for_words(["日本"]); + assert_eq!(plan.missing(), 0); + stamp_with_fonts(&mut doc, 0, "OCR", &mut plan, |p| { + let codes: Vec = p.encoding(0).iter().map(|&(_, c)| c).collect(); + let mut c = format!("BT 3 Tr /PCHelv 1 Tf\n/{} 1 Tf 100 700 Td ", p.fonts[0].name).into_bytes(); + c.extend_from_slice(&pdfcraft_fonts::literal(&codes)); + c.extend_from_slice(b" Tj ET"); + c + }) + .unwrap(); + assert!(plan.fonts[0].name.starts_with("PCOcr"), "{:?}", plan.fonts[0].name); + // The registered font carries a ToUnicode CMap; the engine's own reader decodes the run. + let reopened = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let lines = text::text_lines(&reopened, 0).unwrap(); + assert!(lines.iter().any(|l| l.text.contains("日本")), "{lines:?}"); + // A second stamp on the same page takes the next free name; the first layer's font stands. + let mut second = pdfcraft_fonts::TextPlan::for_words(["語"]); + stamp_with_fonts(&mut doc, 0, "OCR", &mut second, |p| format!("BT 3 Tr /{} 1 Tf 100 600 Td (\u{1}) Tj ET", p.fonts[0].name).into_bytes()) + .unwrap(); + assert_eq!(second.fonts[0].name, format!("{}1", plan.fonts[0].name)); +} + +/// A Type3 face built for page 1 is reused on page 2, but page 2 already has a font under the +/// name page 1 gave it: page 2's own font keeps its name, and the shared face is registered on +/// page 2 under a free one that the plan carries for page 2's content. (The plan is written out +/// by hand, so this runs without the craft-fonts build input.) +#[test] +fn a_reused_type3_face_never_replaces_a_later_page_s_font_of_the_same_name() { + let mut doc = build(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 5 0 R >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 6 0 R /Resources << /Font << /PCOcr 7 0 R >> >> >>", + &stream("", ""), + &stream("", "BT /PCOcr 12 Tf 72 700 Td (own) Tj ET"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + ]); + let glyph = pdfcraft_fonts::GlyphOutline { contours: vec![vec![[0.1, 0.0], [0.9, 0.0], [0.5, 0.8]]], width: 1.0, bbox: [0.1, 0.0, 0.9, 0.8] }; + let mut plan = pdfcraft_fonts::TextPlan::default(); + plan.fonts.push(pdfcraft_fonts::PlannedType3 { family: "Test", glyphs: vec![('日', glyph)], name: String::new(), object: None }); + let draw = |p: &pdfcraft_fonts::TextPlan| format!("BT 3 Tr /{} 1 Tf 100 600 Td (\u{1}) Tj ET", p.fonts[0].name).into_bytes(); + stamp_with_fonts(&mut doc, 0, "OCR", &mut plan, draw).unwrap(); + assert_eq!(plan.fonts[0].name, "PCOcr", "free on page 1"); + let face = plan.fonts[0].object.expect("built once"); + stamp_with_fonts(&mut doc, 1, "OCR", &mut plan, draw).unwrap(); + assert_ne!(plan.fonts[0].name, "PCOcr", "page 2 already has a PCOcr"); + let page2 = &pdfcraft_model::pages(&doc)[1]; + let res = doc.resolve(page2.dict.get(b"Resources").unwrap()); + let fonts = doc.resolve(res.as_dict().unwrap().get(b"Font").unwrap()); + let fonts = fonts.as_dict().unwrap(); + assert_eq!(fonts.get(b"PCOcr").and_then(|o| o.as_ref()), Some(pdfcraft_cos::ObjRef::new(7, 0)), "page 2's own font keeps its name"); + assert_eq!(fonts.get(plan.fonts[0].name.as_bytes()).and_then(|o| o.as_ref()), Some(face), "the shared face, under a free name"); + assert_eq!(plan.fonts[0].object, Some(face), "still built once"); +} + +#[test] +fn planned_type3_charproc_has_valid_metrics_and_encloses_its_ink() { + let glyph = pdfcraft_fonts::GlyphOutline { + contours: vec![vec![[-0.1001, -0.2001], [0.9001, 1.1001]]], + width: 0.8, + bbox: [-0.1001, -0.2001, 0.9001, 1.1001], + }; + let parsed = pdfcraft_content::parse(&text::glyph_path(&glyph)); + assert_eq!(parsed.skipped, 0); + let metrics = &parsed.ops[0]; + assert!(metrics.is("d1")); + assert_eq!(metrics.operands.len(), 6); + assert_eq!((metrics.num(0), metrics.num(1)), (Some(800.0), Some(0.0))); + assert_eq!((metrics.num(2), metrics.num(3), metrics.num(4), metrics.num(5)), (Some(-101.0), Some(-201.0), Some(901.0), Some(1101.0))); +} + #[test] fn image_arrange_refuses_crossed_scope_boundaries_without_changes() { for content in ["/P <> BDC q 10 0 0 10 0 0 cm /Im0 Do EMC Q /Im0 Do", "q /P <> BDC /Im0 Do Q EMC /Im0 Do"] { diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 0fb00324b6..3d1626a576 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -52,6 +52,7 @@ use fallback::{ }; #[cfg(test)] use fallback::{add_type3_font, is_type3_fallback}; +pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; use interp::{Origin, Paint, union}; use lines::page_dict; pub(crate) use lines::{FormBudget, content_streams, content_streams_within, dict_in_place, form_call, resource_dict, rewritten_contents, splice}; diff --git a/crates/edit/src/text/blocks.rs b/crates/edit/src/text/blocks.rs index 9ddf4db4b1..48fe956f4f 100644 --- a/crates/edit/src/text/blocks.rs +++ b/crates/edit/src/text/blocks.rs @@ -31,7 +31,8 @@ pub struct TextBlock { /// Byte ranges in `editor_text` and their source line indexes. pub spans: Vec<(std::ops::Range, usize)>, /// Every line is invisible text (rendering mode 3): an OCR layer over a scanned image, whose - /// pixels show the words. A rewrite draws the new text visibly (see [`BlockStyle::cover`]). + /// pixels show the words. A rewrite draws the new text visibly, on top of the page when + /// the scan is drawn after it (see [`BlockStyle::cover`]). pub invisible: bool, } diff --git a/crates/edit/src/text/fallback.rs b/crates/edit/src/text/fallback.rs index 78f9014a20..469d1739d4 100644 --- a/crates/edit/src/text/fallback.rs +++ b/crates/edit/src/text/fallback.rs @@ -493,6 +493,92 @@ pub(super) fn add_type3_font(doc: &mut Document, fonts_res: &mut Dict, face: &Cr name } +/// A glyph's CharProc body at 1000 units per em, with a six-operand `d1` and its actual ink box. +pub(crate) fn glyph_path(glyph: &pdfcraft_fonts::GlyphOutline) -> Vec { + let scale = 1000.0; + let b = if glyph.contours.is_empty() { [0.0; 4] } else { glyph.bbox }; + let mut out = format!( + "{} 0 {} {} {} {} d1\n", + pdf_num(glyph.width * scale), + pdf_num((b[0] * scale).floor()), + pdf_num((b[1] * scale).floor()), + pdf_num((b[2] * scale).ceil()), + pdf_num((b[3] * scale).ceil()) + ) + .into_bytes(); + for contour in &glyph.contours { + let Some(first) = contour.first() else { continue }; + out.extend_from_slice(format!("{} {} m\n", pdf_num(first[0] * scale), pdf_num(first[1] * scale)).as_bytes()); + for p in contour.iter().skip(1) { + out.extend_from_slice(format!("{} {} l\n", pdf_num(p[0] * scale), pdf_num(p[1] * scale)).as_bytes()); + } + out.extend_from_slice(b"h\n"); + } + out.extend_from_slice(b"f\n"); + out +} + +/// A resource name free in `fonts_res`: `base`, then `base1`, `base2`, … (a page that was +/// OCR'd twice keeps both layers' fonts distinct). +pub(crate) fn unique_font_name(fonts_res: &Dict, base: &str) -> String { + let mut name = String::from(base); + let mut suffix = 0usize; + while fonts_res.contains(name.as_bytes()) { + suffix = suffix.saturating_add(1); + name = format!("{base}{suffix}"); + } + name +} + +/// The Type3 font dictionary for `(character, CharProc body, advance in em)` glyphs, codes from +/// 1: one CharProc per code, `/Differences` glyph names, real `/Widths`, and a `/ToUnicode` CMap +/// so the text extracts back (this is what makes the output Unicode-correct, not just visible). +pub(crate) fn type3_font_dict(doc: &mut Document, glyphs: &[(char, Vec, f64)]) -> Result { + let mut charprocs = Dict::new(); + let mut differences = vec![Object::Int(1)]; + let mut widths = Vec::with_capacity(glyphs.len()); + let mut cmap = String::from( + "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapType 2 def\n1 begincodespacerange\n<01> \nendcodespacerange\n", + ); + cmap.push_str(&format!("{} beginbfchar\n", glyphs.len())); + let mut codes = Vec::with_capacity(glyphs.len()); + for (i, (ch, path, width)) in glyphs.iter().enumerate() { + let code = u8::try_from(i + 1).map_err(|_| EditError::Invalid("too many unique characters for one font".into()))?; + let glyph_name = format!("g{code:02X}"); + let mut pd = Dict::new(); + pd.set(b"Length".to_vec(), path.len() as i64); + let proc_ref = doc.add(Object::Stream(Stream::from_raw(pd, path.clone()))); + charprocs.set(glyph_name.as_bytes().to_vec(), Object::Ref(proc_ref)); + differences.push(Object::name(&glyph_name)); + widths.push(Object::Real((width * 1000.0).round())); + cmap.push_str(&format!("<{code:02X}> <{}>\n", unicode_hex(*ch))); + codes.push((*ch, code, *width)); + } + cmap.push_str("endbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"); + let mut cmap_dict = Dict::new(); + cmap_dict.set(b"Length".to_vec(), cmap.len() as i64); + let cmap_ref = doc.add(Object::Stream(Stream::from_raw(cmap_dict, cmap.into_bytes()))); + let mut encoding = Dict::new(); + encoding.set(b"Type".to_vec(), Object::name("Encoding")); + encoding.set(b"Differences".to_vec(), Object::Array(differences)); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type3")); + font.set(b"Name".to_vec(), Object::name("PCFallback")); + font.set(b"FontBBox".to_vec(), Object::Array(vec![Object::Int(0), Object::Int(-300), Object::Int(1000), Object::Int(1000)])); + font.set( + b"FontMatrix".to_vec(), + Object::Array(vec![Object::Real(0.001), Object::Int(0), Object::Int(0), Object::Real(0.001), Object::Int(0), Object::Int(0)]), + ); + font.set(b"FirstChar".to_vec(), Object::Int(1)); + font.set(b"LastChar".to_vec(), Object::Int(codes.len() as i64)); + font.set(b"Widths".to_vec(), Object::Array(widths)); + font.set(b"Encoding".to_vec(), Object::Dict(encoding)); + font.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + font.set(b"ToUnicode".to_vec(), Object::Ref(cmap_ref)); + Ok(font) +} + /// The runs that show `text` in the fallback: consecutive characters in one font form one run, so /// a font is switched only where the text moves to another font. `None` if a character has no code. pub(super) fn type3_encode(fallback: &Type3Fallback, text: &str) -> Option)>> { diff --git a/crates/edit/src/text/interp.rs b/crates/edit/src/text/interp.rs index 2a9f6da61a..5114a76263 100644 --- a/crates/edit/src/text/interp.rs +++ b/crates/edit/src/text/interp.rs @@ -18,13 +18,33 @@ pub(super) struct TextState { /// graphics state the text was painted with that [`TextState::ops`] doesn't restate. pub(super) stroke: Vec, pub(super) line_width: Option, - ext_gstate: Vec>, + ext_gstate: Vec, /// The dash pattern and line cap in force (see [`Ts`]): compared, not restated. pub(super) dash: Dash, pub(super) line_cap: i64, } impl TextState { + /// The state a page's content starts in (ISO 32000-2 §8.4.1, §9.3.1): what is in force on + /// top of the page, after its content wrapped in `q`/`Q`. + pub(crate) fn initial() -> Self { + TextState { + font: None, + char_spacing: 0.0, + word_spacing: 0.0, + scale: 1.0, + leading: 0.0, + rise: 0.0, + fill: Vec::new(), + render_mode: 0, + stroke: Vec::new(), + line_width: None, + ext_gstate: Vec::new(), + dash: (Vec::new(), 0.0), + line_cap: 0, + } + } + pub(super) fn of(ts: &Ts) -> Self { TextState { font: ts.font.as_ref().map(|(n, _)| (String::from_utf8_lossy(n).into_owned(), ts.size)), @@ -54,13 +74,17 @@ impl TextState { /// /// A `gs` has no inverse, so a selection `current` has and this state doesn't stays in force: /// run these inside a `q`/`Q` that starts from `current`, one per state drawn. - pub(crate) fn paint_ops(&self, current: &TextState) -> Vec { + /// + /// `ctm` is the CTM where they run. A soft mask is placed by the CTM its `gs` ran at + /// (ISO 32000-2 §11.6.5.2), so one selected at another CTM is selected there and the CTM + /// set back. + pub(crate) fn paint_ops(&self, current: &TextState, ctm: [f64; 6]) -> Vec { let n = pdfcraft_content::num; let mut v = Vec::new(); // The selections both share stay in force; the ones after them are selected again in order. let shared = self.ext_gstate.iter().zip(¤t.ext_gstate).take_while(|(a, b)| a == b).count(); - for name in self.ext_gstate.iter().skip(shared) { - v.push(Op::new("gs", vec![Object::Name(name.clone())])); + for sel in self.ext_gstate.iter().skip(shared) { + sel.push_ops(&mut v, ctm); } // A reselected ExtGState may set `/LW`, so the width (already the effective one) follows it. let reselected = !v.is_empty(); @@ -222,7 +246,7 @@ pub(super) struct Ts { /// The line width a `w` or an ExtGState's `/LW` set (`None`: the initial 1.0). pub(super) line_width: Option, /// The ExtGState resources selected with `gs`, oldest first (see [`select_gs`]). - pub(super) ext_gstate: Vec>, + pub(super) ext_gstate: Vec, /// The dash pattern a `d` or an ExtGState's `/D` set (initially solid: `[] 0`). pub(super) dash: Dash, /// The line cap a `J` or an ExtGState's `/LC` set (initially 0, butt). @@ -255,12 +279,38 @@ const MAX_GS: usize = 32; /// Record `gs /name` in `list`. A `gs` sets every parameter its dictionary names to an absolute /// value, so a later selection of the same dictionary overrides the earlier one completely: the /// earlier entry goes, which keeps replaying the list equivalent to replaying the stream. -fn select_gs(list: &mut Vec>, name: &[u8]) { - list.retain(|n| n != name); +fn select_gs(list: &mut Vec, name: &[u8], anchor: Option<[f64; 6]>) { + list.retain(|s| s.name != name); if list.len() >= MAX_GS { list.remove(0); } - list.push(name.to_vec()); + list.push(GsSelection { name: name.to_vec(), anchor }); +} + +/// One `gs` selection: the ExtGState's resource name, and the CTM it ran at when the dictionary +/// has a soft mask, which that CTM places (ISO 32000-2 §11.6.5.2). +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct GsSelection { + name: Vec, + anchor: Option<[f64; 6]>, +} + +impl GsSelection { + /// Select this ExtGState from where the CTM is `ctm`: at its anchor, then back to `ctm`. Both + /// moves need an invertible matrix; without one it is selected where it runs. + fn push_ops(&self, v: &mut Vec, ctm: [f64; 6]) { + let gs = Op::new("gs", vec![Object::Name(self.name.clone())]); + let moves = self.anchor.filter(|a| *a != ctm).and_then(|a| { + let (anchor, here) = (Matrix(a), Matrix(ctm)); + Some((anchor.then(&here.invert()?), here.then(&anchor.invert()?))) + }); + let Some((to, back)) = moves else { + v.push(gs); + return; + }; + let cm = |m: Matrix| Op::new("cm", m.0.iter().map(|x| pdfcraft_content::num(*x)).collect()); + v.extend([cm(to), gs, cm(back)]); + } } /// The text-showing operators of one stream with their text, font, box and baseline. @@ -879,14 +929,15 @@ pub(super) fn interpret( b"Ts" => ts.rise = op.num(0).unwrap_or(0.0), b"Tr" => ts.render_mode = op.operands.first().and_then(Object::as_int).unwrap_or(-1), b"gs" => { - if let Some(name) = op.name(0) { - select_gs(&mut ts.ext_gstate, name); - } let mut held = None; let params = match op.name(0).and_then(|name| states.get(name)) { Some(p) => super::dict_in_place(doc, p, &mut held), None => None, }; + if let Some(name) = op.name(0) { + let masked = params.and_then(|p| p.get(b"SMask")).is_some_and(|m| doc.resolve(m).as_dict().is_some()); + select_gs(&mut ts.ext_gstate, name, masked.then_some(ts.ctm.0)); + } if let Some(params) = params { ts.gs.apply(doc, params); // `/LW` sets the line width as `w` does (ISO 32000-2 §8.4.5). diff --git a/crates/edit/src/text/rewrite.rs b/crates/edit/src/text/rewrite.rs index ec2fac984c..d27d23e0b5 100644 --- a/crates/edit/src/text/rewrite.rs +++ b/crates/edit/src/text/rewrite.rs @@ -702,11 +702,13 @@ fn block_ops(plan: &BlockPlan<'_>, writer: &LineWriter<'_>, wrapped: &[(String, let mut ops = vec![Op::new("q", vec![])]; ops.extend(reveal_ops(plan.block, plan.lines, plan.style, plan.emit.ctm)?); // Moved: a translation inside that state. + let mut ctm = Matrix(plan.emit.ctm); if let Some((ax, ay)) = plan.movement { ops.push(Op::new("cm", vec![n(1.0), n(0.0), n(0.0), n(1.0), n(ax), n(ay)])); + ctm = Matrix([1.0, 0.0, 0.0, 1.0, ax, ay]).then(&ctm); } // Painted as the paragraph was, including what its text object set before the first show. - ops.extend(plan.state.paint_ops(plan.after)); + ops.extend(plan.state.paint_ops(plan.after, ctm.0)); ops.push(Op::new("BT", vec![])); let mut state = plan.state.clone(); state.word_spacing = 0.0; @@ -816,6 +818,11 @@ pub(crate) fn rewrite_block_from( let first = members.first().copied().ok_or_else(|| EditError::Invalid("the paragraph has no source runs".into()))?; members.iter().try_for_each(|line| line.ensure_editable())?; let p = page_dict(doc, page)?; + // Revealed OCR text may go on top of the page (see `revealed_under_paint`): a page that can't + // take that is refused here, before either rewrite path changes anything. + if b.invisible { + crate::check_on_top(doc, &p)?; + } let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let mut fonts_res = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); let dispatch = choose_rewrite_path(doc, &fonts_res, &b, &members, first, text, style)?; @@ -930,9 +937,28 @@ pub(crate) fn rewrite_block_from( char_spacing: o_state.char_spacing, lead, }; - let plan = - BlockPlan { block: &b, lines: &lines, style, emit, after, movement, state: o_state, tm, size, underline: style.underline == Some(true) }; + let at = if split { start } else { emit.bt_op }; + // Revealed OCR text that something is painted over goes on top of the page, where it starts + // from the initial state. + let widest = wrapped.iter().map(|(l, _, _)| advance.of(l)).fold(width, f64::max); + let reach = crate::Reach { rect: b.rect, offset: [dx, dy], width: widest, rows: wrapped.len(), pitch: lead * k, upright }; + let over = crate::revealed_under_paint(doc, page, b.invisible, reach.bounds(), &ops, at); + let initial = TextState::initial(); + let drawn_from = if over { &initial } else { after }; + let plan = BlockPlan { + block: &b, + lines: &lines, + style, + emit, + after: drawn_from, + movement, + state: o_state, + tm, + size, + underline: style.underline == Some(true), + }; let mut block_ops = block_ops(&plan, &writer, &wrapped, font.show_font.clone())?; + let on_top = crate::lift_on_top(over, &mut block_ops); let n = pdfcraft_content::num; block_ops.extend(after.ops().into_iter().filter(|op| !op.is("Tf") || after.font.is_some())); if split { @@ -940,7 +966,6 @@ pub(crate) fn rewrite_block_from( block_ops.push(Op::new("BT", vec![])); block_ops.push(Op::new("Tm", emit.tlm.iter().map(|v| n(*v)).collect())); } - let at = if split { start } else { emit.bt_op }; let new_data = splice(&pieces, &ops, |i| { let insert = if i == at { std::mem::take(&mut block_ops) } else { Vec::new() }; (insert, !dropped.contains(&i)) @@ -957,12 +982,14 @@ pub(crate) fn rewrite_block_from( d.set(b"Resources".to_vec(), Object::Dict(res)); } })?; + crate::draw_on_top(doc, &p, emit.ctm, &on_top)?; Ok(LineEdit { substituted: font.substituted }) } /// The cover (when [`BlockStyle::cover`] asks for one) and visible rendering that start a /// rewritten invisible paragraph, drawn in the space whose CTM is `ctm`. Nothing for visible text. -/// The paragraph's `q`/`Q` restores the rendering mode for the invisible text after it. +/// The paragraph's `q`/`Q` restores the rendering mode for the invisible text after it; one +/// painted over where it was is drawn on top of the page instead ([`crate::draw_on_top`]). pub(crate) fn reveal_ops(block: &TextBlock, lines: &[TextLine], style: &BlockStyle, ctm: [f64; 6]) -> Result, EditError> { if !block.invisible { return Ok(Vec::new()); diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 885a0a7158..7a9204a8c0 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -1,3 +1,4 @@ +use super::super::interp::TextState; use super::grouping::{across, is_centred, row_rect}; use super::{ BlockStyle, CraftFont, Dict, Document, EditError, HashMap, LineEdit, MAX_TYPE3_GLYPHS, Matrix, Metrics, Object, Op, PdfString, Pieces, @@ -401,6 +402,16 @@ fn paragraph_geometry(doc: &Document, page: &pdfcraft_model::Page, block: &TextB Ok((dx, dy, style.width.unwrap_or(natural).clamp(1.0, 14_400.0))) } +/// How far below a row of `size` the next one is written: the source's leading unless a size +/// or line spacing is chosen. +fn row_step(style: &BlockStyle, size: f64, source_leading: f64) -> f64 { + if style.line_spacing.is_none() && style.size.is_none() { + source_leading + } else { + style.line_spacing.unwrap_or(1.5).clamp(0.5, 5.0) * style.size.unwrap_or(size) + } +} + type WrappedStyledRows = (StyledRows, RowRanges, Vec, Vec, Vec); fn wrap_styled_rows( @@ -976,6 +987,7 @@ pub(crate) fn rewrite_styled_block( let o = &lead.origin; // What is in force where the rewrite is drawn, and what the text after it expects. let after = if split { &o.state } else { &o.bt_state }; + let at = if split { start } else { o.bt_op }; validate_contextual_rows(&rows, &kept)?; let measured: Vec<_> = rows.iter().map(|row| measure(row)).collect::>()?; let shaped = prepare_shaped_rows(doc, &fonts, &rows, &kept, advance)?; @@ -989,6 +1001,7 @@ pub(crate) fn rewrite_styled_block( .ok_or_else(|| EditError::Invalid("the paragraph is drawn in a space it cannot be edited in".into()))?; let (tx, ty) = back.apply(dx, dy); block_ops.push(Op::new("cm", vec![n(1.0), n(0.0), n(0.0), n(1.0), n(tx), n(ty)])); + let run_ctm = Matrix([1.0, 0.0, 0.0, 1.0, tx, ty]).then(&Matrix(o.ctm)).0; // A new row can land one leading above the next paragraph and look like the same // paragraph. When every row is in this rewrite, a paragraph marker keeps them in one // scope so the neighbour stays out (#2517). A row left in the source stream cannot share @@ -1007,6 +1020,19 @@ pub(crate) fn rewrite_styled_block( }) .rfind(|v| *v > 0.0) .unwrap_or(first.size * 1.5); + // Revealed OCR text that something is painted over goes on top of the page, where it starts + // from the initial state. + let largest = + block.lines.iter().filter_map(|i| lines.get(*i)).map(|l| l.size).chain(style.ranges.iter().filter_map(|r| r.size)).fold(first.size, f64::max); + // At least one line of the largest glyphs, so a larger range reaches up as far as it draws. + const LINE_EM: f64 = 1.2; + let pitch = source_leading.max(row_step(style, largest, source_leading)).max(largest * LINE_EM); + let widest = measured.iter().copied().fold(width, f64::max); + // This path lays rows out along user-space baselines, so the paragraph is upright here. + let reach = crate::Reach { rect: block.rect, offset: [dx, dy], width: widest, rows: rows.len(), pitch, upright: true }; + let over = crate::revealed_under_paint(doc, page, block.invisible, reach.bounds(), &ops, at); + let initial = TextState::initial(); + let drawn_from = if over { &initial } else { after }; let mut baseline = first.origin.baseline; for (row_index, row) in rows.iter().enumerate() { let anchor = block.rows.get(row_index).and_then(|r| r.first()).and_then(|i| lines.get(*i)); @@ -1073,7 +1099,7 @@ pub(crate) fn rewrite_styled_block( // object (`gs`, stroke, line width, `Tr`; #3508/#3509) from the state the rewrite is // drawn in: a `gs` has no inverse, so one run's ExtGState would stay for the next. block_ops.push(Op::new("q", vec![])); - block_ops.extend(state.paint_ops(after)); + block_ops.extend(state.paint_ops(drawn_from, run_ctm)); if let Some((_, shaped, _)) = shaped_runs.get(&(row_index, pos)) { let resource = glyph_resources.get(&(row_index, pos)).ok_or_else(|| EditError::Invalid("shaped font is missing".into()))?; let mut properties = Dict::new(); @@ -1164,21 +1190,17 @@ pub(crate) fn rewrite_styled_block( x += segment_width; pos = end; } - baseline -= if style.line_spacing.is_none() && style.size.is_none() { - source_leading - } else { - style.line_spacing.unwrap_or(1.5).clamp(0.5, 5.0) * style.size.unwrap_or(anchor.size) - }; + baseline -= row_step(style, anchor.size, source_leading); } push_whole_paragraph_marker(&mut block_ops, marked, "EMC"); block_ops.push(Op::new("Q", vec![])); + let on_top = crate::lift_on_top(over, &mut block_ops); block_ops.extend(after.ops()); if split { block_ops.insert(0, Op::new("ET", vec![])); block_ops.push(Op::new("BT", vec![])); block_ops.push(Op::new("Tm", o.tlm.iter().map(|v| n(*v)).collect())); } - let at = if split { start } else { o.bt_op }; // A kept source row sits outside `block_ops`. Bound the whole paragraph when a marker is // safe. The close is not part of `block_ops`: that vector can end inside a text object // when the rewrite splits one, and a kept row's operators come later in the stream. @@ -1190,5 +1212,6 @@ pub(crate) fn rewrite_styled_block( d.set(b"Contents".to_vec(), contents); d.set(b"Resources".to_vec(), Object::Dict(res)); })?; + crate::draw_on_top(doc, &p, o.ctm, &on_top)?; Ok(LineEdit { substituted }) } diff --git a/crates/engine/src/actions.rs b/crates/engine/src/actions.rs index 3e942f6a2f..249279e09d 100644 --- a/crates/engine/src/actions.rs +++ b/crates/engine/src/actions.rs @@ -197,14 +197,14 @@ fn apply_step(s: &mut Session, id: DocId, step: &Step, log: &mut Vec) -> return Ok(None); } }; - // Steps that find nothing to do (no comments to flatten, …) aren't failures. A removal step - // that didn't remove is: the file would be passed on as cleaned with the data still in it. + // A step that finds nothing to do is not a failure. Any other error fails the file, so nothing is + // published for it: the file would otherwise be passed on without the step it was asked to apply. let removes = matches!(step, Step::RemoveHiddenInformation | Step::Sanitize); match s.apply(id, edit) { Ok(()) => log.push(step.label().to_string()), Err(EditError::Redact(pdfcraft_redact::RedactError::NothingToApply)) if removes => log.push(format!("{}: nothing to remove", step.label())), - Err(e) if removes => return Err(e), - Err(e) => log.push(format!("{}: {e}", step.label())), + Err(EditError::Redact(pdfcraft_redact::RedactError::NothingToApply)) => log.push(format!("{}: nothing to do", step.label())), + Err(e) => return Err(e), } Ok(None) } diff --git a/crates/engine/src/camera_clean.rs b/crates/engine/src/camera_clean.rs index 9c4120be56..d1768a2a83 100644 --- a/crates/engine/src/camera_clean.rs +++ b/crates/engine/src/camera_clean.rs @@ -1,6 +1,8 @@ -//! Clean up a straightened camera photo (Enhance ▸ Camera image ▸ enhancement level): even out -//! the lighting so the paper turns white (room-light colour casts, a hand's shadow, darker -//! edges go with it), then raise the contrast and drop isolated specks. Colour ink stays colour. +//! Clean up a straightened camera photo (Enhance ▸ Camera image ▸ enhancement level) or a +//! scanned page's picture (Enhance ▸ Scanned document, [`crate::scan_clean`]): even out the +//! lighting so the paper turns white (room-light colour casts, a hand's shadow, darker edges and +//! book gutters go with it), then raise the contrast and drop isolated specks. Colour ink stays +//! colour. //! //! The paper's illumination is estimated per channel on a coarse grid of blocks, from each //! block's bright end (the paper, not the ink), smoothed into a continuous map and divided out. @@ -8,6 +10,8 @@ /// The strongest a level can be. pub const MAX_LEVEL: u8 = 100; +/// The level from which [`clean`] also drops isolated specks. +pub const DESPECKLE_FROM: u8 = 60; /// How an enhanced camera page looks. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -66,34 +70,158 @@ pub fn apply(rgb: &[u8], width: u32, height: u32, look: Look, level: u8) -> Opti let even = clean(rgb, width, height, 50)?; let t = 0.45 + 0.4 * level.min(MAX_LEVEL) as f32 / MAX_LEVEL as f32; let mut grey: Vec = even.as_chunks::<3>().0.iter().map(|p| if (lum(p) as f32 / 255.0) < t { 0 } else { 255 }).collect(); - despeckle_grey(&mut grey, w, h); + despeckle(&mut grey, 1, w, h); Pixels::Grey(grey) } }) } -/// Clean `rgb` (RGB8, `width` × `height`) at `level` (0 = unchanged … 100 = strongest). -/// Returns `None` when the buffer doesn't match the size. +/// Clean `rgb` (RGB8, `width` × `height`) at `level` (0 = unchanged … 100 = strongest), from +/// [`DESPECKLE_FROM`] without isolated specks. Returns `None` when the buffer doesn't match the +/// size. pub fn clean(rgb: &[u8], width: u32, height: u32, level: u8) -> Option> { + clean_samples(rgb, 3, width, height, CleanUp { level, edge_shadows: false, despeckle: level >= DESPECKLE_FROM }) +} + +/// What [`clean_samples`] does. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CleanUp { + /// 0 (unchanged) … 100: even out the paper's illumination and raise the contrast, as + /// [`clean`]'s level. + pub level: u8, + /// Even out shadows along the picture's edges first (a book's gutter, a copier's dark edge): + /// lightened line by line to the paper beside them, within the outer fifth of each side. + pub edge_shadows: bool, + /// Drop isolated dark specks. + pub despeckle: bool, +} + +/// [`clean`] for samples of `n` components (1: grey, 3: RGB), with each step chosen apart (each +/// works at level 0 too). Returns `None` when `n` is neither or the buffer doesn't match the size. +pub fn clean_samples(px: &[u8], n: usize, width: u32, height: u32, how: CleanUp) -> Option> { let (w, h) = (width as usize, height as usize); - if w == 0 || h == 0 || w.checked_mul(h)?.checked_mul(3)? != rgb.len() { + if !(n == 1 || n == 3) || w == 0 || h == 0 || w.checked_mul(h)?.checked_mul(n)? != px.len() { return None; } - let level = level.min(MAX_LEVEL); - if level == 0 { - return Some(rgb.to_vec()); + let level = how.level.min(MAX_LEVEL); + let mut out = px.to_vec(); + if how.edge_shadows { + even_edges(&mut out, n, w, h); } + if level > 0 { + out = even_out(&out, n, w, h, level); + } + if how.despeckle { + despeckle(&mut out, n, w, h); + } + Some(out) +} + +/// The share of the width (and of the height) at each side where [`even_edges`] works. +const EDGE_BAND: f32 = 0.2; +/// The most [`even_edges`] lightens a line by; a darker line (a scanner's black border, a +/// picture) is not taken for shadowed paper and stays as it is. +const MAX_EDGE_GAIN: f32 = 3.0; + +/// Even out shadows that run along the picture's edges, in place: a book's gutter, a copier's +/// dark border, a page that curved away from the glass. Within the outer [`EDGE_BAND`] of each +/// side, each column (then each row) gets the paper's level along it (per channel, its 90th +/// percentile, so the text across it doesn't count), and is lightened to the brightest paper of +/// that band. The block grid of [`background`] is too coarse for such a narrow shadow: its +/// blocks borrow the brighter paper next to them. +fn even_edges(px: &mut [u8], n: usize, w: usize, h: usize) { + for vertical in [true, false] { + // Columns: lines across the width, each running down the height; rows the other way. + let (lines, along) = if vertical { (w, h) } else { (h, w) }; + let band = ((lines as f32 * EDGE_BAND) as usize).min(lines / 2); + if band == 0 { + continue; + } + let at = |line: usize, i: usize| if vertical { i * w + line } else { line * w + i }; + // The paper's level of every line in the bands and a few beyond them. + let reach = band.saturating_add(3).min(lines); + let measured = |line: usize| line < reach || line >= lines.saturating_sub(reach); + let mut level: Vec> = vec![None; lines]; + let mut hist = [[0u32; 256]; 3]; + for (line, slot) in level.iter_mut().enumerate().filter(|(line, _)| measured(*line)) { + hist.iter_mut().for_each(|hh| hh.fill(0)); + for i in 0..along { + let p = at(line, i) * n; + for (c, hh) in hist.iter_mut().enumerate().take(n) { + if let Some(v) = px.get(p + c) + && let Some(count) = hh.get_mut(*v as usize) + { + *count += 1; + } + } + } + let want = (along as f32 * 0.9) as u32; + *slot = Some(std::array::from_fn(|c| { + let mut seen = 0; + let v = hist.get(c).and_then(|hh| { + hh.iter().position(|&count| { + seen += count; + seen > want + }) + }); + v.unwrap_or(255) as f32 / 255.0 + })); + } + // Smoothed: the median of five neighbouring lines (fewer at the very edge, centred all + // the same, so a gradient isn't shifted), so one odd line (a rule, a fold) doesn't stand + // out. + let smooth: Vec> = (0..lines) + .map(|line| { + level.get(line).copied().flatten()?; + let r = line.min(lines - 1 - line).min(2); + Some(std::array::from_fn(|c| { + let mut near: Vec = (line - r..=line + r).filter_map(|l| level.get(l).copied().flatten()).map(|v| v[c]).collect(); + near.sort_by(f32::total_cmp); + near.get(near.len() / 2).copied().unwrap_or(1.0) + })) + }) + .collect(); + // Each side's band, and the lines its paper is measured on (the band and a few beyond). + for (shadowed, measured) in [(0..band, 0..reach), (lines - band..lines, lines.saturating_sub(reach)..lines)] { + // The brightest paper there: what the shadow is lightened to. + let paper: [f32; 3] = + std::array::from_fn(|c| measured.clone().filter_map(|l| smooth.get(l).copied().flatten()).map(|v| v[c]).fold(0.0, f32::max)); + for line in shadowed { + let Some(own) = smooth.get(line).copied().flatten() else { continue }; + let gain: [f32; 3] = std::array::from_fn(|c| { + let g = paper[c] / own[c].max(1e-3); + if (1.0..=MAX_EDGE_GAIN).contains(&g) { g } else { 1.0 } + }); + if gain.iter().take(n).all(|&g| g == 1.0) { + continue; + } + for i in 0..along { + let p = at(line, i) * n; + for (c, g) in gain.iter().enumerate().take(n) { + if let Some(v) = px.get_mut(p + c) { + *v = (*v as f32 * g).round().clamp(0.0, 255.0) as u8; + } + } + } + } + } + } +} + +/// The lighting evened out and the contrast raised at `level` (1–100); `px` holds `w` × `h` +/// pixels of `n` components. +fn even_out(px: &[u8], n: usize, w: usize, h: usize, level: u8) -> Vec { let k = level as f32 / MAX_LEVEL as f32; - let background = background(rgb, w, h); + let background = background(px, n, w, h); // Lighting is evened out fully from half strength on; contrast keeps growing to the top. let flatten = (2.0 * k).min(1.0); let white = 1.0 - 0.12 * k; // what reaches white let black = 0.30 * k; // what reaches black - let mut out = Vec::with_capacity(rgb.len()); - for (i, px) in rgb.as_chunks::<3>().0.iter().enumerate() { + let mut out = Vec::with_capacity(px.len()); + for (i, p) in px.chunks_exact(n).enumerate() { let (x, y) = (i % w, i / w); let bg = background.at(x, y); - for (c, &v) in px.iter().enumerate() { + for (c, &v) in p.iter().enumerate() { let v = v as f32 / 255.0; let b = bg.get(c).copied().unwrap_or(1.0).max(0.05); let even = (v / b).min(1.0); @@ -102,10 +230,7 @@ pub fn clean(rgb: &[u8], width: u32, height: u32, level: u8) -> Option> out.push((stretched * 255.0).round() as u8); } } - if level >= 60 { - despeckle(&mut out, w, h); - } - Some(out) + out } /// The paper's illumination: a grid of per-channel values, read back with bilinear @@ -140,9 +265,10 @@ impl Background { } } -/// Estimate the paper's illumination: per block, the 90th percentile of each channel (bright -/// paper, not dark ink); blocks that are mostly ink borrow from their neighbours; then smoothed. -fn background(rgb: &[u8], w: usize, h: usize) -> Background { +/// Estimate the paper's illumination: per block, the 90th percentile of each of the `n` channels +/// (bright paper, not dark ink); blocks that are mostly ink borrow from their neighbours; then +/// smoothed. Channels past `n` stay white. +fn background(px: &[u8], n: usize, w: usize, h: usize) -> Background { let block = (w.min(h) / 16).clamp(8, 128); let (cols, rows) = (w.div_ceil(block), h.div_ceil(block)); let mut cells = Vec::with_capacity(cols * rows); @@ -150,20 +276,20 @@ fn background(rgb: &[u8], w: usize, h: usize) -> Background { for by in 0..rows { for bx in 0..cols { hist.iter_mut().for_each(|hh| hh.fill(0)); - let mut n = 0u32; + let mut count = 0u32; for y in by * block..((by + 1) * block).min(h) { for x in bx * block..((bx + 1) * block).min(w) { - if let Some(p) = rgb.get((y * w + x) * 3..(y * w + x) * 3 + 3) { + if let Some(p) = px.get((y * w + x) * n..(y * w + x) * n + n) { for (c, &v) in p.iter().enumerate() { if let Some(slot) = hist.get_mut(c).and_then(|hh| hh.get_mut(v as usize)) { *slot += 1; } } - n += 1; + count += 1; } } } - let want = (n as f32 * 0.9) as u32; + let want = (count as f32 * 0.9) as u32; cells.push(std::array::from_fn(|c| { let mut seen = 0; let v = hist.get(c).and_then(|hh| { @@ -219,19 +345,15 @@ fn background(rgb: &[u8], w: usize, h: usize) -> Background { Background { cols, rows, block: block as f32, cells } } -/// [`despeckle`] for one grey byte per pixel. -fn despeckle_grey(grey: &mut [u8], w: usize, h: usize) { - let mut rgb: Vec = grey.iter().flat_map(|&v| [v, v, v]).collect(); - despeckle(&mut rgb, w, h); - for (g, p) in grey.iter_mut().zip(rgb.as_chunks::<3>().0.iter()) { - *g = p[0]; - } -} - -/// Lighten dark pixels with at most one dark neighbour (specks), in place. -fn despeckle(rgb: &mut [u8], w: usize, h: usize) { - let lum = |p: &[u8; 3]| (p[0] as u32 * 299 + p[1] as u32 * 587 + p[2] as u32 * 114) / 1000; - let dark: Vec = rgb.as_chunks::<3>().0.iter().map(|p| lum(p) < 128).collect(); +/// Lighten dark pixels with at most one dark neighbour (specks), in place; `px` holds `w` × `h` +/// pixels of `n` components (1 or 3). +fn despeckle(px: &mut [u8], n: usize, w: usize, h: usize) { + let lum = |p: &[u8]| match p { + [r, g, b] => (*r as u32 * 299 + *g as u32 * 587 + *b as u32 * 114) / 1000, + [v, ..] => *v as u32, + [] => 255, + }; + let dark: Vec = px.chunks_exact(n.max(1)).map(|p| lum(p) < 128).collect(); let is_dark = |x: isize, y: isize| { x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h && dark.get(y as usize * w + x as usize).copied().unwrap_or(false) }; @@ -243,7 +365,7 @@ fn despeckle(rgb: &mut [u8], w: usize, h: usize) { let neighbours = (-1..=1).flat_map(|dy| (-1..=1).map(move |dx| (dx, dy))).filter(|&(dx, dy)| (dx, dy) != (0, 0) && is_dark(x + dx, y + dy)).count(); if neighbours <= 1 - && let Some(p) = rgb.get_mut((y as usize * w + x as usize) * 3..(y as usize * w + x as usize) * 3 + 3) + && let Some(p) = px.get_mut((y as usize * w + x as usize) * n..(y as usize * w + x as usize) * n + n) { p.fill(255); } @@ -388,5 +510,40 @@ mod tests { assert_eq!(clean(&[], 0, 0, 50), None); assert_eq!(clean(&[0; 12], 2, 3, 50), None); assert_eq!(clean(&[9; 3], 1, 1, 255).map(|v| v.len()), Some(3)); + let every = CleanUp { level: 100, edge_shadows: true, despeckle: true }; + assert_eq!(clean_samples(&[0; 12], 2, 2, 3, every), None, "two components"); + assert_eq!(clean_samples(&[0; 5], 1, 2, 3, every), None); + for (w, h) in [(1, 1), (1, 9), (9, 1), (2, 2), (5, 3)] { + assert_eq!(clean_samples(&vec![100; w * h], 1, w as u32, h as u32, every).map(|v| v.len()), Some(w * h)); + } + } + + #[test] + fn edge_shadows_lighten_shadowed_paper_but_not_a_black_border_or_the_middle() { + // Grey paper (200) with a shadow darkening the left 20 columns to 120, a black scanner + // border in the right 8 columns, and a dark block in the middle. + let (w, h) = (100usize, 60usize); + let mut grey = vec![200u8; w * h]; + for y in 0..h { + for x in 0..w { + let v = if x >= w - 8 { + 5 + } else if (40..60).contains(&x) && (20..40).contains(&y) { + 30 + } else if x < 20 { + 120 + (80 * x / 20) as u8 + } else { + 200 + }; + grey[y * w + x] = v; + } + } + let out = clean_samples(&grey, 1, w as u32, h as u32, CleanUp { level: 0, edge_shadows: true, despeckle: false }).unwrap(); + for x in [0, 5, 19] { + assert!(out[30 * w + x].abs_diff(200) <= 3, "column {x}: {} → {}", grey[30 * w + x], out[30 * w + x]); + } + assert_eq!(out[30 * w + w - 3], 5, "a black border is no shadow"); + assert_eq!(out[30 * w + 50], 30, "the middle is untouched"); + assert_eq!(out[30 * w + 30], 200); } } diff --git a/crates/engine/src/catalog.rs b/crates/engine/src/catalog.rs index e479232bbb..d6be8fd4eb 100644 --- a/crates/engine/src/catalog.rs +++ b/crates/engine/src/catalog.rs @@ -265,9 +265,9 @@ pub static TOOL_GROUPS: &[ToolGroup] = &[ items: &[ item("In this file", "scan-text", "ocr.recognize", Ready), item("In multiple files", "files", "ocr.recognize_batch", Ready), - item("Enhance scanned file", "sparkles", "ocr.enhance", Planned("M10")), + item("Enhance scanned file", "sparkles", "ocr.enhance", Ready), item("Enhance camera image", "camera", "ocr.enhance_camera", Ready), - item("Correct recognized text", "text-select", "ocr.correct", Planned("M10")), + item("Correct recognized text", "text-select", "ocr.correct", Ready), ], }], }, diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 0647726452..7cee14657e 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -220,6 +220,10 @@ pub const COMMANDS: &[CommandSpec] = &[ // Alt-drag for moving windows (several Linux window managers). // Copy events stay with the frontend so focused text inputs and page clipboard routing keep ownership. c("edit.copy", "Copy", EDIT, None, Copying, "copy"), + // Comments travel between documents as XFDF (M14): ⌘C with comments selected copies them, + // ⌘V pastes the copies onto the current page; the strip's Copy/Paste in the menus too. + c("comment.copy", "Copy comments", EDIT, None, Document, "copy"), + c("comment.paste", "Paste comments", EDIT, None, Annotate, "clipboard-paste"), c("edit.column_select", "Column select", EDIT, None, Document, "text-select"), c("edit.snapshot", "Take a snapshot", EDIT, None, Copying, "camera"), c("view.full_screen", "Full screen mode", VIEW, Some(Shortcut::cmd("L")), Document, "maximize"), @@ -355,7 +359,9 @@ pub const COMMANDS: &[CommandSpec] = &[ c("tools.document_js", "Document JavaScripts…", None, None, Modification, "file-code"), c("ocr.recognize", "Recognize text…", None, None, Modification, "scan-text"), c("ocr.enhance_camera", "Enhance camera image", None, None, Modification, "camera"), + c("ocr.enhance", "Enhance scanned file…", None, None, Modification, "sparkles"), c("ocr.recognize_batch", "Recognize text in multiple files…", None, None, Nothing, "files"), + c("ocr.correct", "Correct recognized text…", None, None, Document, "text-select"), c("a11y.check", "Check for accessibility…", None, None, Document, "accessibility"), c("a11y.find_untagged", "Find untagged content", None, None, Document, "search"), c("a11y.autotag", "Automatically tag PDF", None, None, Document, "tag"), diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..de739101d5 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -63,6 +63,15 @@ impl Session { Ok(pdfcraft_compare::compare(&a.words(), &b.words())) } + /// [`Session::compare`], keeping the changes on `pages` of `new` (0-based; empty: every page). + pub fn compare_pages(&self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let mut c = self.compare(old, new)?; + if !pages.is_empty() { + c.changes.retain(|ch| pages.iter().any(|page| (ch.new.page..=ch.new.last_page).contains(page))); + } + Ok(c) + } + /// Visual compare: regions where page n of `new` looks different from page n of `old` /// (rendered at `dpi`), as (page, user-space box in `new`). pub fn compare_visual(&self, old: DocId, new: DocId, dpi: f32) -> Result, EditError> { @@ -95,7 +104,12 @@ impl Session { /// The compare report as a new PDF (not opened). pub fn compare_report(&self, old: DocId, new: DocId) -> Result>, EditError> { - let c = self.compare(old, new)?; + self.compare_report_of(old, new, &[]) + } + + /// [`Session::compare_report`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn compare_report_of(&self, old: DocId, new: DocId, pages: &[usize]) -> Result>, EditError> { + let c = self.compare_pages(old, new, pages)?; let name = |id| self.get(id).map(|d| d.name.clone()).unwrap_or_default(); self.create_report("Compare Report", &pdfcraft_compare::report(&c, &name(old), &name(new))) } @@ -104,7 +118,12 @@ impl Session { /// (blue, green) and a note where text was deleted (red), authored "Compare". One undoable /// step; returns how many comments were added. pub fn mark_differences(&mut self, old: DocId, new: DocId) -> Result { - let c = self.compare(old, new)?; + self.mark_differences_of(old, new, &[]) + } + + /// [`Session::mark_differences`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn mark_differences_of(&mut self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let c = self.compare_pages(old, new, pages)?; let mut edits = Vec::new(); for ch in &c.changes { let color = colour(ch.kind); @@ -187,6 +206,20 @@ pub struct OfficeExport { /// `Err(false)`: it can't be encoded). type PreparedImage = Result<(&'static str, Arc<[u8]>), bool>; +/// Displayed points per user-space unit on `info`'s page (`/UserUnit`), 1 when the page has no +/// usable size. +pub(crate) fn display_unit(info: &pdfcraft_render::PageInfo) -> f64 { + let crop_width = f64::from(info.crop[2] - info.crop[0]); + let display_width = f64::from(if info.rotation.is_multiple_of(180) { info.width } else { info.height }); + if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 } +} + +/// Bold and italic as a font's name says them (`Arial-BoldItalicMT`). +pub(crate) fn weight_and_slant(base_font: &str) -> (bool, bool) { + let f = base_font.to_ascii_lowercase(); + (f.contains("bold") || f.contains("black") || f.contains("heavy"), f.contains("italic") || f.contains("oblique")) +} + impl crate::Document { /// The pages as paragraphs and images (for Word, HTML and RTF export), including what /// form XObjects draw. @@ -228,9 +261,7 @@ impl crate::Document { let b = info.user_to_view(r[2] as f32, r[3] as f32); [a[0].min(b[0]) as f64, (info.height - a[1].max(b[1])) as f64, a[0].max(b[0]) as f64, (info.height - a[1].min(b[1])) as f64] }; - let crop_width = f64::from(info.crop[2] - info.crop[0]); - let display_width = f64::from(if info.rotation % 180 == 0 { info.width } else { info.height }); - let unit = if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 }; + let unit = display_unit(info); let reading = pdfcraft_edit::reading_blocks_checked(cos, i).unwrap_or_default(); let forms_unread = reading.truncated; let blocks = reading @@ -238,7 +269,7 @@ impl crate::Document { .into_iter() .filter(|b| !b.text.trim().is_empty()) .map(|b| { - let f = b.base_font.to_ascii_lowercase(); + let (bold, italic) = weight_and_slant(&b.base_font); // `text` is trimmed; the editor text keeps the source's own spaces. let space_around = [b.editor_text.starts_with(char::is_whitespace), b.editor_text.ends_with(char::is_whitespace)]; pdfcraft_export::Block { @@ -246,8 +277,8 @@ impl crate::Document { text: b.text, rect: visible_rect(b.rect), size: b.size * unit, - bold: f.contains("bold") || f.contains("black") || f.contains("heavy"), - italic: f.contains("italic") || f.contains("oblique"), + bold, + italic, color: b.color, rotation: b.rotation - f64::from(info.rotation), } diff --git a/crates/engine/src/copy_image.rs b/crates/engine/src/copy_image.rs index 53b890f27c..9d5668ccf5 100644 --- a/crates/engine/src/copy_image.rs +++ b/crates/engine/src/copy_image.rs @@ -112,7 +112,7 @@ pub(crate) fn added_picture(cos: &Cos, file: &[u8]) -> Option { } /// The largest raster image Form `form` draws through its resources (nested Forms included). -fn picture_in_form(cos: &Cos, form: ObjRef) -> Option { +pub(crate) fn picture_in_form(cos: &Cos, form: ObjRef) -> Option { let mut seen = HashSet::from([form]); let mut best: Option<(u64, ObjRef)> = None; let res = resources(cos, form)?; diff --git a/crates/engine/src/dates.rs b/crates/engine/src/dates.rs index c9382c1b02..5ede67c811 100644 --- a/crates/engine/src/dates.rs +++ b/crates/engine/src/dates.rs @@ -582,6 +582,18 @@ pub fn date_language(code: &str) -> Option<&'static DateLanguage> { DATE_LANGUAGES.iter().find(|l| l.code.eq_ignore_ascii_case(code.trim())) } +/// Form date fields (`AFDate_FormatEx`, `util.printd`, `util.scand`) write month and weekday +/// names in date language `code` from now on, as a localized Acrobat writes its own language's: +/// `dd-mmm-yy` is `18-ott-26` in Italian (#5303). An unknown code is English. +pub fn use_form_date_language(code: &str) { + let l = date_language(code).unwrap_or(&DATE_LANGUAGES[0]); + let names = pdfcraft_forms::af::DateNames { months: l.months, months_short: l.months_short, days: l.days, days_short: l.days_short }; + if pdfcraft_forms::af::date_names() != names { + pdfcraft_forms::af::set_date_names(names); + pdfcraft_js::set_date_names(l.months, l.months_short, l.days, l.days_short); + } +} + /// The characters of `text` that Fill & Sign can't write into a PDF yet, each once. Its text is /// drawn in Helvetica with WinAnsi encoding (Western European letters), so anything else (`ř`, /// Japanese, Chinese) would become `?` in the file. diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index fa98a18417..5405467edd 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -8,7 +8,8 @@ use pdfcraft_render::{DocInfo, RenderConfig, RenderPool}; use crate::history::Editor; use crate::{ CalculationOrder, CheckStyle, DocId, Edit, EditError, EditableObject, FormFieldKind, Hidden, InitialView, Label, MarkKind, SecuritySummary, - SignatureInfo, TabOrder, TrustStore, XfaLayout, a11y, form_appearance, guard, js, measure, optimize, renderer_password, unrestricted, + SignatureInfo, StatusCaptions, TabOrder, TrustStore, XfaLayout, a11y, form_appearance, guard, js, measure, optimize, renderer_password, + signatures_of, unrestricted, }; #[cfg(test)] use crate::{record_link_scan, record_marker_scan}; @@ -19,8 +20,9 @@ pub struct Document { pub path: Option, /// The working file: what Save writes. pub bytes: Arc>, - /// What is displayed (the page view, `page_render`, page images): `bytes`, plus appearances - /// drawn for display only for comments that have none ([`display_bytes`]). Never saved. + /// What is displayed (the page view, `page_render`, page images): `bytes`, plus what is drawn + /// for display only ([`display_bytes`]): appearances for comments that have none, and the + /// states of named-layer signatures. Never saved. pub display: Arc>, pub info: DocInfo, pub renderer: RenderPool, @@ -41,6 +43,9 @@ pub struct Document { /// Signature fields, validated against the session's trust store (Signatures panel). pub signatures: Arc>, pub(crate) trust: Arc, + /// The words signature status layers are drawn with (the session's, see + /// [`Session::set_signature_captions`](crate::Session::set_signature_captions)). + pub(crate) captions: Arc, pub(crate) sig_cache: Arc, pub(crate) editor: Option, pub(crate) config: RenderConfig, @@ -539,7 +544,7 @@ impl Document { _ => "Attachments only", }; let pending = editor.cos.encryption_changed(); - let permissions = if pending { pdfcraft_cos::Permissions { bits: h.permissions().bits, owner: false } } else { h.permissions() }; + let permissions = if pending { pdfcraft_cos::Permissions { owner: false, ..h.permissions() } } else { h.permissions() }; Some(SecuritySummary { method: method.into(), owner: h.auth() == pdfcraft_cos::Auth::Owner && !pending, permissions, pending }) } @@ -631,6 +636,12 @@ impl Document { pub fn repair_log(&self) -> Vec { self.editor.as_ref().map(|e| e.cos.repair_log().to_vec()).unwrap_or_default() } + + /// Objects of the file that could not be read and read as null, one line each (see + /// [`pdfcraft_cos::Document::unreadable_objects`]): only those read so far. + pub fn unreadable_objects(&self) -> Vec { + self.editor.as_ref().map(|e| e.cos.unreadable_objects()).unwrap_or_default() + } } /// A title that names no document: what a web browser writes when it saves a blank page or a @@ -643,25 +654,84 @@ pub(crate) fn is_placeholder_title(title: &str) -> bool { address || t == "untitled" } -/// The working file plus appearances for comments that have none, and for form fields the file -/// leaves to the viewer (#3470). Signature artwork is preserved; validation badges belong to the -/// viewer and never enter these bytes, which image/text exports and Print as Image also consume. -pub(crate) fn display_bytes(editor: Option<&mut Editor>, bytes: &Arc>) -> Arc> { +/// What is displayed: the working file plus, in memory only and never saved, the appearances +/// PdfCraft draws for comments that have none ([`pdfcraft_annot::with_missing_appearances`]) and +/// for form fields the file leaves to the viewer (#3470), and the signature states drawn into +/// named-layer signature appearances ([`pdfcraft_sign::show_status_layers`]). The working file is +/// unchanged, and so is everything that reads it (saving, signatures, …). Image and text exports +/// and Print as Image read these bytes, so they show the page as displayed; validation badges +/// belong to the viewer and never enter them. +/// +/// `signatures` is input and output: each one whose state gets drawn comes back marked +/// (`verdict_shown`). A mark describes the bytes returned, so every mark is cleared first and a +/// copy that cannot be written takes its marks with it. +pub(crate) fn display_bytes( + editor: Option<&mut Editor>, + bytes: &Arc>, + signatures: &mut [SignatureInfo], + captions: &StatusCaptions, +) -> Arc> { + for sig in signatures.iter_mut() { + sig.verdict_shown = false; + } let Some(e) = editor else { return bytes.clone() }; let comments = pdfcraft_annot::with_missing_appearances_cached(&e.cos, &mut e.appearances); let fields = pdfcraft_forms::with_field_appearances_cached(comments.as_ref().unwrap_or(&e.cos), &mut e.field_appearances); - let Some(display) = fields.or(comments) else { + if let Some(mut display) = fields.or(comments) { + // Written whether or not a signature changed: the copy exists for the comments and fields. + let _ = pdfcraft_sign::show_status_layers(&mut display, signatures, captions); + return write_display_copy(e, bytes, &display, signatures); + } + // No comment or field needed an appearance, so there is no copy yet: build one only when a + // signature's state would change it. + if !pdfcraft_sign::has_status_layers(&e.cos, signatures) { return bytes.clone(); - }; - match write_incremental(&display, &SaveOptions::default()) { + } + let mut display = e.cos.clone(); + if !pdfcraft_sign::show_status_layers(&mut display, signatures, captions) { + return bytes.clone(); + } + write_display_copy(e, bytes, &display, signatures) +} + +/// Write the display copy, or keep the working file when it cannot be written. The page then +/// shows the file's own artwork, so the marks are cleared. Losing the copy is a repair, so it is +/// logged, once however many refreshes lose it again. +pub(crate) fn write_display_copy( + e: &mut Editor, + bytes: &Arc>, + display: &pdfcraft_cos::Document, + signatures: &mut [SignatureInfo], +) -> Arc> { + match write_incremental(display, &SaveOptions::default()) { Ok(written) => Arc::new(written), Err(err) => { - e.cos.note_repair(format!("missing comment or field appearances could not be drawn ({err})")); + for sig in signatures.iter_mut() { + sig.verdict_shown = false; + } + let line = format!("the display copy could not be written, so pages show the file as stored ({err})"); + if !e.cos.repair_log().contains(&line) { + e.cos.note_repair(line); + } bytes.clone() } } } +/// Carry the marks across a revalidation that leaves the display copy alone, when it may: only +/// when every verdict is the one the copy was drawn with. Returns whether it did; otherwise the +/// copy has to be drawn again. +pub(crate) fn carry_verdict_marks(previous: &[SignatureInfo], next: &mut [SignatureInfo]) -> bool { + let unchanged = previous.len() == next.len() + && previous.iter().zip(next.iter()).all(|(p, n)| p.field == n.field && p.revision == n.revision && p.status == n.status); + if unchanged { + for (p, n) in previous.iter().zip(next.iter_mut()) { + n.verdict_shown = p.verdict_shown; + } + } + unchanged +} + /// `comments` (the display copy with comment appearances, if it needed any; else the document /// itself) with its form fields drawn from their values where the file leaves that to the viewer. fn with_field_appearances(comments: Option, doc: &pdfcraft_cos::Document) -> Option { @@ -688,17 +758,28 @@ pub(crate) fn info_fields<'a>(info: &'a mut DocInfo, cos: &pdfcraft_cos::Documen ] } -/// The copy PDFCraft displays when opening `bytes`: with appearances drawn, in memory only, for -/// comments that have none and form fields the file leaves to the viewer. Render paths outside the app, such as the CLI, use this so their page -/// images agree with the app and independent viewers. It never changes the source document, and an -/// unparseable file, a write failure or a parser panic falls back to the original bytes. +/// The copy PDFCraft displays when opening `bytes` with no trusted certificates and English +/// captions ([`display_bytes`]): appearances drawn, in memory only, for comments that have none +/// and form fields the file leaves to the viewer, and the state of each named-layer signature. +/// Render paths outside the app, such as the CLI, use this so their page images agree with the app +/// and independent viewers. It never changes the source document: a file with nothing to draw, an +/// unparseable file, a write failure or a parser panic gives back the original bytes. pub fn display_bytes_for(bytes: Arc>, password: Option<&str>) -> Arc> { guard(|| { - pdfcraft_cos::Document::open_with_password(bytes.clone(), password) - .ok() - .and_then(|cos| with_field_appearances(pdfcraft_annot::with_missing_appearances(&cos), &cos)) - .and_then(|display| write_incremental(&display, &SaveOptions::default()).ok()) - .map(Arc::new) + let cos = pdfcraft_cos::Document::open_with_password(bytes.clone(), password).ok()?; + let mut signatures = signatures_of(&cos, &bytes, &TrustStore::default(), &pdfcraft_sign::DigestCache::default()); + let layered = pdfcraft_sign::has_status_layers(&cos, &signatures); + let (mut display, added) = match with_field_appearances(pdfcraft_annot::with_missing_appearances(&cos), &cos) { + Some(display) => (display, true), + None if layered => (cos, false), + None => return None, + }; + // As in the app, a layer recognised but not drawn after all makes no copy. + let drawn = layered && pdfcraft_sign::show_status_layers(&mut display, &mut signatures, &StatusCaptions::default()); + if !(added || drawn) { + return None; + } + write_incremental(&display, &SaveOptions::default()).ok().map(Arc::new) }) .ok() .flatten() diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index 8f43689aa9..1bcf24108f 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -3,10 +3,11 @@ use std::sync::Arc; use crate::{ - AddedContent, AddedText, Background, BoxSpec, EditError, FieldProps, FieldValue, FillMark, HeaderFooter, Hidden, ImageEdit, InitialView, Label, - LineEnding, LinkAction, LinkStyle, MarkFile, MarkKind, Markup, NewAnnotation, NewField, NoteIcon, ObjectTarget, PageBox, Protection, ReviewState, - Rgb, Shape, Standard, Style, TabOrder, TransitionSpec, Watermark, a11y, copy_image, decode_data_file, embed_image, field_table, fit_picture, - image_content_type, is_xfa_data_file, js, mark_source, measure, open_source, strip_xfa, unrestricted, xfa, xfa_after_data_change, + AddedContent, AddedText, Background, BorderShape, BoxSpec, EditError, FieldProps, FieldValue, FillMark, HeaderFooter, Hidden, ImageEdit, + InitialView, Label, LineEnding, LinkAction, LinkStyle, MarkFile, MarkKind, Markup, NewAnnotation, NewField, NoteIcon, ObjectTarget, PageBox, + Protection, ReviewState, Rgb, Shape, Standard, Style, TabOrder, TransitionSpec, Watermark, a11y, copy_image, decode_data_file, embed_image, + field_table, fit_picture, image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, open_source, strip_xfa, unrestricted, + xfa, xfa_after_data_change, }; mod labels; @@ -119,6 +120,15 @@ pub enum Edit { bold: Option, italic: Option, }, + /// Write `bookmarks` as children of `parent` from child `index` on, or with `replace` in + /// place of the whole outline (`parent` must then be `[]`). A bookmark whose page this + /// document lacks is kept without a destination (see [`pdfcraft_organize::import_outline`]). + ImportBookmarks { + parent: Vec, + index: usize, + bookmarks: Vec, + replace: bool, + }, /// Bookmarks for the tagged headings (H, H1–H6), nested by level under a new first /// "Untitled" bookmark (New Bookmarks from Structure). BookmarksFromStructure, @@ -157,6 +167,17 @@ pub enum Edit { file: MarkFile, author: String, }, + /// Rich Media ▸ Add 3D: a U3D or PRC `model` as a 3D annotation in `rect` on `page`, named + /// `name` (its alternate description). Its poster is the picture in `poster` (a PDF page or an + /// image) fitted inside `rect`, keeping its shape and centred, or a labelled frame without one. + AddModel3d { + page: usize, + rect: [f64; 4], + model: Arc>, + poster: Option, + activation: pdfcraft_annot::Activation3d, + name: String, + }, /// Delete the comment at `index` in the page's `/Annots`, with its pop-up and replies. DeleteAnnotation { page: usize, @@ -243,12 +264,35 @@ pub enum Edit { index: usize, fill: Option, }, + /// Draw a rectangle's, oval's, polygon's or text box's fill, or a text box's text, at an + /// opacity of its own (0–1), the line or border keeping the comment's; `None` leaves that + /// part as it is. `StyleAnnotation`'s opacity sets the whole comment's. + PartOpacity { + page: usize, + index: usize, + fill: Option, + text: Option, + }, /// Change the font size (pt) of a text box, typewriter text or callout. FontSizeAnnotation { page: usize, index: usize, size: f64, }, + /// Give a text box, typewriter text or callout a border colour of its own (`None`: the + /// text's again). + BorderColorAnnotation { + page: usize, + index: usize, + color: Option, + }, + /// Give a text box or typewriter text an oval, a circle or a rectangle for its border: the + /// text box keeps its place, the oval or circle goes around it. + BorderShapeAnnotation { + page: usize, + index: usize, + shape: BorderShape, + }, /// Comment properties ▸ General / note icon. SetAnnotationInfo { page: usize, @@ -386,6 +430,13 @@ pub enum Edit { page: usize, words: Vec, }, + /// Scan & OCR ▸ Enhance ▸ Scanned document: put the scan images + /// [`ScanCleanJob::run`](crate::scan_clean::ScanCleanJob::run) cleaned in place of the old + /// ones (each object must still be an image). Built by + /// [`Session::apply_scan_clean`](crate::Session::apply_scan_clean). + EnhanceScan { + images: Vec<(pdfcraft_cos::ObjRef, pdfcraft_cos::Stream)>, + }, /// Edit a PDF ▸ Edit text: replace the text of line `line` (from `Document::text_lines`) on /// `page`, in its own font when it can show it, else in Helvetica. EditTextLine { @@ -534,6 +585,15 @@ pub enum Edit { name: String, bytes: Arc>, }, + /// Paste copied comments (XFDF from [`Session::export_comments`]) onto `page` as new + /// comments: copies under names of their own, so nothing is replaced and pasting twice + /// gives two sets; replies keep their copied parent. + PasteComments { + xfdf: String, + page: usize, + /// Where the copies go: moved by (dx, dy) in user space, or at their own positions. + offset: Option<[f64; 2]>, + }, /// Remove Hidden Information: the chosen categories. RemoveHidden { which: Vec, diff --git a/crates/engine/src/edit/labels.rs b/crates/engine/src/edit/labels.rs index 5343804ef3..615ac0fea5 100644 --- a/crates/engine/src/edit/labels.rs +++ b/crates/engine/src/edit/labels.rs @@ -38,6 +38,7 @@ pub const EDIT_LABELS: &[&str] = &[ "Set bookmark destination", "Colour", "Style", + "Import bookmarks", "New bookmarks from structure", "New bookmarks from text styles", "Number pages", @@ -68,7 +69,9 @@ pub const EDIT_LABELS: &[&str] = &[ "Add inserted text", "Add file attachment", "Add stamp", + "Add 3D model", "Delete comment", + "Paste comments", "Edit comment", "Reply", "Set status None", @@ -99,6 +102,7 @@ pub const EDIT_LABELS: &[&str] = &[ "Set alternate text", "Mark figure as decorative", "Recognize text", + "Enhance scanned file", "Run JavaScript", "Run form script", "Save as {level}", @@ -193,6 +197,7 @@ impl Edit { // The bookmark Style and Colour menus' own names, already said in every language. Edit::SetBookmarkColor { .. } => Label::of("Colour"), Edit::SetBookmarkStyle { .. } => Label::of("Style"), + Edit::ImportBookmarks { .. } => Label::of("Import bookmarks"), Edit::BookmarksFromStructure => Label::of("New bookmarks from structure"), Edit::BookmarksFromTextStyles { .. } => Label::of("New bookmarks from text styles"), Edit::NumberPages { .. } => Label::of("Number pages"), @@ -205,7 +210,9 @@ impl Edit { Edit::SetMeasurementScale { .. } => Label::of("Set measurement scale"), Edit::AddAnnotation(a) => Label::of(add_annotation(&a.shape)), Edit::AddCustomStamp { .. } => Label::of("Add stamp"), + Edit::AddModel3d { .. } => Label::of("Add 3D model"), Edit::DeleteAnnotation { .. } => Label::of("Delete comment"), + Edit::PasteComments { .. } => Label::of("Paste comments"), Edit::SetAnnotationContents { .. } => Label::of("Edit comment"), Edit::ReplyToAnnotation { .. } => Label::of("Reply"), // The state's name is part of the wording: a language may need to agree it with the rest. @@ -225,9 +232,13 @@ impl Edit { Edit::MoveAnnotation { .. } => Label::of("Move comment"), Edit::ReshapeAnnotation { .. } => Label::of("Reshape comment"), Edit::ResizeAnnotation { .. } => Label::of("Resize comment"), - Edit::StyleAnnotation { .. } | Edit::FillAnnotation { .. } | Edit::FontSizeAnnotation { .. } | Edit::SetAnnotationInfo { .. } => { - Label::of("Change comment properties") - } + Edit::StyleAnnotation { .. } + | Edit::FillAnnotation { .. } + | Edit::PartOpacity { .. } + | Edit::FontSizeAnnotation { .. } + | Edit::BorderColorAnnotation { .. } + | Edit::BorderShapeAnnotation { .. } + | Edit::SetAnnotationInfo { .. } => Label::of("Change comment properties"), Edit::SetFieldValue { name, .. } => Label::with("Fill in {name}", name.as_str()), Edit::SetFieldImage { name, .. } => Label::with("Set the image of {name}", name.as_str()), Edit::ResetForm { .. } => Label::of("Clear form"), @@ -248,6 +259,7 @@ impl Edit { Edit::MarkDecorative { .. } => Label::of("Mark figure as decorative"), Edit::MarkTagArtifact { .. } => Label::of("Mark content as artifact"), Edit::AddOcrText { .. } => Label::of("Recognize text"), + Edit::EnhanceScan { .. } => Label::of("Enhance scanned file"), Edit::ApplyScriptChanges { .. } => Label::of("Run JavaScript"), Edit::XfaEvent { .. } => Label::of("Run form script"), // The level is the standard's own name ("PDF/A-2b"), so it goes in as it stands. diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 5ed70aed7a..4719de302f 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -1,6 +1,6 @@ use super::{ AddedContent, Arc, Edit, EditError, FieldValue, ImageEdit, NewAnnotation, Shape, a11y, copy_image, decode_data_file, embed_image, fit_picture, - image_content_type, is_xfa_data_file, js, mark_source, measure, open_source, strip_xfa, unrestricted, xfa, xfa_after_data_change, + image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, open_source, strip_xfa, unrestricted, xfa, xfa_after_data_change, }; use crate::SourceProblem; use crate::sources::{comment_source, import_pdf_comments}; @@ -27,6 +27,7 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re | Edit::SetBookmarkPage { .. } | Edit::SetBookmarkColor { .. } | Edit::SetBookmarkStyle { .. } + | Edit::ImportBookmarks { .. } | Edit::BookmarksFromStructure | Edit::NumberPages { .. } => { if p.assemble() { @@ -48,7 +49,9 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re } Edit::AddMeasurement(_) | Edit::AddAnnotation(_) + | Edit::PasteComments { .. } | Edit::AddCustomStamp { .. } + | Edit::AddModel3d { .. } | Edit::DeleteAnnotation { .. } | Edit::SetAnnotationContents { .. } | Edit::ReplyToAnnotation { .. } @@ -62,7 +65,10 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re | Edit::ResizeAnnotation { .. } | Edit::StyleAnnotation { .. } | Edit::FillAnnotation { .. } + | Edit::PartOpacity { .. } | Edit::FontSizeAnnotation { .. } + | Edit::BorderColorAnnotation { .. } + | Edit::BorderShapeAnnotation { .. } | Edit::SetAnnotationInfo { .. } | Edit::SetMeasurementScale { .. } => { if p.annotate() { @@ -136,6 +142,7 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re | Edit::MarkDecorative { .. } | Edit::MarkTagArtifact { .. } | Edit::AddOcrText { .. } + | Edit::EnhanceScan { .. } | Edit::SetDocumentScript { .. } | Edit::SetFieldScript { .. } | Edit::RemoveStandardId { .. } @@ -454,6 +461,9 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::SetBookmarkPage { path, page } => pdfcraft_organize::set_bookmark_page(doc, path, *page)?, Edit::SetBookmarkColor { path, color } => pdfcraft_organize::set_bookmark_color(doc, path, *color)?, Edit::SetBookmarkStyle { path, bold, italic } => pdfcraft_organize::set_bookmark_style(doc, path, *bold, *italic)?, + Edit::ImportBookmarks { parent, index, bookmarks, replace } => { + pdfcraft_organize::import_outline(doc, parent, *index, bookmarks, *replace)?; + } Edit::BookmarksFromStructure => { let entries: Vec<_> = pdfcraft_a11y::headings(doc) .into_iter() @@ -489,6 +499,14 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::AddAnnotation(a) => { pdfcraft_annot::add_annotation(doc, a, &cx.meta())?; } + Edit::PasteComments { xfdf, page, offset } => { + // Names from the edit's own id generator: unique to this paste within the document. + let prefix = cx.meta().id; + let report = pdfcraft_xfdf::import_comments(doc, xfdf, Some(*page), Some(&prefix), *offset)?; + if report.comments == 0 { + return Err(EditError::Invalid("the clipboard holds no comments".into())); + } + } Edit::AddCustomStamp { page, rect, name, file, author } => { let src = mark_source(doc, file)?; let (sw, sh) = (src.size.0.max(1.0), src.size.1.max(1.0)); @@ -509,6 +527,16 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E let index = pdfcraft_annot::add_annotation(doc, &new, &cx.meta())?; pdfcraft_annot::orient_image_stamp(doc, *page, index, rotation)?; } + Edit::AddModel3d { page, rect, model, poster, activation, name } => { + // A file that is not a model is refused before its poster is brought in. + pdfcraft_annot::check_model(model)?; + let poster = match poster { + Some(file) => Some(model_poster(doc, *rect, file)?), + None => None, + }; + let new = pdfcraft_annot::NewModel3d { page: *page, rect: *rect, model, activation: *activation, poster, contents: name }; + pdfcraft_annot::add_3d_model(doc, &new, &cx.meta())?; + } Edit::DeleteAnnotation { page, index } => pdfcraft_annot::delete_annotation(doc, *page, *index)?, Edit::SetAnnotationContents { page, index, text } => pdfcraft_annot::set_contents(doc, *page, *index, text, &cx.meta())?, Edit::ReplyToAnnotation { page, index, text, author } => { @@ -559,7 +587,10 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E )?; } Edit::FillAnnotation { page, index, fill } => pdfcraft_annot::set_fill(doc, *page, *index, *fill, &cx.meta())?, + Edit::PartOpacity { page, index, fill, text } => pdfcraft_annot::set_part_opacity(doc, *page, *index, *fill, *text, &cx.meta())?, Edit::FontSizeAnnotation { page, index, size } => pdfcraft_annot::set_font_size(doc, *page, *index, *size, &cx.meta())?, + Edit::BorderColorAnnotation { page, index, color } => pdfcraft_annot::set_border_color(doc, *page, *index, *color, &cx.meta())?, + Edit::BorderShapeAnnotation { page, index, shape } => pdfcraft_annot::set_border_shape(doc, *page, *index, *shape, &cx.meta())?, Edit::SetAnnotationInfo { page, index, author, subject, icon } => { pdfcraft_annot::set_info(doc, *page, *index, author.as_deref(), subject.as_deref(), *icon, &cx.meta())?; } @@ -687,8 +718,9 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::RemoveStandardId { standard } => pdfcraft_organize::remove_standard_id(doc, *standard)?, Edit::SetDocumentScript { name, script } => pdfcraft_forms::set_document_script(doc, name, script.as_deref())?, Edit::AddOcrText { page, words } => { - pdfcraft_edit::stamp(doc, *page, "OCR", pdfcraft_ocr::text_layer(words))?; + crate::ocr::layer::add_text(doc, *page, words)?; } + Edit::EnhanceScan { images } => crate::scan_clean::write_back(doc, images)?, Edit::EditPageImage { page, index, change } => run_edit_page_image(doc, *page, *index, change)?, Edit::MoveObjects { page, objects, offset } => pdfcraft_edit::move_objects(doc, *page, objects, *offset)?, Edit::EditTextBlock { page, block, text, style } => { diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index 1330faf272..4512bcef43 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -1,8 +1,8 @@ //! Export a PDF ▸ Image and Text (execution plan M10.3, the first formats). //! //! [`Exporter`] renders pages to PNG, JPEG or TIFF at a resolution and extracts reading-order -//! text, from a -//! document's working file (so unsaved edits and hidden layers are respected, as on screen). +//! text, from the copy a document displays ([`Document::export_source`]): its working file, so +//! unsaved edits and hidden layers are respected, with what is drawn for display only. use std::collections::BTreeMap; use std::panic::{AssertUnwindSafe, catch_unwind}; @@ -67,9 +67,10 @@ pub struct ExportSource { impl Document { /// What an export reads, for exporting on another thread. These are the displayed - /// bytes, so they may carry missing comment appearances PdfCraft drew. Signature - /// artwork stays original; viewer validation badges are not document content and - /// do not enter image/text exports or Print as Image. Saving uses the working file. + /// bytes, so they may carry appearances PdfCraft drew that the file itself does not hold: + /// missing comment appearances, and the signature states of named-layer signature + /// appearances. Exports and Print as Image follow the page as shown; the viewer's badges + /// are not document content and never enter them. Saving uses the working file. pub fn export_source(&self) -> ExportSource { ExportSource { bytes: self.display.clone(), diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs index fafa65b4c2..ce51df5e61 100644 --- a/crates/engine/src/history.rs +++ b/crates/engine/src/history.rs @@ -53,7 +53,9 @@ pub(crate) fn scope_of(edit: &Edit) -> Scope { Edit::AddAnnotation(a) if matches!(a.shape, Shape::Attachment { .. }) => Scope::Full, Edit::AddMeasurement(_) | Edit::AddAnnotation(_) + | Edit::PasteComments { .. } | Edit::AddCustomStamp { .. } + | Edit::AddModel3d { .. } | Edit::DeleteAnnotation { .. } | Edit::SetAnnotationContents { .. } | Edit::ReplyToAnnotation { .. } @@ -67,7 +69,10 @@ pub(crate) fn scope_of(edit: &Edit) -> Scope { | Edit::ResizeAnnotation { .. } | Edit::StyleAnnotation { .. } | Edit::FillAnnotation { .. } + | Edit::PartOpacity { .. } | Edit::FontSizeAnnotation { .. } + | Edit::BorderColorAnnotation { .. } + | Edit::BorderShapeAnnotation { .. } | Edit::SetAnnotationInfo { .. } => Scope::Comments, Edit::AddLink { .. } | Edit::SetLink { .. } | Edit::DeleteLink { .. } | Edit::RemoveLinks { .. } | Edit::AddLinks { .. } => Scope::Links, Edit::SetFieldValue { .. } | Edit::ResetForm { .. } | Edit::SetFieldImage { .. } | Edit::ApplyScriptChanges { .. } => Scope::Form, @@ -77,6 +82,7 @@ pub(crate) fn scope_of(edit: &Edit) -> Scope { | Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } | Edit::EditPageImage { .. } + | Edit::EnhanceScan { .. } | Edit::AddText { .. } | Edit::AddImage { .. } | Edit::UpdateContent { .. } diff --git a/crates/engine/src/js.rs b/crates/engine/src/js.rs index d1ee5c15b0..f581504204 100644 --- a/crates/engine/src/js.rs +++ b/crates/engine/src/js.rs @@ -199,6 +199,24 @@ impl Scripts for JsRunner { /// Run field `target`'s script for `event` over the form's fields, keeping what it showed and /// asked for; a refusal carries its last message. fn run(&mut self, event: FieldEvent, script: &str, target: &Field, value: &str, fields: &[Field]) -> ScriptResult { + // As in Acrobat, a failing script leaves the value alone (the error goes to the console). + self.run_logged(event, script, target, value, fields, true).unwrap_or_else(|| ScriptResult { + rc: true, + value: value.to_string(), + changes: Vec::new(), + message: None, + }) + } + + fn rerun_with_english_dates(&mut self, event: FieldEvent, script: &str, target: &Field, value: &str, fields: &[Field]) -> Option { + pdfcraft_js::with_english_printd(true, || self.run_logged(event, script, target, value, fields, false)) + } +} + +impl JsRunner { + /// [`Scripts::run`], keeping what the script showed and asked for only when `log`; `None` + /// when the script failed. + fn run_logged(&mut self, event: FieldEvent, script: &str, target: &Field, value: &str, fields: &[Field], log: bool) -> Option { let states: Vec<_> = fields .iter() .map(|f| { @@ -212,17 +230,18 @@ impl Scripts for JsRunner { let states = std::sync::Arc::new(states); let ev = pdfcraft_js::Event::field(event.name(), &target.name, value); let o = pdfcraft_js::run_shared(script, &ev, &self.doc, std::sync::Arc::clone(&states), &self.doc_scripts, Limits::default()); - self.output.absorb(&o); - if o.error.is_some() { - // As in Acrobat, a failing script leaves the value alone (the error goes to the console). - return ScriptResult { rc: true, value: value.to_string(), changes: Vec::new(), message: None }; + if log { + self.output.absorb(&o); } - ScriptResult { + if o.error.is_some() { + return None; + } + Some(ScriptResult { rc: o.rc, value: o.value.clone(), changes: changes(&states, &o.changed), message: (!o.rc).then(|| o.alerts.last().map(|m| m.text.clone())).flatten(), - } + }) } } @@ -410,12 +429,16 @@ impl Session { let mut out = Vec::new(); for page in list { let Some(info) = doc.info.pages.get(page) else { continue }; - let res = r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); let shapes = pdfcraft_forms::detect::page_shapes(cos, page); let existing: Vec<[f64; 4]> = doc.form.iter().flat_map(|f| f.widgets.iter()).filter(|w| w.page == Some(page)).map(|w| w.rect).collect(); let (found, turn) = if shapes.boxes.is_empty() && shapes.rules.is_empty() && crate::scan_fields::is_scan(doc, page, info) { - self.detect_on_scan(id, &mut r, page, res.text.as_deref(), &existing, &taken) + // Labels come from the page's own text. Words that only /ActualText names (a stamp + // drawn as outlines) are no labels, and leave the scan to be read. + let own = crate::ocr::own_text(&mut r, page).ok(); + self.detect_on_scan(id, &mut r, page, own.as_ref(), &existing, &taken) } else { + let res = + r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); let words = res.text.map(|t| detection_words(&t, info)).unwrap_or_default(); (pdfcraft_forms::detect::detect(&words, &shapes, &existing, &taken), 0) }; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index fe139164be..4fb8d8a472 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -35,6 +35,7 @@ mod history; pub mod js; mod layers; pub mod links; +pub mod model3d; pub mod ocr; mod open; pub mod optimizer; @@ -43,6 +44,8 @@ mod pictures; pub mod print_raster; pub mod printcheck; mod protection; +mod rich_copy; +pub mod scan_clean; mod scan_colours; mod scan_fields; mod session; @@ -66,12 +69,16 @@ pub use links::{ MAX_REMOTE_DOCUMENT_BYTES, RemoteDocumentPathError, read_remote_document, remote_document_path_is_absolute, resolve_remote_document_path, }; pub use pdfcraft_edit::strip_subset_tag; +pub use rich_copy::RichCopy; pub use signature_image::{ImageSignaturePreview, SignatureImage}; pub use signature_scan::{PhotoRect, ScanError, SignaturePhoto, Turn}; pub use stamp_preview::{CommentLayer, StampPreview}; pub use text_preview::{TextEditPreview, UnmarkedPages}; -pub use pdfcraft_organize::{BoxSpec, FieldConflicts, PageBox, SplitBy, Transition, TransitionSpec, mime_for_name, split_ranges}; +pub use pdfcraft_organize::{ + BoxSpec, FieldConflicts, MAX_IMPORT_BOOKMARKS, OutlineNode, OutlineTree, PageBox, SplitBy, Transition, TransitionSpec, mime_for_name, + split_ranges, unmapped as outline_unmapped, view_arity as outline_view_arity, +}; pub use pdfcraft_organize::LabelStyle; pub use pdfcraft_organize::Standard; @@ -86,7 +93,8 @@ pub use pdfcraft_edit::added::{ }; pub use pdfcraft_edit::paragraph_align; pub use pdfcraft_edit::{ - Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, MarkKind, Watermark, + Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, Ink as TextInk, + MarkKind, Watermark, }; pub use pdfcraft_forms::{ BorderStyle, CalculatedField, CalculationIssue, CalculationOrder, CheckStyle, Field as FormField, FieldAction, FieldChange, FieldFont, @@ -108,9 +116,10 @@ pub use pdfcraft_fonts::{MAX_SIGNATURE_CHARS, ScriptOutline, script_missing, scr pub use pdfcraft_annot::appearance as annot_text; pub use pdfcraft_annot::links::{Highlight as LinkHighlight, LinkAction, LinkItem, LinkStyle}; pub use pdfcraft_annot::{ - AppearanceCache, AttachIcon, FillMark, LineEnding, Markup, NOTE_SIZE, NewAnnotation, NoteIcon, OverlayFont, OverlayLook, Props as CommentProps, - ReviewState, Rgb, Shape, StampGroup, StampKind, Style, rect_quad, + Activation3d, AppearanceCache, AttachIcon, BorderShape, FillMark, LineEnding, Markup, NOTE_SIZE, NewAnnotation, NoteIcon, OverlayFont, + OverlayLook, Props as CommentProps, ReviewState, Rgb, Shape, StampGroup, StampKind, Style, rect_quad, }; +pub use pdfcraft_annot::{MAX_MODEL_BYTES as MAX_3D_MODEL_BYTES, ModelFormat as Model3dFormat}; pub use pdfcraft_forms::detect; pub use pdfcraft_optimize as optimize; pub use pdfcraft_preflight as pdfa; @@ -121,8 +130,9 @@ pub use pdfcraft_redact::patterns::{ }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; pub use pdfcraft_sign as sign; -pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; +pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, StatusCaptions, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; +pub use pdfcraft_xfdf::{Outline as CommentOutline, comment_count, comment_outlines, comments_bounds, is_xfdf_comments}; use bookmarks::outline_rows; pub use bookmarks::{BookmarkPage, BookmarkRow}; @@ -131,7 +141,7 @@ pub use comments::{SummarySort, comment_kind, comment_summary, comment_type_name #[cfg(test)] use document::page_runs; pub use document::{Document, EditNotice, TextNotice, display_bytes_for, render_password_for}; -use document::{display_bytes, info_fields, recovered_render_password, same_pages, shown_under_redactions, undrawn_notices}; +use document::{carry_verdict_marks, display_bytes, info_fields, recovered_render_password, same_pages, shown_under_redactions, undrawn_notices}; pub use edit::{BATCH_LABELS, EDIT_LABELS, Edit}; pub use error::{EditError, SourceProblem}; pub use fill_sign::{displayed_offset, text_box_at, typed_signature_graphic, typed_signature_shape}; @@ -140,7 +150,7 @@ use layers::{apply_layer_state, set_default_layer_state, use_layer_choices}; pub use open::{OpenContext, PreparedDocument}; pub use page_filter::{PageOrientation, PageParity, filter_pages}; pub use pictures::{MarkFile, image_dimensions}; -use pictures::{embed_image, fit_picture, mark_source}; +use pictures::{embed_image, fit_picture, mark_source, model_poster}; use protection::unrestricted; pub use protection::{Changes, Printing, Protection, SecuritySummary}; pub use session::Session; diff --git a/crates/engine/src/model3d.rs b/crates/engine/src/model3d.rs new file mode 100644 index 0000000000..b95a332a22 --- /dev/null +++ b/crates/engine/src/model3d.rs @@ -0,0 +1,19 @@ +//! The 3D models on a page (ISO 32000-2 §13.6), read so a renderer can draw them. + +pub use pdfcraft_annot::{MAX_MODEL_BYTES, Model3d, Model3dKind}; + +use crate::Document; + +impl Document { + /// The `/Annots` positions of the 3D annotations on 0-based `page`, in order. + pub fn model_3d_annotations(&self, page: usize) -> Result, String> { + let editor = self.editor.as_ref().ok_or("the document can't be read")?; + pdfcraft_annot::model_3d_annotations(&editor.cos, page).map_err(|e| e.to_string()) + } + + /// The decoded model of the 3D annotation at `/Annots` position `index` on 0-based `page`. + pub fn model_3d(&self, page: usize, index: usize) -> Result { + let editor = self.editor.as_ref().ok_or("the document can't be read")?; + pdfcraft_annot::model_3d(&editor.cos, page, index).map_err(|e| e.to_string()) + } +} diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index 6b9869cead..c4854d7e9f 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -3,16 +3,105 @@ //! //! Recognition is slow (about a second a page), so it is split from the edit: [`Session::ocr_job`] //! captures what it needs, [`OcrJob::run`] works anywhere (the UI runs it on a worker thread) and -//! [`Session::apply_ocr`] applies the result as one undoable step. +//! [`Session::apply_ocr`] applies the result as one undoable step. The result is only applied +//! while the document is still the one that was read — [`Session::apply_ocr_checked`] refuses a +//! stale result with a named reason instead of writing into a changed document. use std::sync::{Arc, Mutex}; +#[cfg(not(target_arch = "wasm32"))] +use std::time::{Duration, Instant}; +use pdfcraft_render::text::TextFilter; use pdfcraft_render::{PageInfo, PageRenderer, RenderConfig, RenderRequest, RequestKind}; -pub use pdfcraft_ocr::{LANGUAGES, Models, Ocr, OcrError, PlacedWord}; +pub use pdfcraft_ocr::{ + ConfidenceBand, LANGUAGES, MergeStrategy, Models, Ocr, OcrError, PlacedWord, Recognizer, band_for, is_suspect, low_confidence, +}; use crate::{DocId, Edit, EditError, Session}; +pub mod dictionary; +pub(crate) mod layer; + +/// Bound recognition/review inputs before cloning or planning glyphs. +pub fn validate_ocr_words<'a>(words: impl IntoIterator) -> Result { + layer::validate_words(words).map_err(|e| e.to_string()) +} + +/// Which recognition engine to use. The default is ocrs, the pure-Rust engine; "Automatic" is +/// an ensemble — ocrs reading first, Tesseract as a second opinion, merged per the strategy — +/// when both are available. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum EngineChoice { + /// The pure-Rust ocrs engine (the default). + #[default] + Ocrs, + /// The installed `tesseract` binary, as an external process. + Tesseract, + /// Ensemble: the default engine first, the other one as a secondary when available. + Auto, +} + +impl EngineChoice { + /// Parse a choice; unknown names fall back to the default (ocrs), like Acrobat's dialogs + /// fall back for unknown persisted values. + pub fn parse(s: &str) -> EngineChoice { + match s.trim().to_ascii_lowercase().as_str() { + "tesseract" => EngineChoice::Tesseract, + "auto" | "automatic" | "ensemble" => EngineChoice::Auto, + _ => EngineChoice::Ocrs, + } + } + + /// The strategy this choice reads with when none was chosen. Automatic exists to bring in + /// the second engine, so it asks it when the first reading is weak or of unknown + /// confidence ([`MergeStrategy::ConfidenceWeighted`]); a single named engine reads alone. + pub fn default_strategy(self) -> MergeStrategy { + match self { + EngineChoice::Auto => MergeStrategy::ConfidenceWeighted, + EngineChoice::Ocrs | EngineChoice::Tesseract => MergeStrategy::PrimaryOnly, + } + } + + /// The name settings and tools use for this choice. + pub fn name(self) -> &'static str { + match self { + EngineChoice::Ocrs => "ocrs", + EngineChoice::Tesseract => "tesseract", + EngineChoice::Auto => "auto", + } + } +} + +/// What to produce from a recognition. **Searchable** (the default) adds the existing +/// invisible-text layer on the original page; **EditableText** writes a *new* document with the +/// reviewed words as visible text and no scan image (the writer is the Verify screen's; the mode +/// is plumbed through here). Unknown mode strings mean Searchable. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum OutputMode { + #[default] + Searchable, + EditableText, +} + +impl OutputMode { + /// Parse a mode name (trimmed, case-insensitive); unknown strings mean Searchable. + pub fn parse(s: &str) -> OutputMode { + match s.trim().to_ascii_lowercase().as_str() { + "editable" | "editable-text" | "editable_text" | "editabletext" => OutputMode::EditableText, + _ => OutputMode::Searchable, + } + } + + /// The name settings and tools use for this mode. + pub fn name(self) -> &'static str { + match self { + OutputMode::Searchable => "searchable", + OutputMode::EditableText => "editable", + } + } +} + /// Recognize Text settings. #[derive(Clone, Debug, PartialEq)] pub struct OcrSettings { @@ -29,6 +118,40 @@ pub struct OcrSettings { /// that is read, so a scan with a few comments on it is still a scan. `false`: everything /// the page shows. pub native_text_only: bool, + /// Leave whole files that already have text alone (the batch flow's "Skip files that + /// already contain text"; a skipped file is its own result bucket, never an OCR success). + pub skip_text_files: bool, + /// Read pages even when the skip options would leave them alone ("Force OCR (override + /// skip options)"). + pub force_ocr: bool, + /// Which engine reads the pages. + pub engine: EngineChoice, + /// How the engines' readings combine (see [`MergeStrategy`]). + pub strategy: MergeStrategy, + /// Preprocessing, all off by default (a scan is passed through as rendered): + /// straighten pages scanned at 90°/180°/270°. + pub auto_rotate: bool, + /// Preprocessing: straighten skewed text lines. + pub deskew: bool, + /// Preprocessing: 3×3 median filter. + pub denoise: bool, + /// Preprocessing: Sauvola binarization. + pub binarize: bool, + /// What to produce (searchable text layer, or a new editable document). + pub output_mode: OutputMode, + /// Restrict recognition to this rectangle of the rendered page image, in pixels + /// `[x0, y0, x1, y1]` (the Verify screen's "Re-OCR this region"). + pub region: Option<[f32; 4]>, + /// The batch note's separate per-word threshold: words with `0 <= conf <= threshold` are + /// counted for "N low-confidence word(s) on page(s) …" (see [`low_confidence`]; unrelated + /// to the review overlay's bands, which use [`band_for`]). + pub low_confidence_threshold: f32, + /// Where the Tesseract program lives, when the user configured it (a preference — the + /// `PDFCRAFT_TESSERACT` environment variable is the fallback in the engine; never taken + /// from document data). A configured path must be absolute to an existing, executable + /// file; an invalid configuration makes the engine unavailable rather than silently + /// searching the PATH. `None` means the PATH search decides. + pub tesseract_path: Option, } /// A recognised word whose box is at least this much covered by the page's own text is that @@ -39,6 +162,18 @@ const ALREADY_ON_PAGE: f32 = 0.5; /// is a resolution on paper this size or smaller. const SHEET: f32 = 842.0; +/// The text of `page`'s own: its glyphs, without the words drawn as outlines or pictures that only +/// `/ActualText` names. +pub(crate) fn own_text(r: &mut PageRenderer, page: usize) -> Result { + r.filtered_text(page, TextFilter { painted: true, ..Default::default() }) +} + +/// Whether `page` has text of its own ([`own_text`]). A stamp drawn as outlines on a scan still +/// leaves the scan to read; its words are not read twice, since they count as already on the page. +fn has_own_text(r: &mut PageRenderer, page: usize) -> bool { + own_text(r, page).is_ok_and(|t| t.glyphs.iter().any(|g| !g.text.trim().is_empty())) +} + /// How much of `word` the `text` boxes cover, from 0 to 1. Boxes are `[x0, y0, x1, y1]` in page /// view space. Overlaps between the text boxes are counted twice (glyphs rarely overlap), so the /// sum is capped at the whole word. @@ -56,7 +191,24 @@ fn covered(word: [f32; 4], text: &[[f32; 4]]) -> f32 { impl Default for OcrSettings { fn default() -> Self { - OcrSettings { dpi: 300.0, language: "en".into(), skip_text_pages: true, native_text_only: false } + OcrSettings { + dpi: 300.0, + language: "en".into(), + skip_text_pages: true, + native_text_only: false, + skip_text_files: false, + force_ocr: false, + engine: EngineChoice::default(), + strategy: MergeStrategy::default(), + auto_rotate: false, + deskew: false, + denoise: false, + binarize: false, + output_mode: OutputMode::default(), + region: None, + low_confidence_threshold: 60.0, + tesseract_path: None, + } } } @@ -69,31 +221,406 @@ pub struct OcrPage { pub already_on_page: usize, /// Why the page was not read, if it was skipped. pub skipped: Option, + /// Notes about a partial reading: the words stand, but something on the way did not (an + /// ensemble partner failed and the page keeps the primary engine's reading alone). + pub notes: Vec, } impl OcrPage { pub fn text(&self) -> String { self.words.iter().map(|w| w.text.as_str()).collect::>().join(" ") } -} -/// The recogniser, loaded once (it takes a moment) and shared. -pub fn engine() -> Result, String> { - static OCR: Mutex>> = Mutex::new(None); - let mut slot = OCR.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - if let Some(o) = slot.as_ref() { - return Ok(o.clone()); + /// Mean of the confidences the page's words report (`None` when none do). + pub fn mean_confidence(&self) -> Option { + let (sum, n) = self.words.iter().filter_map(|w| w.confidence).fold((0.0f64, 0usize), |(s, n), c| (s + c as f64, n + 1)); + (n > 0).then_some((sum / n as f64) as f32) + } + + /// The words to flag for review: their confidence is in the Low band ([`band_for`]). + /// Excluded are words the caller wants gone — removed words, words in the user dictionary, + /// skipped words — which is review state, so this reports the indices and the reviewer + /// filters them. + pub fn suspects(&self) -> Vec { + self.words.iter().enumerate().filter(|(_, w)| pdfcraft_ocr::is_suspect(w.confidence)).map(|(i, _)| i).collect() } - let o = Arc::new(Ocr::find().map_err(|e| e.to_string())?); - *slot = Some(o.clone()); - Ok(o) } -/// Whether the recognition models are installed. -pub fn available() -> bool { +/// The engines a job reads with: one primary and, for ensemble strategies, an optional +/// secondary. +pub struct Recognizers { + pub primary: Arc, + pub secondary: Option>, +} + +struct EngineCache { + ocrs: Option>, +} + +/// How long a negative engine probe (no program there) is remembered. A short TTL, not +/// forever: a tesseract installed while the app runs is found by the next job, without a +/// restart. An engine that was found is remembered for the session. +#[cfg(not(target_arch = "wasm32"))] +const NEGATIVE_TTL: Duration = Duration::from_secs(30); + +/// The remembered Tesseract probe: the configured location it answers for, the engine it +/// found (`None` for a probe that failed), and when that probe happened. +#[cfg(not(target_arch = "wasm32"))] +struct CachedTesseract { + key: Option, + found: Option>, + at: Instant, +} + +/// Whether a probe result still counts: a found engine for the session, a failed probe until +/// [`NEGATIVE_TTL`] passes (split out so tests can pass the clock). +#[cfg(not(target_arch = "wasm32"))] +fn probe_fresh(found: bool, at: Instant, now: Instant) -> bool { + found || now.duration_since(at) < NEGATIVE_TTL +} + +/// The remembered Tesseract probe (one slot: the last location asked about). +#[cfg(not(target_arch = "wasm32"))] +static TESSERACT_PROBE: Mutex> = Mutex::new(None); + +/// Whether a background probe, started for the UI by [`tesseract_engine_now`], is running now. +/// At most one runs at a time. +#[cfg(not(target_arch = "wasm32"))] +static PROBING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// The Tesseract engine: found once (it answers `--list-langs` through a process) and then +/// remembered, per configured path; a failed probe is retried after [`NEGATIVE_TTL`]. No +/// static mutex is held while the discovery process runs: the cache is read under the lock, +/// released for the probe, and the result is written back when it arrives (concurrent probes +/// of the same location may both run; the last write wins and both answers are valid). A +/// user-configured path (`settings.tesseract_path`, else the `PDFCRAFT_TESSERACT` +/// environment variable) is validated before any process starts; an invalid configuration is +/// unavailable rather than a PATH fallback. This waits for the probe: the UI thread uses +/// [`tesseract_engine_now`]. +#[cfg(not(target_arch = "wasm32"))] +fn tesseract_engine(configured: Option<&str>) -> Option> { + if let Cached::Fresh(found) = cached_tesseract(configured) { + return found; + } + let key = configured.map(str::to_string); + // The probe (a `--list-langs` process, up to the timeout) runs outside every static mutex. + let found = match configured { + Some(path) => pdfcraft_ocr::tesseract::TesseractCli::at(std::path::PathBuf::from(path)).map(|t| Arc::new(t) as Arc), + None => pdfcraft_ocr::tesseract::TesseractCli::find().map(|t| Arc::new(t) as Arc), + }; + *TESSERACT_PROBE.lock().unwrap_or_else(|e| e.into_inner()) = Some(CachedTesseract { key, found: found.clone(), at: Instant::now() }); + found +} + +/// What the UI may know about the Tesseract probe without waiting for it. +#[cfg(not(target_arch = "wasm32"))] +enum Probe { + /// An answer: the engine, or `None` when there is none. It may be an old one that a + /// background probe is refreshing. + Known(Option>), + /// No answer yet: the first background probe is running (or could not be started). + Pending, +} + +/// What the cache holds for a location. +#[cfg(not(target_arch = "wasm32"))] +enum Cached { + /// A usable answer (see [`probe_fresh`]). + Fresh(Option>), + /// A failed probe older than [`NEGATIVE_TTL`]: still the best known answer, but due again. + Stale(Option>), + /// Nothing for this location. + Missing, +} + +/// [`tesseract_engine`] for the UI thread, which must never wait on a process: the cached +/// answer is returned at once, and when it is missing or due for a retry one background thread +/// runs the probe (it fills the cache). The answer is [`Probe::Pending`] only until the first +/// probe of a location lands, so a retry never makes the choice flicker. +#[cfg(not(target_arch = "wasm32"))] +fn tesseract_engine_now(configured: Option<&str>) -> Probe { + let known = match cached_tesseract(configured) { + Cached::Fresh(found) => return Probe::Known(found), + Cached::Stale(found) => Some(found), + Cached::Missing => None, + }; + if !PROBING.swap(true, std::sync::atomic::Ordering::AcqRel) { + let configured = configured.map(str::to_string); + let started = std::thread::Builder::new().name("pdfcraft-tesseract-probe".into()).spawn(move || { + let _ = tesseract_engine(configured.as_deref()); + PROBING.store(false, std::sync::atomic::Ordering::Release); + }); + if started.is_err() { + PROBING.store(false, std::sync::atomic::Ordering::Release); + } + } + known.map_or(Probe::Pending, Probe::Known) +} + +/// Whether the Tesseract probe is running in the background right now. The interface asks it +/// while it shows an engine choice that read as "still checking", to look again soon. +pub fn engine_probe_running() -> bool { + #[cfg(not(target_arch = "wasm32"))] + { + PROBING.load(std::sync::atomic::Ordering::Acquire) + } + #[cfg(target_arch = "wasm32")] + { + false + } +} + +/// What the cache holds for `configured`. Never starts a process. +#[cfg(not(target_arch = "wasm32"))] +fn cached_tesseract(configured: Option<&str>) -> Cached { + let slot = TESSERACT_PROBE.lock().unwrap_or_else(|e| e.into_inner()); + cache_lookup(slot.as_ref(), configured, Instant::now()) +} + +/// [`cached_tesseract`] over an explicit slot and clock (so tests can pass both). +#[cfg(not(target_arch = "wasm32"))] +fn cache_lookup(slot: Option<&CachedTesseract>, configured: Option<&str>, now: Instant) -> Cached { + match slot { + Some(c) if c.key.as_deref() == configured => { + if probe_fresh(c.found.is_some(), c.at, now) { + Cached::Fresh(c.found.clone()) + } else { + Cached::Stale(c.found.clone()) + } + } + _ => Cached::Missing, + } +} + +/// Whether the ocrs models are installed, remembered like the Tesseract probe (found: for the +/// session; missing: for [`NEGATIVE_TTL`]), so per-frame UI checks do not search the disk. +#[cfg(not(target_arch = "wasm32"))] +fn models_installed_cached() -> bool { + static MODELS: Mutex> = Mutex::new(None); + let mut slot = MODELS.lock().unwrap_or_else(|e| e.into_inner()); + if let Some((found, at)) = *slot + && probe_fresh(found, at, Instant::now()) + { + return found; + } + let found = Models::find().is_some(); + *slot = Some((found, Instant::now())); + found +} + +/// The web build has no disk search worth caching. +#[cfg(target_arch = "wasm32")] +fn models_installed_cached() -> bool { Models::find().is_some() } +/// The program's `--version` line, remembered per program path for the session (`ocr_status` +/// asks every time it is called; the version of an installed binary does not change under a +/// running app often enough to start a process each time). +#[cfg(not(target_arch = "wasm32"))] +fn tesseract_version(program: &str) -> Option { + static VERSIONS: Mutex)>> = Mutex::new(Vec::new()); + if let Some((_, v)) = VERSIONS.lock().unwrap_or_else(|e| e.into_inner()).iter().find(|(p, _)| p == program) { + return v.clone(); + } + // Validated (absolute, existing, executable) before the process starts; run outside the lock. + let path = std::path::PathBuf::from(program); + let version = pdfcraft_ocr::tesseract::program_is_usable(&path).then(|| pdfcraft_ocr::tesseract::TesseractCli::new(path).version()).flatten(); + let mut cache = VERSIONS.lock().unwrap_or_else(|e| e.into_inner()); + if cache.len() < 16 && !cache.iter().any(|(p, _)| p == program) { + cache.push((program.to_string(), version.clone())); + } + version +} + +/// The Tesseract program the engine would run, named for `ocr_status`: the user-configured +/// location when one is set, else what the PATH search resolves to. This function itself +/// starts no process (the PATH search only looks at files), so a path here says where the +/// engine looks, not that a program answers there; [`engines_status`], which reports it, does +/// run the cached `--list-langs` probe and a cached `--version`. `None` when nothing is +/// configured and the PATH holds no `tesseract` (or, on the web, when there is no process to +/// run). +#[cfg(not(target_arch = "wasm32"))] +pub fn tesseract_program_path(configured: Option<&str>) -> Option { + match configured { + Some(p) => Some(p.to_string()), + None => pdfcraft_ocr::tesseract::path_search().map(|p| p.to_string_lossy().into_owned()), + } +} + +/// One engine's status, for `ocr_status`: whether it can run, why not when it cannot, and for +/// Tesseract the program it would run and that program's version (both `None` on the web). +#[derive(Clone, Debug, PartialEq)] +pub struct EngineStatus { + pub id: &'static str, + pub available: bool, + pub why_not: Option, + pub program: Option, + pub version: Option, +} + +/// The engines' status, in the order the Engine dropdown lists them: ocrs (built in), then +/// tesseract (the external program). Tesseract's availability is judged the same way +/// [`recognizers`] judges it under the default settings: the `PDFCRAFT_TESSERACT` environment +/// variable when set, else the PATH. This waits for the (cached) Tesseract probe; the tools +/// and tests call it, never the UI thread. +pub fn engines_status() -> Vec { + engines_status_for(&OcrSettings::default()) +} + +/// [`engines_status`] for the location a job's `settings` name: `settings.tesseract_path` when +/// set, else the environment variable, else the PATH, as [`recognizers`] decides it. (No tool +/// takes a location, because an agent must not choose which program runs; the tools report the +/// default settings.) +pub fn engines_status_for(settings: &OcrSettings) -> Vec { + let ocrs = Models::find().is_some(); + let mut out = vec![EngineStatus { id: "ocrs", available: ocrs, why_not: (!ocrs).then(models_missing), program: None, version: None }]; + #[cfg(not(target_arch = "wasm32"))] + { + let configured = configured_tesseract(settings); + let configured = configured.as_deref(); + let available = tesseract_engine(configured).is_some(); + let program = tesseract_program_path(configured); + // The version is a `--version` run against the validated program path, remembered per + // path; a program that answers `--list-langs` but not `--version` still counts as + // available. + let version = if available { program.as_deref().and_then(tesseract_version) } else { None }; + let why_not = (!available).then(|| match configured { + Some(p) => format!("the configured tesseract location is not an absolute path to an existing, executable file: {p}"), + None => tesseract_unavailable(), + }); + out.push(EngineStatus { id: "tesseract", available, why_not, program, version }); + } + #[cfg(target_arch = "wasm32")] + { + let _ = settings; + out.push(EngineStatus { id: "tesseract", available: false, why_not: Some(tesseract_unavailable()), program: None, version: None }); + } + out +} + +/// The web build never spawns processes, so it never has Tesseract. +#[cfg(target_arch = "wasm32")] +fn tesseract_engine(configured: Option<&str>) -> Option> { + let _ = configured; + None +} + +/// The configured Tesseract location: the user preference wins; the environment variable is +/// the fallback (a preference file rarely survives onto another machine). Both are user +/// choices — this is never taken from document data. +fn configured_tesseract(settings: &OcrSettings) -> Option { + settings.tesseract_path.clone().or_else(|| std::env::var_os("PDFCRAFT_TESSERACT").map(|v| v.to_string_lossy().into_owned())) +} + +/// The engine selector: builds each engine once and hands out the pair `settings.engine` +/// asks for. "Automatic" is an ensemble when both engines are available; a named engine that +/// cannot start fails with its cause named. +pub fn recognizers(settings: &OcrSettings) -> Result { + // The Tesseract probe (a `--list-langs` process) runs before any static mutex is taken, so + // recognition on other threads is never blocked behind someone's discovery. A job that + // names its engines never starts it: only Tesseract and Automatic read anything from it, + // so an ocrs-only job pays no process and no wait. + let configured = configured_tesseract(settings); + let tesseract = + if matches!(settings.engine, EngineChoice::Tesseract | EngineChoice::Auto) { tesseract_engine(configured.as_deref()) } else { None }; + static ENGINES: Mutex> = Mutex::new(None); + let mut slot = ENGINES.lock().unwrap_or_else(|e| e.into_inner()); + let cache = slot.get_or_insert_with(|| EngineCache { ocrs: None }); + if cache.ocrs.is_none() { + // Cheap to re-attempt while the models are missing; expensive work is cached forever. + cache.ocrs = pdfcraft_ocr::OcrsRecognizer::find().ok().map(|r| Arc::new(r) as Arc); + } + let ocrs = cache.ocrs.clone(); + match settings.engine { + EngineChoice::Ocrs => Ok(Recognizers { primary: ocrs.ok_or_else(models_missing)?, secondary: None }), + EngineChoice::Tesseract => Ok(Recognizers { primary: tesseract.ok_or_else(tesseract_unavailable)?, secondary: None }), + EngineChoice::Auto => match (ocrs, tesseract) { + (Some(p), s) => Ok(Recognizers { primary: p, secondary: s }), + (None, Some(p)) => Ok(Recognizers { primary: p, secondary: None }), + (None, None) => Err(models_missing()), + }, + } +} + +/// Why Tesseract cannot run, named for the user. +#[cfg(not(target_arch = "wasm32"))] +fn tesseract_unavailable() -> String { + "the tesseract program was not found (install it, or add it to the PATH)".into() +} + +/// Why Tesseract cannot run, named for the user (the web build has no processes). +#[cfg(target_arch = "wasm32")] +fn tesseract_unavailable() -> String { + "the tesseract engine is not available in the web build".into() +} + +/// Why the ocrs engine cannot run, in the end-user wording the UI catalogs translate (the +/// same text as [`pdfcraft_ocr::OcrError::NoModels`]: no developer commands in front of users). +fn models_missing() -> String { + pdfcraft_ocr::OcrError::NoModels.to_string() +} + +/// Whether recognition can run at all: the ocrs models are installed, or (off the web) a +/// Tesseract program is found. This may run the Tesseract probe (a process, cached) and so +/// waits for it: tools and tests call it; the UI asks [`engine_unavailable_reason`], which +/// never waits. +pub fn available() -> bool { + let configured = std::env::var_os("PDFCRAFT_TESSERACT").map(|v| v.to_string_lossy().into_owned()); + Models::find().is_some() || tesseract_engine(configured.as_deref()).is_some() +} + +/// Why `engine` cannot run right now, named for the UI's tooltips (the Engine dropdown +/// disables the choice with this text); `None` when it can. The app UI has no Tesseract +/// location field of its own, so this judges the environment variable, exactly like +/// [`available`] (no tool takes a per-job location either; `OcrSettings::tesseract_path` is +/// for library callers, and [`recognizers`] validates it on its own). +/// +/// Called on the UI thread every frame, so it never waits on a process: the answers are +/// cached ([`NEGATIVE_TTL`] for a missing engine, refreshed in the background), and until the +/// first Tesseract probe lands the choice reads as "still checking" (see +/// [`engine_probe_running`], to ask for another frame meanwhile). +pub fn engine_unavailable_reason(engine: EngineChoice) -> Option { + let ocrs = models_installed_cached(); + #[cfg(not(target_arch = "wasm32"))] + let tesseract = { + let configured = std::env::var_os("PDFCRAFT_TESSERACT").map(|v| v.to_string_lossy().into_owned()); + match tesseract_engine_now(configured.as_deref()) { + Probe::Known(Some(_)) => Ok(()), + Probe::Known(None) => Err(tesseract_unavailable()), + Probe::Pending => Err(tesseract_checking()), + } + }; + #[cfg(target_arch = "wasm32")] + let tesseract: Result<(), String> = Err(tesseract_unavailable()); + match engine { + EngineChoice::Ocrs => (!ocrs).then(models_missing), + EngineChoice::Tesseract => tesseract.err(), + EngineChoice::Auto => match (ocrs, tesseract) { + (true, _) | (false, Ok(())) => None, + (false, Err(why)) => Some(format!("{}; {why}", models_missing())), + }, + } +} + +/// Why Tesseract cannot be chosen yet: its probe is still running in the background. +#[cfg(not(target_arch = "wasm32"))] +fn tesseract_checking() -> String { + "still checking whether the tesseract program can run; try again in a moment".into() +} + +/// What a started job expects of the document when its result is applied: the same open +/// document, unchanged. [`Session::stale_reason`] names any difference it finds. +#[derive(Clone)] +pub struct OcrGuard { + pub(crate) doc: DocId, + pub(crate) pages: usize, + pub(crate) revisions: usize, + /// The document's working bytes when the job started; any edit rewrites them, so a shared + /// `Arc` pointer means "untouched". + pub(crate) bytes: Arc>, +} + /// Everything recognition needs from a document, detached from the session. pub struct OcrJob { bytes: Arc>, @@ -104,40 +631,163 @@ pub struct OcrJob { native: Vec>, pub pages: Vec, pub settings: OcrSettings, + /// What to check before applying this job's result. + pub guard: OcrGuard, +} + +/// A page's "has text" probe for skip-text: at least one non-blank text run. A failed probe +/// (the text layer could not be read) never enables a skip. +fn page_has_text(text: Option<&Arc>) -> bool { + text.is_some_and(|t| t.glyphs.iter().any(|g| !g.text.trim().is_empty())) +} + +/// Clamp a region to the image, snapping outward to whole pixels. `None` when nothing of it +/// is inside the image (empty, inverted, NaN, or wholly outside). +pub fn clamp_region(region: [f32; 4], width: u32, height: u32) -> Option<[u32; 4]> { + let (w, h) = (f64::from(width), f64::from(height)); + if region.iter().any(|v| !v.is_finite()) { + return None; + } + let (x0, y0, x1, y1) = (region[0] as f64, region[1] as f64, region[2] as f64, region[3] as f64); + let (x0, y0) = ((x0.floor().max(0.0).min(w)) as u32, (y0.floor().max(0.0).min(h)) as u32); + let (x1, y1) = ((x1.ceil().max(0.0).min(w)) as u32, (y1.ceil().max(0.0).min(h)) as u32); + (x1 > x0 && y1 > y0).then_some([x0, y0, x1, y1]) +} + +/// Map a recognized word from its source page's user space into the editable output's page +/// user space (the new page is the source's displayed size, unrotated): user → view → new +/// user. The box never moves relative to what the reviewer saw. +fn editable_word(w: &PlacedWord, info: &PageInfo) -> PlacedWord { + let to_new = |p: [f64; 2]| { + let v = info.user_to_view(p[0] as f32, p[1] as f32); + [f64::from(v[0]), f64::from(info.height) - f64::from(v[1])] + }; + let origin = to_new(w.origin); + let br = to_new([w.origin[0] + w.across[0], w.origin[1] + w.across[1]]); + let tl = to_new([w.origin[0] + w.up[0], w.origin[1] + w.up[1]]); + PlacedWord { + text: w.text.clone(), + origin, + across: [br[0] - origin[0], br[1] - origin[1]], + up: [tl[0] - origin[0], tl[1] - origin[1]], + ..w.clone() + } } impl OcrJob { + /// The pixels-per-point this job renders `page` at (the real render dpi is this times 72). + /// The Verify screen uses it to express a region picked on its own preview in the pixels + /// the job will actually read. A scan is read at its own resolution (enlarging it only + /// blurs the letters), at most the chosen one. + pub fn render_scale(&self, page: usize) -> f32 { + let wanted = self.settings.dpi.clamp(72.0, 600.0) / 72.0; + let Some(native) = self.native.get(page).copied().flatten() else { return wanted }; + // A scan whose page is larger than a sheet (often a box given in pixels, one point per + // pixel, #1848) is measured as if shrunk to one: otherwise 1 px/pt is already its full + // image, and no "Downsample to" choice could ask for less. + let fit = self.infos.get(page).map_or(1.0, |i| (SHEET / i.width.max(i.height).max(1e-3)).min(1.0)); + native.clamp(fit, wanted * fit) + } + + /// Whether any of the job's pages already has text (at least one non-blank run): the + /// batch flow's "Skip files that already contain text" probe. A failed probe never + /// enables a skip, so a page whose text cannot be read counts as no text there. + pub fn has_text(&self) -> bool { + let mut r = PageRenderer::new(self.bytes.clone(), RenderConfig { password: self.password.as_deref().map(Arc::from), ..Default::default() }); + self.pages.iter().any(|&page| { + let t = r.render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }); + page_has_text(t.text.as_ref()) + }) + } + /// Read the pages. `progress(done, total)` is called before each page; returning `false` /// stops (the pages read so far are returned). - pub fn run(self, ocr: &Ocr, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { + pub fn run(self, recognizers: &Recognizers, mut progress: impl FnMut(usize, usize) -> bool) -> Vec { let config = RenderConfig { password: self.password.as_deref().map(Arc::from), hide_comments: self.settings.native_text_only, ..Default::default() }; let mut r = PageRenderer::new(self.bytes.clone(), config); let mut out = Vec::new(); let total = self.pages.len(); + let options = pdfcraft_ocr::RecognizeOptions { language: self.settings.language.clone() }; for (done, &page) in self.pages.iter().enumerate() { if !progress(done, total) { break; } let Some(info) = self.infos.get(page) else { continue }; - let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()) }; - // The boxes of the page's own text (an earlier OCR layer's included), in view space. + let skip = |why: &str| OcrPage { page, words: Vec::new(), already_on_page: 0, skipped: Some(why.into()), notes: Vec::new() }; + // The boxes of all text on the page, in view space: an earlier OCR layer's, and words + // only /ActualText names, so neither is read again. let existing: Vec<[f32; 4]> = r .render(RenderRequest { page, kind: RequestKind::Text, scale: 1.0, ..Default::default() }) .text .map(|t| t.glyphs.iter().filter(|g| !g.text.trim().is_empty()).map(|g| g.rect).collect()) .unwrap_or_default(); - if self.settings.skip_text_pages && !existing.is_empty() { + if self.settings.skip_text_pages && !self.settings.force_ocr && !existing.is_empty() && has_own_text(&mut r, page) { out.push(skip("the page already contains text")); continue; } - let shot = r.render(RenderRequest { page, scale: self.scale(page, info), ..Default::default() }); + // A scan is read at its own resolution (enlarging it only blurs the letters), at + // most the chosen one. + let scale = self.render_scale(page); + let shot = r.render(RenderRequest { page, scale, ..Default::default() }); if let Some(e) = shot.error { out.push(skip(&e)); continue; } - let lines = match ocr.recognize(&shot.rgba, shot.width, shot.height) { - Ok(l) => l, + let full = match pdfcraft_ocr::OcrImage::new(shot.width, shot.height, shot.rgba.to_vec()) { + Ok(img) => img, + Err(e) => { + out.push(skip(&e.to_string())); + continue; + } + }; + // The optional region: crop the rendered page, clamped; an empty region skips the + // page with a named reason (nothing was asked of it). + let (image, offset) = match self.settings.region { + None => (full, [0.0f32; 2]), + Some(region) => match clamp_region(region, shot.width, shot.height) { + Some([x0, y0, x1, y1]) => match crop(&full, x0, y0, x1, y1) { + Some(c) => (c, [x0 as f32, y0 as f32]), + None => { + out.push(skip("the selected region is empty")); + continue; + } + }, + None => { + out.push(skip("the selected region is outside the page")); + continue; + } + }, + }; + // Preprocessing sits between the page raster and recognition; its word boxes come + // back in processed coordinates and are mapped to the ORIGINAL raster's before + // placement. The render dpi is the real one (pixels per point × 72), never a + // fixed value. + let pre = pdfcraft_ocr::preprocess::preprocess( + image, + scale * 72.0, + &pdfcraft_ocr::preprocess::PreprocessOptions { + auto_rotate: self.settings.auto_rotate, + deskew: self.settings.deskew, + denoise: self.settings.denoise, + binarize: self.settings.binarize, + }, + ); + let (image, inverse) = match pre { + Ok(p) => (p.image, p.inverse_transform), + Err(e) => { + out.push(skip(&e.to_string())); + continue; + } + }; + let outcome = match pdfcraft_ocr::recognize_with_strategy( + recognizers.primary.as_ref(), + recognizers.secondary.as_deref(), + self.settings.strategy, + &image, + &options, + ) { + Ok(o) => o, Err(e) => { out.push(skip(&e.to_string())); continue; @@ -145,30 +795,69 @@ impl OcrJob { }; let scale = shot.width as f32 / info.width.max(1e-3); let to_user = |x: f32, y: f32| { - let [u, v] = info.view_to_user(x / scale, y / scale); + // processed pixels → original raster pixels (the exact inverse) → past the + // region crop → page view space. + let (ox, oy) = inverse.apply(f64::from(x), f64::from(y)); + let [u, v] = info.view_to_user((ox as f32 + offset[0]) / scale, (oy as f32 + offset[1]) / scale); [u as f64, v as f64] }; - // Words the page's own text already has are not added a second time. - let (fresh, already): (Vec<_>, Vec<_>) = - lines.iter().flat_map(|l| &l.words).partition(|w| covered(w.rect.map(|v| v / scale), &existing) < ALREADY_ON_PAGE); - let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect(); - out.push(OcrPage { page, words, already_on_page: already.len(), skipped: None }); + // Validate recognition geometry and compare against existing text in view space, + // undoing preprocessing and region offsets before testing overlap. + let (fresh, already): (Vec<_>, Vec<_>) = outcome.lines.iter().flat_map(|l| &l.words).filter(|w| box_writable(w)).partition(|w| { + let [l, t, r, b] = w.rect; + let bounds = [(l, t), (r, t), (r, b), (l, b)].into_iter().fold( + [f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY], + |mut bounds, (x, y)| { + let (x, y) = inverse.apply(f64::from(x), f64::from(y)); + let x = (x as f32 + offset[0]) / scale; + let y = (y as f32 + offset[1]) / scale; + bounds[0] = bounds[0].min(x); + bounds[1] = bounds[1].min(y); + bounds[2] = bounds[2].max(x); + bounds[3] = bounds[3].max(y); + bounds + }, + ); + covered(bounds, &existing) < ALREADY_ON_PAGE + }); + let words = fresh.into_iter().map(|w| PlacedWord::place(w, to_user)).collect::>(); + let mut notes = outcome.notes; + // Only visible editable output needs display glyphs. Searchable output preserves + // every Unicode character with empty Type3 programs and ToUnicode maps. + if self.settings.output_mode == OutputMode::EditableText && !words.is_empty() { + let missing = pdfcraft_fonts::TextPlan::for_words(words.iter().map(|w| w.text.as_str())).missing(); + if missing > 0 { + notes.push(format!("{missing} characters had no font that could show them and appear as ? in the output")); + } + } + out.push(OcrPage { page, words, already_on_page: already.len(), skipped: None, notes }); } progress(total, total); out } +} - /// The render scale (pixels per point) `page` is read at. A scan is read at its own - /// resolution (enlarging it only blurs the letters), at most the chosen one. - fn scale(&self, page: usize, info: &PageInfo) -> f32 { - let wanted = self.settings.dpi.clamp(72.0, 600.0) / 72.0; - let Some(native) = self.native.get(page).copied().flatten() else { return wanted }; - // A scan whose page is larger than a sheet (often a box given in pixels, one point per - // pixel, #1848) is measured as if shrunk to one: otherwise 1 px/pt is already its full - // image, and no "Downsample to" choice could ask for less. - let fit = (SHEET / info.width.max(info.height).max(1e-3)).min(1.0); - native.clamp(fit, wanted * fit) +/// Whether a word's pixel box may reach the content stream: width and height (right − left, +/// bottom − top) must be positive and finite. Anything else — a zero, negative, NaN or +/// infinite size — is dropped before placement rather than written into the page. +fn box_writable(word: &pdfcraft_ocr::Word) -> bool { + let [l, t, r, b] = word.rect; + let (width, height) = (r - l, b - t); + width > 0.0 && height > 0.0 && width.is_finite() && height.is_finite() +} + +/// Crop `[x0, y0, x1, y1]` out of `image` as a new [`OcrImage`] (the sides are already clamped +/// by [`clamp_region`]). +fn crop(image: &pdfcraft_ocr::OcrImage, x0: u32, y0: u32, x1: u32, y1: u32) -> Option { + let (w, h) = (x1.checked_sub(x0)?, y1.checked_sub(y0)?); + let stride = image.width() as usize * 4; + let mut out = Vec::with_capacity((w as usize).checked_mul(h as usize)?.checked_mul(4)?); + for y in y0..y1 { + let start = y as usize * stride + x0 as usize * 4; + let row = image.rgba().get(start..start + (x1 - x0) as usize * 4)?; + out.extend_from_slice(row); } + pdfcraft_ocr::OcrImage::new(w, h, out).ok() } impl Session { @@ -193,7 +882,8 @@ impl Session { .reduce(f32::max) }) .collect(); - Some(OcrJob { bytes: doc.bytes.clone(), password: doc.password.clone(), infos: doc.info.pages.clone(), native, pages, settings }) + let guard = OcrGuard { doc: id, pages: all, revisions: doc.revision_ends().len(), bytes: doc.bytes.clone() }; + Some(OcrJob { bytes: doc.bytes.clone(), password: doc.password.clone(), infos: doc.info.pages.clone(), native, pages, settings, guard }) } /// Add the words found by [`OcrJob::run`] as one undoable step. Returns the number of words. @@ -207,14 +897,129 @@ impl Session { Ok(words) } + /// Why a job's result may not be applied to this session any more: the document was closed, + /// edited, re-saved, or its pages changed since the job started. `None` means it is safe. + pub fn stale_reason(&self, guard: &OcrGuard) -> Option { + let Some(doc) = self.get(guard.doc) else { + return Some("the document is no longer open".into()); + }; + if !Arc::ptr_eq(&doc.bytes, &guard.bytes) { + return Some("the document changed since recognition started".into()); + } + let now = doc.info.pages.len(); + if now != guard.pages { + return Some(format!("the page count changed from {} to {} since recognition started", guard.pages, now)); + } + let revisions = doc.revision_ends().len(); + if revisions != guard.revisions { + let had = guard.revisions; + return Some(format!("the document gained revisions ({had} → {revisions}) since recognition started")); + } + None + } + + /// What applying a result to document `id` expects of it right now. After a checked apply + /// succeeds the document has changed (that apply is its newest edit), so a caller that + /// applies more of the same review in further steps (page by page) takes a fresh guard + /// here; any other edit in between still makes the next apply stale. + pub fn ocr_guard(&self, id: DocId) -> Option { + let doc = self.get(id)?; + Some(OcrGuard { doc: id, pages: doc.info.pages.len(), revisions: doc.revision_ends().len(), bytes: doc.bytes.clone() }) + } + + /// [`Self::apply_ocr`], but only while the document is still the one that was read; a stale + /// result is refused with a named reason and nothing is written. + pub fn apply_ocr_checked(&mut self, id: DocId, found: &[OcrPage], guard: &OcrGuard) -> Result { + if guard.doc != id { + return Err("the result belongs to another document".into()); + } + if let Some(why) = self.stale_reason(guard) { + return Err(why); + } + self.apply_ocr(id, found).map_err(|e| e.to_string()) + } + + /// The editable-text output: a NEW document with the recognized words as VISIBLE text at + /// their original boxes and no scan image — one page per source page, blank where nothing + /// was recognized, each page the displayed size of its source. The open document is never + /// touched; the caller chooses where the bytes are written. Refuses when there are no + /// words at all (an empty document would silently lose the scan's content). + pub fn editable_ocr_bytes(&self, id: DocId, found: &[OcrPage]) -> Result>, String> { + let doc = self.get(id).ok_or("no such document")?; + let words = layer::validate_words(found.iter().flat_map(|p| &p.words)).map_err(|e| e.to_string())?; + if words == 0 { + return Err("the recognition found no words, so there is no editable text to write".into()); + } + // The words per page, in user space; and one plan over all of them, so a face's Type3 + // font is built once and shared by every page that uses it. + let page_words: Vec> = (0..doc.info.pages.len()) + .map(|p| { + let Some(info) = doc.info.pages.get(p) else { return Vec::new() }; + found.iter().find(|f| f.page == p).map(|f| f.words.iter().map(|w| editable_word(w, info)).collect()).unwrap_or_default() + }) + .collect(); + layer::validate_words(page_words.iter().flatten()).map_err(|e| e.to_string())?; + let mut plan = pdfcraft_fonts::TextPlan::for_words(page_words.iter().flat_map(|ws| ws.iter().map(|w| w.text.as_str()))); + let pages: Vec<(f64, f64, pdfcraft_cos::Dict, Vec)> = (0..doc.info.pages.len()) + .map(|p| { + let info = doc.info.pages.get(p); + (info.map_or(612.0, |i| f64::from(i.width)), info.map_or(792.0, |i| f64::from(i.height)), pdfcraft_cos::Dict::new(), Vec::new()) + }) + .collect(); + let mut created = pdfcraft_create::from_contents(&pages).map_err(|e| e.to_string())?; + // One shared Helvetica for every page: added once into the new document, then linked + // from each page's resources as /PCHelv (the name the text layers draw with). The + // plan's fonts are linked per page too; their objects are shared. + let mut font = pdfcraft_cos::Dict::new(); + font.set(b"Type".to_vec(), pdfcraft_cos::Object::name("Font")); + font.set(b"Subtype".to_vec(), pdfcraft_cos::Object::name("Type1")); + font.set(b"BaseFont".to_vec(), pdfcraft_cos::Object::name("Helvetica")); + font.set(b"Encoding".to_vec(), pdfcraft_cos::Object::name("WinAnsiEncoding")); + let fr = created.add(font); + // The page tree's kids, direct array or indirect reference (a fresh create document + // keeps /Kids inline) — the output's pages are linked through them. + let kids = created + .root() + .and_then(|r| created.get(r).as_dict().cloned()) + .and_then(|d| d.reference(b"Pages")) + .and_then(|p| created.get(p).as_dict().cloned()) + .and_then(|d| match d.get(b"Kids").map(|k| created.resolve(k)).as_deref() { + Some(pdfcraft_cos::Object::Array(a)) => Some(a.clone()), + Some(pdfcraft_cos::Object::Ref(r)) => created.get(*r).as_array().cloned(), + _ => None, + }); + for (pi, kid) in kids.into_iter().flatten().enumerate() { + let pdfcraft_cos::Object::Ref(page) = kid else { continue }; + let _ = created.update_dict(page, |d| { + let mut res = d.get(b"Resources").and_then(|o| o.as_dict().cloned()).unwrap_or_default(); + let mut fonts = res.get(b"Font").and_then(|o| o.as_dict().cloned()).unwrap_or_default(); + fonts.set(b"PCHelv".to_vec(), pdfcraft_cos::Object::Ref(fr)); + res.set(b"Font".to_vec(), pdfcraft_cos::Object::Dict(fonts)); + d.set(b"Resources".to_vec(), pdfcraft_cos::Object::Dict(res)); + }); + // The page's content is written after the fonts carry their resource names, so + // the runs can reference them. + if page_words.get(pi).is_some_and(|ws| !ws.is_empty()) { + pdfcraft_edit::add_type3_fonts(&mut created, pi, &mut plan).map_err(|e| e.to_string())?; + let word_offset = page_words.iter().take(pi).map(Vec::len).sum(); + let content = pdfcraft_ocr::visible_text_layer_at(&page_words[pi], &plan, word_offset); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"PCMark".to_vec(), pdfcraft_cos::Object::name("OCR")); + let stream = created.add(pdfcraft_cos::Object::Stream(pdfcraft_cos::Stream::flate(dict, &content))); + let _ = created.update_dict(page, |d| d.set(b"Contents".to_vec(), pdfcraft_cos::Object::Ref(stream))); + } + } + self.write_new(&created).map_err(|e| e.to_string()) + } + /// Recognise text on `pages` and add it, in one call (the CLI and agents use this). pub fn recognize_text(&mut self, id: DocId, pages: &[usize], settings: OcrSettings) -> Result, String> { let job = self.ocr_job(id, pages, settings).ok_or("no such document")?; if let Some(why) = self.get(id).and_then(|d| d.read_only_reason.clone()) { return Err(why); } - let ocr = engine()?; - let found = job.run(&ocr, |_, _| true); + let recognizers = recognizers(&job.settings)?; + let found = job.run(&recognizers, |_, _| true); self.apply_ocr(id, &found).map_err(|e| e.to_string())?; Ok(found) } @@ -226,22 +1031,68 @@ pub struct FileResult { /// The new file (incrementally saved), or the original when nothing was recognised. pub bytes: Arc>, pub pages: Vec, + /// Why the whole file was left alone ("Skip files that already contain text"): a skipped + /// file is its own result bucket, never counted as an OCR success. + pub file_skipped: Option, } impl FileResult { pub fn words(&self) -> usize { self.pages.iter().map(|p| p.words.len()).sum() } + + /// Whether the file was passed over entirely (never an OCR success). + pub fn skipped(&self) -> bool { + self.file_skipped.is_some() + } +} + +/// The output names a multi-file run would write more than once: the batch writes every +/// result into one folder under its input's file name, so two inputs of the same name from +/// different folders would overwrite each other. Compared case-insensitively (the Windows and +/// macOS file systems are), in first-seen order, each listed once. +pub fn duplicate_output_names<'a>(names: impl IntoIterator) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut listed = std::collections::HashSet::new(); + let mut out = Vec::new(); + for name in names { + let key = name.to_lowercase(); + if !seen.insert(key.clone()) && listed.insert(key) { + out.push(name.to_string()); + } + } + out +} + +/// The batch note's data: how many words sit at or under `threshold` ([`low_confidence`], the +/// separate batch rule), and the 1-based, sorted pages they are on. `None` when there are +/// none, so a note is only written when there is something to review. +pub fn low_confidence_pages(pages: &[OcrPage], threshold: f32) -> Option<(usize, Vec)> { + let (mut count, mut page_numbers) = (0usize, Vec::new()); + for p in pages { + if p.skipped.is_some() { + continue; + } + let on_page = p.words.iter().filter(|w| w.confidence.is_some_and(|c| pdfcraft_ocr::low_confidence(c, threshold))).count(); + if on_page > 0 { + count += on_page; + page_numbers.push(p.page + 1); + } + } + page_numbers.sort_unstable(); + (count > 0).then_some((count, page_numbers)) } /// Recognize text in multiple files: one PDF's bytes in, the searchable PDF out. `progress` -/// works as for [`OcrJob::run`]. +/// works as for [`OcrJob::run`]. A file that the skip-files option passes over comes back with +/// [`FileResult::file_skipped`] set and its original bytes; the caller decides what to do with +/// it, but it is never an OCR success. pub fn recognize_file( name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings, - ocr: &Ocr, + recognizers: &Recognizers, progress: impl FnMut(usize, usize) -> bool, ) -> Result { let mut s = Session::new(); @@ -249,15 +1100,31 @@ pub fn recognize_file( if let Some(why) = s.get(id).and_then(|d| d.read_only_reason.clone()) { return Err(why); } - let job = s.ocr_job(id, &[], settings).ok_or("the document could not be read")?; - let pages = job.run(ocr, progress); + let job = s.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; + // The skip-files probe runs before any engine work: a file with text needs no engine at + // all, and the original bytes come back unchanged. + if settings.skip_text_files && !settings.force_ocr && job.has_text() { + let pages = job + .pages + .iter() + .map(|&page| OcrPage { + already_on_page: 0, + page, + words: Vec::new(), + skipped: Some("the file already contains text".into()), + notes: Vec::new(), + }) + .collect(); + return Ok(FileResult { bytes, pages, file_skipped: Some("the file already contains text".into()) }); + } + let pages = job.run(recognizers, progress); if s.apply_ocr(id, &pages).map_err(|e| e.to_string())? == 0 { - return Ok(FileResult { bytes, pages }); + return Ok(FileResult { bytes, pages, file_skipped: None }); } // Checked like a save before the caller writes it (#2166). let bytes = s.save_bytes(id).map_err(|e| e.to_string())?; s.check_save(id, &bytes).map_err(|e| e.to_string())?; - Ok(FileResult { bytes, pages }) + Ok(FileResult { bytes, pages, file_skipped: None }) } #[cfg(test)] @@ -265,6 +1132,34 @@ mod tests { use super::*; use crate::export; + /// One 300 × 300 page drawing `content`, with Helvetica as `/F1`. + fn page(content: &str) -> Arc> { + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + content.len() + ); + Arc::new(pdf.into_bytes()) + } + + #[test] + fn a_stamp_drawn_as_outlines_leaves_a_scan_to_read() { + let stamp = page("/Span <> BDC 100 40 60 20 re f EMC"); + let mut r = PageRenderer::new(stamp, RenderConfig::default()); + assert!(!has_own_text(&mut r, 0), "words only /ActualText names are no text of the page's own"); + // They still count as on the page, so reading the scan doesn't add them twice. + let text = r.render(RenderRequest { page: 0, kind: RequestKind::Text, scale: 1.0, ..Default::default() }).text.unwrap(); + assert_eq!(text.plain_text(), "APPROVED"); + } + + #[test] + fn printed_glyphs_count_as_own_text() { + let mut r = PageRenderer::new(page("BT /F1 12 Tf 20 150 Td (Total) Tj ET"), RenderConfig::default()); + assert!(has_own_text(&mut r, 0)); + } + /// #1028: a recognised word over the page's own text is that text again; one in an empty /// part of the page (a picture) is new. #[test] @@ -282,28 +1177,6 @@ mod tests { assert_eq!(covered([100.5, 52.0, 134.0, 61.0], &[]), 0.0); } - /// #1848: an A4 scan at 300 dpi whose page box is its pixel size (2480 x 3508 pt, one pixel - /// per point) gets a smaller bitmap for each lower "Downsample to" choice, and a low-resolution - /// letter page is still never shrunk below one pixel per point. - #[test] - fn downsample_shrinks_a_page_box_given_in_pixels() { - let mut s = Session::new(); - let page = |w: u32, h: u32| export::encode_png(w, h, &vec![255; (w * h * 4) as usize]).unwrap(); - let bytes = s.create_from_images(&[("a4.png".into(), page(2480, 3508)), ("letter.png".into(), page(612, 792))]).unwrap(); - let id = s.open("scan.pdf", None, bytes, None).unwrap(); - let mut r = PageRenderer::new(s.get(id).unwrap().bytes.clone(), RenderConfig::default()); - for (dpi, a4) in [(72.0, [595, 842]), (150.0, [1240, 1754]), (300.0, [2480, 3508]), (600.0, [2480, 3508])] { - let job = s.ocr_job(id, &[], OcrSettings { dpi, ..Default::default() }).unwrap(); - for (page, want) in [(0, a4), (1, [612, 792])] { - let scale = job.scale(page, &job.infos[page]); - let shot = r.render(RenderRequest { page, scale, ..Default::default() }); - assert!(shot.error.is_none(), "{:?}", shot.error); - let near = |got: u32, want: u32| got.abs_diff(want) <= 1; - assert!(near(shot.width, want[0]) && near(shot.height, want[1]), "{dpi} dpi, page {page}: {}x{}", shot.width, shot.height); - } - } - } - #[test] fn coverage_of_overlapping_or_broken_boxes_stays_between_zero_and_one() { // A glyph drawn twice (fake bold) does not cover more than the whole word. @@ -317,4 +1190,846 @@ mod tests { let c = covered([0.0, 0.0, 10.0, 10.0], &broken); assert!((0.0..=1.0).contains(&c), "{c}"); } + use crate::tests::fixture; + use pdfcraft_render::TextGlyph; + + fn glyph(text: &str) -> TextGlyph { + TextGlyph { text: text.into(), rect: [0.0, 0.0, 1.0, 1.0], quad: [[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]], direction: [1.0, 0.0] } + } + + /// The skip-text probe: at least one non-blank run; blank-only or missing text never + /// enables a skip. + #[test] + fn has_text_probe_needs_one_non_blank_run() { + let none: Option> = None; + assert!(!page_has_text(none.as_ref()), "a failed probe never enables a skip"); + assert!(!page_has_text(Some(Arc::new(pdfcraft_render::PageText::default())).as_ref())); + let blank = pdfcraft_render::PageText { glyphs: vec![glyph(" "), glyph("")], ..Default::default() }; + assert!(!page_has_text(Some(Arc::new(blank)).as_ref())); + let has = pdfcraft_render::PageText { glyphs: vec![glyph(" "), glyph("a")], ..Default::default() }; + assert!(page_has_text(Some(Arc::new(has)).as_ref())); + } + + #[test] + fn output_mode_parsing_falls_back_to_searchable() { + assert_eq!(OutputMode::parse("editable"), OutputMode::EditableText); + assert_eq!(OutputMode::parse(" Editable-Text "), OutputMode::EditableText); + assert_eq!(OutputMode::parse("searchable"), OutputMode::Searchable); + assert_eq!(OutputMode::parse("nonsense"), OutputMode::Searchable, "unknown strings mean Searchable"); + assert_eq!(OutputMode::parse(""), OutputMode::Searchable); + } + + #[test] + fn engine_choice_parsing() { + assert_eq!(EngineChoice::parse("tesseract"), EngineChoice::Tesseract); + assert_eq!(EngineChoice::parse(" AUTO "), EngineChoice::Auto); + assert_eq!(EngineChoice::parse("ocrs"), EngineChoice::Ocrs); + assert_eq!(EngineChoice::parse("wat"), EngineChoice::Ocrs, "unknown names fall back to the default"); + assert_eq!(EngineChoice::default().name(), "ocrs"); + } + + /// Automatic reads with the second engine when the first is unsure unless a strategy was + /// chosen: "Primary only" would never run it. A named engine reads alone. + #[test] + fn automatic_defaults_to_the_confidence_weighted_strategy() { + assert_eq!(EngineChoice::Auto.default_strategy(), MergeStrategy::ConfidenceWeighted); + assert_eq!(EngineChoice::Ocrs.default_strategy(), MergeStrategy::PrimaryOnly); + assert_eq!(EngineChoice::Tesseract.default_strategy(), MergeStrategy::PrimaryOnly); + } + + /// Settings default to today's behavior: ocrs, PrimaryOnly, no preprocessing, searchable. + #[test] + fn settings_default_to_the_ocrs_status_quo() { + let s = OcrSettings::default(); + assert_eq!(s.engine, EngineChoice::Ocrs); + assert_eq!(s.strategy, MergeStrategy::PrimaryOnly); + assert_eq!(s.output_mode, OutputMode::Searchable); + assert!(!s.auto_rotate && !s.deskew && !s.denoise && !s.binarize); + assert_eq!(s.region, None); + assert_eq!(s.low_confidence_threshold, 60.0); + assert_eq!(s.tesseract_path, None, "no configured tesseract path by default: the PATH decides"); + } + + /// A configured tesseract path that is not an absolute, existing, executable file makes + /// the Tesseract choice unavailable — the explicit setting is never silently replaced by + /// a PATH search (and this holds whatever this machine has installed). + #[test] + fn an_invalid_configured_tesseract_path_is_unavailable_not_a_fallback() { + let bogus = OcrSettings { engine: EngineChoice::Tesseract, tesseract_path: Some("not/a/program".into()), ..Default::default() }; + match recognizers(&bogus) { + Ok(r) => panic!("a bogus configured path must not resolve: {:?}", r.primary.id()), + Err(e) => assert!(e.contains("tesseract"), "{e}"), + } + } + + /// A failed engine probe is remembered only for the negative TTL — a program installed + /// while the app runs is found by a later job, without a restart — while a found engine is + /// remembered for the session. + #[test] + fn a_negative_probe_expires_a_found_engine_does_not() { + let now = Instant::now(); + let old = now.checked_sub(Duration::from_secs(31)).unwrap(); + assert!(!probe_fresh(false, old, now), "past the TTL the probe is retried"); + assert!(probe_fresh(false, now - Duration::from_secs(5), now), "a fresh failure is not re-probed"); + assert!(probe_fresh(true, old, now), "a found engine is kept"); + } + + /// `ocr_status` and a job judge the same location: one named by the job's settings is the + /// one reported (here an invalid one, so no process starts and the answer is the same on + /// every machine). + #[test] + fn the_status_follows_the_location_a_job_names() { + let bogus = OcrSettings { tesseract_path: Some("not/a/program".into()), ..Default::default() }; + let tesseract = engines_status_for(&bogus).into_iter().find(|e| e.id == "tesseract").expect("tesseract is always listed"); + assert!(!tesseract.available, "{tesseract:?}"); + assert!(tesseract.why_not.as_deref().is_some_and(|why| why.contains("not/a/program")), "{tesseract:?}"); + assert_eq!(tesseract.program.as_deref(), Some("not/a/program")); + } + + /// What the interface sees of the cache: a fresh answer as is, an expired failed probe as + /// the best known answer (shown while it is retried, so the choice does not flicker), and + /// nothing for a location nobody asked about. + #[test] + fn the_cache_lookup_keeps_a_stale_answer_and_knows_its_location() { + let now = Instant::now(); + let old = now.checked_sub(Duration::from_secs(31)).unwrap(); + let failed = |key: Option<&str>, at| CachedTesseract { key: key.map(str::to_string), found: None, at }; + assert!(matches!(cache_lookup(None, None, now), Cached::Missing)); + assert!(matches!(cache_lookup(Some(&failed(None, now)), None, now), Cached::Fresh(None))); + assert!(matches!(cache_lookup(Some(&failed(None, old)), None, now), Cached::Stale(None))); + assert!(matches!(cache_lookup(Some(&failed(Some("/x"), now)), None, now), Cached::Missing), "another location is another question"); + let found = CachedTesseract { key: None, found: Some(Arc::new(Fake::new())), at: old }; + assert!(matches!(cache_lookup(Some(&found), None, now), Cached::Fresh(Some(_))), "a found engine stays fresh"); + } + + /// Engine discovery never deadlocks or corrupts the cache: concurrent lookups with + /// different configured locations (all invalid, so they are refused before any process + /// starts) all finish. + #[test] + fn concurrent_engine_lookups_all_finish() { + let handles: Vec<_> = (0..8) + .map(|i| { + std::thread::spawn(move || { + let configured = if i % 2 == 0 { Some(format!("not/a/program-{i}")) } else { None }; + let _ = tesseract_engine(configured.as_deref()); + }) + }) + .collect(); + for h in handles { + h.join().expect("no panic, no deadlock"); + } + } + + /// The preprocessing flags are not just plumbing: a page read with all four on still + /// yields the same words, in the same places (needs the models). + #[test] + fn preprocessing_flags_flow_through_the_job() { + if !available() { + eprintln!("skipped: no OCR engine available"); + return; + } + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "The quick brown fox jumps over the lazy dog.").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = crate::export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + let scan = s.create_from_images(&[("scan.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, scan, None).unwrap(); + let settings = OcrSettings { auto_rotate: true, deskew: true, denoise: true, binarize: true, ..Default::default() }; + let found = s.recognize_text(id, &[], settings).unwrap(); + let text = crate::tests::page_texts(&s, id)[0].to_lowercase(); + // Binarizing an anti-aliased render thins the thin strokes, so edge letters drop + // ("lazy" reads as "azy", "brown" as "rown"); the match rests on what stays stable — + // the middle of "quick" (some CPUs' kernels read the model's "q" as an "a", the same + // "uick" the OCR test in crate::tests pins) and whole short words. + for w in ["uick", "fox", "over"] { + assert!(text.contains(w), "{text}"); + } + // The words were placed, meaning boxes survived the trip back to raster coordinates. + assert!(!found[0].words.is_empty()); + } + + /// suspects(): only Low-band words, by index. + #[test] + fn page_suspects_are_the_low_band_words() { + let word = |t: &str, c: Option| PlacedWord { + text: t.into(), + origin: [0.0; 2], + across: [1.0, 0.0], + up: [0.0, 1.0], + confidence: c, + source: "ocrs".into(), + }; + let page = OcrPage { + already_on_page: 0, + page: 0, + words: vec![word("a", Some(95.0)), word("b", Some(50.0)), word("c", None), word("d", Some(69.0))], + skipped: None, + notes: Vec::new(), + }; + assert_eq!(page.suspects(), vec![1, 3]); + let mean = page.mean_confidence().unwrap_or_default(); + assert!((mean - 71.333).abs() < 0.01, "{mean}"); + } + + /// Region clamping: interior kept, negative extents snapped outward, garbage refused. + #[test] + fn regions_clamp_and_refuse_garbage() { + assert_eq!(clamp_region([10.0, 10.0, 20.0, 20.0], 100, 100), Some([10, 10, 20, 20])); + assert_eq!(clamp_region([-5.0, -5.5, 20.4, 20.0], 100, 100), Some([0, 0, 21, 20]), "snapped outward, clamped to the image"); + assert_eq!(clamp_region([50.0, 50.0, 200.0, 200.0], 100, 100), Some([50, 50, 100, 100])); + assert_eq!(clamp_region([10.0, 10.0, 10.0, 20.0], 100, 100), None, "empty"); + assert_eq!(clamp_region([20.0, 10.0, 10.0, 20.0], 100, 100), None, "inverted"); + assert_eq!(clamp_region([f32::NAN, 0.0, 1.0, 1.0], 100, 100), None, "NaN"); + assert_eq!(clamp_region([-20.0, -20.0, -5.0, -5.0], 100, 100), None, "wholly outside"); + } + + /// A word box only reaches the content stream with positive, finite geometry: NaN, + /// infinite, negative and zero-sized boxes are dropped before placement, so nothing + /// non-finite is ever stamped onto the page. + #[test] + fn non_finite_and_negative_word_boxes_are_dropped_before_placement() { + let word = |rect: [f32; 4]| pdfcraft_ocr::Word::new("w", rect, "ocrs"); + for (why, rect) in [ + ("NaN width", [f32::NAN, 0.0, 10.0, 10.0]), + ("NaN height", [0.0, 0.0, 10.0, f32::NAN]), + ("infinite right", [0.0, 0.0, f32::INFINITY, 10.0]), + ("1e39 height overflows to inf", [0.0, 0.0, 10.0, 1e39f64 as f32]), + ("negative width", [10.0, 0.0, 0.0, 10.0]), + ("negative height", [0.0, 10.0, 10.0, 0.0]), + ("zero size", [5.0, 5.0, 5.0, 5.0]), + ] { + assert!(!box_writable(&word(rect)), "{why}: {rect:?}"); + } + assert!(box_writable(&word([0.0, 0.0, 10.0, 10.0])), "a sane box is placed"); + } + + /// A crop takes exactly the asked pixels (this feeds words back to page space, so edges + /// matter). + #[test] + fn crops_take_the_asked_pixels() { + let img = pdfcraft_ocr::OcrImage::new(3, 2, (0..6).flat_map(|p| vec![p as u8; 4]).collect()).unwrap(); + let c = crop(&img, 1, 0, 3, 2).unwrap(); + assert_eq!((c.width(), c.height()), (2, 2)); + assert_eq!(c.rgba()[0], 1, "the crop starts at x=1"); + assert_eq!(c.rgba()[4], 2, "the first row's second column is x=2"); + assert_eq!(c.rgba()[8], 4, "the second row starts at y=1"); + } + + /// The stale guard: edits and closing are named and the result is refused; a page-count or + /// revision difference is named too (any edit rewrites the working bytes, so those branches + /// are reached with a hand-built guard). + #[test] + fn stale_results_are_named_and_refused() { + let mut s = Session::new(); + let id = s.open("t.pdf", None, Arc::new(fixture(2)), None).unwrap(); + let job = s.ocr_job(id, &[], OcrSettings::default()).unwrap(); + assert_eq!(s.stale_reason(&job.guard), None, "freshly captured"); + // Any edit rewrites the working bytes. + s.apply(id, Edit::AddOcrText { page: 0, words: vec![] }).unwrap(); + assert_eq!(s.stale_reason(&job.guard).as_deref(), Some("the document changed since recognition started")); + assert!(s.apply_ocr_checked(id, &[], &job.guard).is_err(), "a stale result is never applied"); + // A fresh guard passes. + let fresh = s.ocr_job(id, &[], OcrSettings::default()).unwrap(); + assert_eq!(s.stale_reason(&fresh.guard), None); + // A page count and a revision count from another life are named too. + let doc = s.get(id).unwrap(); + let (bytes, revisions) = (doc.bytes.clone(), doc.revision_ends().len()); + let guard = OcrGuard { doc: id, pages: 5, revisions, bytes }; + assert!(s.stale_reason(&guard).is_some_and(|r| r.contains("page count changed")), "{}", s.stale_reason(&guard).unwrap()); + let guard = OcrGuard { doc: id, pages: 2, revisions: revisions + 3, bytes: s.get(id).unwrap().bytes.clone() }; + assert!(s.stale_reason(&guard).is_some_and(|r| r.contains("revisions")), "{}", s.stale_reason(&guard).unwrap()); + // A guard for a different document refuses up front. + let other = s.open("other.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let cross = s.apply_ocr_checked(other, &[], &fresh.guard); + assert!(cross.as_ref().is_err_and(|e| e.contains("another document")), "{cross:?}"); + // And so is a closed document. + s.close(id); + assert_eq!(s.stale_reason(&fresh.guard).as_deref(), Some("the document is no longer open")); + } + + /// The stale guard, exercised end to end through the recognition flow itself (the flow the + /// UI's Verify screen and the automation's two-phase path drive): capture a job on a + /// synthetic scanned page, run the recognition, edit the document, and the captured result + /// is refused with its named reason and nothing written; a fresh recognition over the + /// edited document applies. + #[test] + fn stale_results_are_refused_end_to_end_through_the_recognition_flow() { + if !available() { + eprintln!("skipped: no OCR engine available"); + return; + } + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "The quick brown fox jumps over the lazy dog.").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = crate::export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + let scan = s.create_from_images(&[("scan.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, scan, None).unwrap(); + + // Recognize: the job (carrying its guard) is captured first, then the reading runs. + let job = s.ocr_job(id, &[], OcrSettings::default()).unwrap(); + let guard = job.guard.clone(); + let r = recognizers(&job.settings).unwrap(); + let found = job.run(&r, |_, _| true); + assert!(!found[0].words.is_empty(), "the scan was read: {:?}", found[0].skipped); + + // The document is edited while the result sits around; any edit rewrites the bytes. + s.apply(id, Edit::AddOcrText { page: 0, words: vec![] }).unwrap(); + + // Applying the now-stale result is refused with its named reason and writes nothing. + let after_edit = s.get(id).unwrap().bytes.clone(); + assert!(!Arc::ptr_eq(&after_edit, &guard.bytes), "the edit rewrote the working bytes"); + let revisions = s.get(id).unwrap().revision_ends().len(); + let why = s.apply_ocr_checked(id, &found, &guard).unwrap_err(); + assert_eq!(why, "the document changed since recognition started", "{why}"); + assert_eq!(s.get(id).unwrap().revision_ends().len(), revisions, "the refusal wrote no revision"); + assert!(Arc::ptr_eq(&s.get(id).unwrap().bytes, &after_edit), "the refused apply wrote nothing"); + assert!(crate::tests::page_texts(&s, id)[0].trim().is_empty(), "no text layer landed"); + + // A fresh recognition over the current document applies: same flow, valid now. + let job = s.ocr_job(id, &[], OcrSettings::default()).unwrap(); + let guard = job.guard.clone(); + assert_eq!(s.stale_reason(&guard), None, "a fresh job is not stale"); + let found = job.run(&r, |_, _| true); + let words = s.apply_ocr_checked(id, &found, &guard).unwrap(); + assert!(words > 0, "the re-run found the words again"); + let text = crate::tests::page_texts(&s, id)[0].to_lowercase(); + assert!(text.contains("fox"), "{text}"); + } + + /// The selector refuses an unusable engine with its cause named, and resolves the ensemble + /// when both engines exist (here: whatever this machine has for ocrs and tesseract). + #[test] + fn selector_names_the_missing_engine() { + let tesseract = OcrSettings { engine: EngineChoice::Tesseract, ..Default::default() }; + match recognizers(&tesseract) { + Ok(r) => assert_eq!(r.primary.id(), "tesseract"), + Err(e) => assert!(e.contains("tesseract"), "{e}"), + } + let ocrs = OcrSettings { engine: EngineChoice::Ocrs, ..Default::default() }; + match recognizers(&ocrs) { + Ok(r) => assert_eq!(r.primary.id(), "ocrs"), + Err(e) => assert!(e.contains("models"), "{e}"), + } + } + + /// An ocrs-only choice never starts the Tesseract probe: the probe is a `--list-langs` + /// process, and a job that names its engine needs no reading from Tesseract. The test's + /// fake program touches a marker file when it runs, so the probe's start is observable; + /// it needs a Unix shell, like the other process fixtures. + #[cfg(unix)] + #[test] + fn an_ocrs_only_choice_never_probes_tesseract() { + use std::os::unix::fs::PermissionsExt; + let probe = |name: &str| { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-probe-{}-{name}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let program = dir.join("fake-tesseract"); + std::fs::write(&program, "#!/bin/sh\ntouch \"$0.probed\"\nexit 1\n").unwrap(); + let mut perms = std::fs::metadata(&program).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&program, perms).unwrap(); + (dir.join("fake-tesseract.probed"), program.to_string_lossy().into_owned()) + }; + // An ocrs-only job: the configured location stays untouched. + let (marker, program) = probe("ocrs-only"); + let ocrs_only = OcrSettings { engine: EngineChoice::Ocrs, tesseract_path: Some(program), ..Default::default() }; + let _ = recognizers(&ocrs_only); + assert!(!marker.exists(), "an ocrs-only job must not start the tesseract probe"); + std::fs::remove_file(marker.parent().unwrap().join("fake-tesseract")).ok(); + + // A Tesseract job does validate its program (that is how it is found usable). + let (marker, program) = probe("tesseract"); + let wants = OcrSettings { engine: EngineChoice::Tesseract, tesseract_path: Some(program), ..Default::default() }; + let _ = recognizers(&wants); + assert!(marker.exists(), "a tesseract job probes its program"); + } + + /// A fake engine for the batch skip logic: reads one confident word from anything, counts + /// its calls, and never touches the filesystem. + struct Fake { + calls: std::sync::atomic::AtomicUsize, + } + + impl Fake { + fn new() -> Fake { + Fake { calls: std::sync::atomic::AtomicUsize::new(0) } + } + fn calls(&self) -> usize { + self.calls.load(std::sync::atomic::Ordering::SeqCst) + } + } + + impl Recognizer for Fake { + fn id(&self) -> &'static str { + "fake" + } + fn available(&self) -> bool { + true + } + fn languages(&self) -> Vec<(String, String)> { + vec![("en".into(), "English".into())] + } + fn recognize(&self, _image: &pdfcraft_ocr::OcrImage, _options: &pdfcraft_ocr::RecognizeOptions) -> Result, OcrError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(vec![pdfcraft_ocr::Line { words: vec![pdfcraft_ocr::Word::new("word", [1.0, 1.0, 9.0, 9.0], "fake").with_confidence(95.0)] }]) + } + } + + fn fake_recognizers() -> (Recognizers, Arc) { + let fake = Arc::new(Fake::new()); + (Recognizers { primary: fake.clone(), secondary: None }, fake) + } + + /// Skip-files: a file that already has text is passed over entirely — the engine never + /// runs, the original bytes come back, and the result is its own skipped bucket. + #[test] + fn skip_files_passes_text_files_over_without_running_the_engine() { + let s = Session::new(); + let bytes = s.create_from_text("t", "Already typed here").unwrap(); + let (r, fake) = fake_recognizers(); + let settings = OcrSettings { skip_text_files: true, ..Default::default() }; + let out = recognize_file("t.pdf", bytes.clone(), None, settings, &r, |_, _| true).unwrap(); + assert_eq!(fake.calls(), 0, "the engine never ran"); + assert_eq!(out.bytes.as_ptr(), bytes.as_ptr(), "the original bytes came back"); + assert_eq!(out.file_skipped.as_deref(), Some("the file already contains text")); + assert!(out.skipped() && out.words() == 0); + } + + /// Force OCR overrides the file skip: the file is read even though it has text, and the + /// result is an OCR success (new bytes, words placed), never a skipped bucket. + #[test] + fn force_ocr_overrides_the_file_skip() { + let s = Session::new(); + let bytes = s.create_from_text("t", "Already typed here").unwrap(); + let (r, fake) = fake_recognizers(); + let settings = OcrSettings { skip_text_files: true, force_ocr: true, ..Default::default() }; + let out = recognize_file("t.pdf", bytes, None, settings, &r, |_, _| true).unwrap(); + assert_eq!(fake.calls(), 1); + assert_eq!(out.file_skipped, None); + assert_eq!(out.words(), 1); + assert!(!out.bytes.is_empty()); + } + + /// Force OCR also overrides the per-page skip: a page with text is read anyway. + #[test] + fn force_ocr_overrides_the_page_skip() { + let s = Session::new(); + let bytes = s.create_from_text("t", "A page with text").unwrap(); + let (r, fake) = fake_recognizers(); + let settings = OcrSettings { force_ocr: true, ..Default::default() }; + let out = recognize_file("t.pdf", bytes, None, settings, &r, |_, _| true).unwrap(); + assert_eq!(fake.calls(), 1, "the page's own text did not stop the run"); + assert_eq!(out.pages[0].skipped, None); + } + + /// The per-page skip stays the default: with force off, a text page is passed over (and a + /// second run over a searchable output is a no-op — nothing left to read). + #[test] + fn page_skip_leaves_text_pages_alone_and_a_rerun_is_a_noop() { + let s = Session::new(); + let bytes = s.create_from_text("t", "A page with text").unwrap(); + let (r, fake) = fake_recognizers(); + let out = recognize_file("t.pdf", bytes, None, OcrSettings::default(), &r, |_, _| true).unwrap(); + assert_eq!(fake.calls(), 0); + assert_eq!(out.pages[0].skipped.as_deref(), Some("the page already contains text")); + assert_eq!(out.words(), 0); + // Second run over the same (already text-bearing) file: skipped pages again, no calls. + let again = recognize_file("t.pdf", out.bytes, None, OcrSettings::default(), &r, |_, _| true).unwrap(); + assert_eq!(fake.calls(), 0); + assert_eq!(again.pages[0].skipped.as_deref(), Some("the page already contains text")); + } + + /// Two inputs of one name (from different folders, or differing only in case) would + /// write the same output: both are found, each once. + #[test] + fn duplicate_output_names_are_found_case_insensitively() { + assert!(duplicate_output_names(["a.pdf", "b.pdf"]).is_empty()); + assert_eq!(duplicate_output_names(["a.pdf", "b.pdf", "A.PDF", "b.pdf", "a.pdf"]), ["A.PDF", "b.pdf"]); + } + + /// The batch note's data: counts words at or under the threshold per page (1-based, sorted; + /// skipped pages never contribute), and says nothing when there is nothing to review. + #[test] + fn low_confidence_pages_counts_the_batch_rule() { + let word = |c: f32| PlacedWord { + text: "w".into(), + origin: [0.0; 2], + across: [1.0, 0.0], + up: [0.0, 1.0], + confidence: Some(c), + source: "ocrs".into(), + }; + let page = |n: usize, confs: &[f32], skipped: Option<&str>| OcrPage { + already_on_page: 0, + page: n, + words: confs.iter().map(|c| word(*c)).collect(), + skipped: skipped.map(str::to_string), + notes: Vec::new(), + }; + let pages = vec![ + page(0, &[95.0, 40.0], None), + page(1, &[60.0, 60.5, 10.0], None), + page(2, &[59.0], Some("skipped pages never count")), + page(3, &[0.0], None), + ]; + let (count, pages_on) = low_confidence_pages(&pages, 60.0).unwrap(); + assert_eq!((count, pages_on), (4, vec![1, 2, 4]), "60 counts, 60.5 does not, the skipped page is left out"); + let all_high = vec![page(0, &[95.0, 90.5], None)]; + assert_eq!(low_confidence_pages(&all_high, 60.0), None, "no low-confidence words, no note"); + } + + /// The editable writer: refuses an empty payload, writes the words as visible text at + /// their boxes on pages of the source's displayed size, adds no scan image, and never + /// touches the open document. + #[test] + fn editable_writer_refuses_empty_and_writes_visible_words() { + let mut s = Session::new(); + let id = s.open("src.pdf", None, Arc::new(fixture(2)), None).unwrap(); + assert!( + s.editable_ocr_bytes(id, &[OcrPage { already_on_page: 0, page: 0, words: vec![], skipped: None, notes: Vec::new() }]).is_err(), + "zero words are refused" + ); + let info = s.get(id).unwrap().info.pages[0].clone(); + // A word in the source page's user space, high on the page (the fixture's pages are + // 200 x 300 pt, so the box stays inside them). + let w = PlacedWord { + text: "Editable".into(), + origin: [20.0, 200.0], + across: [120.0, 0.0], + up: [0.0, 12.0], + confidence: Some(88.0), + source: "ocrs".into(), + }; + let found = vec![ + OcrPage { already_on_page: 0, page: 0, words: vec![w], skipped: None, notes: Vec::new() }, + OcrPage { already_on_page: 0, page: 1, words: vec![], skipped: None, notes: Vec::new() }, + ]; + let before = s.get(id).unwrap().bytes.clone(); + let bytes = s.editable_ocr_bytes(id, &found).unwrap(); + // The open document was never touched. + assert!(Arc::ptr_eq(&s.get(id).unwrap().bytes, &before), "the source bytes are the same allocation"); + let mut out = Session::new(); + let oid = out.open("ocr.pdf", None, bytes, None).unwrap(); + let otext = crate::tests::page_texts(&out, oid)[0].clone().to_lowercase(); + assert!(otext.contains("editable"), "{otext}"); + let odoc = out.get(oid).unwrap(); + assert_eq!(odoc.info.pages.len(), 2, "the blank source page stays a page"); + assert!(crate::tests::page_texts(&out, oid)[1].trim().is_empty(), "the page with no words stays blank"); + assert!(odoc.page_images(0).is_empty(), "no scan image is carried over"); + let (w, h) = (odoc.info.pages[0].width, odoc.info.pages[0].height); + assert_eq!((w, h), (info.width, info.height), "pages keep the source's displayed size"); + } + + /// The editable writer maps through a rotated source page: a word drawn in user space + /// reads upright on the new page, and the new page has the source's displayed size. + #[test] + fn editable_writer_maps_rotated_pages() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let text = s.create_from_text("t", "rotated source").unwrap(); + let id = s.open("r.pdf", None, text, None).unwrap(); + s.apply(id, Edit::RotatePages { pages: vec![0], degrees: 90 }).unwrap(); + let info = s.get(id).unwrap().info.pages[0].clone(); + assert_eq!(info.rotation, 90); + // The word sits at view (20, 48)..(120, 60) on the displayed page; its box comes in + // user space, exactly as recognition left it. + let (o, br, tl) = (info.view_to_user(20.0, 60.0), info.view_to_user(120.0, 60.0), info.view_to_user(20.0, 48.0)); + let w = PlacedWord { + text: "Word".into(), + origin: [f64::from(o[0]), f64::from(o[1])], + across: [f64::from(br[0] - o[0]), f64::from(br[1] - o[1])], + up: [f64::from(tl[0] - o[0]), f64::from(tl[1] - o[1])], + confidence: None, + source: "ocrs".into(), + }; + let bytes = s.editable_ocr_bytes(id, &[OcrPage { already_on_page: 0, page: 0, words: vec![w], skipped: None, notes: Vec::new() }]).unwrap(); + let mut out = Session::new(); + let oid = out.open("ocr.pdf", None, bytes, None).unwrap(); + let otext = crate::tests::page_texts(&out, oid)[0].clone(); + assert!(otext.contains("Word"), "{otext}"); + // The new page is the displayed size (portrait, not the rotated-away landscape). + let oinfo = out.get(oid).unwrap().info.pages[0].clone(); + assert_eq!((oinfo.width, oinfo.height), (info.width, info.height)); + } + + /// The searchable layer round-trips the recognized words through the engine's own + /// extraction, characters beyond WinAnsi included, without needing any display fonts. + #[test] + fn searchable_layer_round_trips_non_win_ansi_words() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let word = PlacedWord { + text: "日本語".into(), + origin: [20.0, 250.0], + across: [80.0, 0.0], + up: [0.0, 10.0], + confidence: None, + source: "ocrs".into(), + }; + s.apply(id, Edit::AddOcrText { page: 0, words: vec![word] }).unwrap(); + let text = crate::tests::page_texts(&s, id)[0].clone(); + assert!(text.contains("日本語"), "{text}"); + } + + /// The fontless searchable layer retains its original pixels, handles multiple code + /// fonts and supplementary Unicode, and does not replace existing resource names. + #[test] + fn searchable_unicode_preserves_pixels_save_undo_and_resources() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let before = crate::export::Exporter::new(s.get(id).unwrap()).png(0, 72.0).unwrap(); + let mut lines = vec!["中文 English 2026".to_string()]; + let chars: Vec = (0x4e00..0x4f10).filter_map(char::from_u32).chain(['𠮷']).collect(); + lines.extend(chars.chunks(90).map(|c| c.iter().collect())); + let text = lines.join("\n"); + let words: Vec<_> = lines + .iter() + .enumerate() + .map(|(i, line)| PlacedWord { + text: line.clone(), + origin: [20.0, 80.0 - 10.0 * i as f64], + across: [160.0, 0.0], + up: [0.0, 5.0], + confidence: None, + source: "test".into(), + }) + .collect(); + s.apply(id, Edit::AddOcrText { page: 0, words: words.clone() }).unwrap(); + assert!(crate::tests::page_texts(&s, id)[0].contains(&text)); + assert_eq!(crate::export::Exporter::new(s.get(id).unwrap()).png(0, 72.0).unwrap(), before); + let saved = s.save_bytes(id).unwrap(); + let reopened = s.open("searchable.pdf", None, saved, None).unwrap(); + assert!(crate::tests::page_texts(&s, reopened)[0].contains(&text)); + assert_eq!(crate::export::Exporter::new(s.get(reopened).unwrap()).png(0, 72.0).unwrap(), before); + s.undo(id).unwrap(); + assert!(!crate::tests::page_texts(&s, id)[0].contains(&text)); + s.redo(id).unwrap(); + let mut second = words[0].clone(); + second.text = "另一行".into(); + second.origin[1] = 100.0; + s.apply(id, Edit::AddOcrText { page: 0, words: vec![second] }).unwrap(); + let extracted = crate::tests::page_texts(&s, id)[0].clone(); + assert!(extracted.contains(&text) && extracted.contains("另一行"), "{extracted}"); + } + + #[test] + fn searchable_unicode_rejects_excessive_or_non_finite_input_atomically() { + let mut s = Session::new(); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let word = + PlacedWord { text: "日".into(), origin: [20.0, 80.0], across: [30.0, 0.0], up: [0.0, 5.0], confidence: None, source: "test".into() }; + let before = crate::tests::page_texts(&s, id); + let mut non_finite = word.clone(); + non_finite.origin[0] = f64::NAN; + let mut overflow = word.clone(); + overflow.origin[0] = f64::MAX; + overflow.across[0] = f64::MAX; + let mut too_long = word.clone(); + too_long.text = "a".repeat(4 * 1024 * 1024 + 1); + for words in [vec![non_finite], vec![overflow], vec![too_long], vec![word; 65537]] { + assert!(s.apply(id, Edit::AddOcrText { page: 0, words }).is_err()); + assert_eq!(crate::tests::page_texts(&s, id), before); + } + } + + #[test] + fn editable_export_refuses_invalid_or_excessive_aggregate_input_without_changing_source() { + let mut s = Session::new(); + let id = s.open("src.pdf", None, Arc::new(fixture(2)), None).unwrap(); + let before = s.save_bytes(id).unwrap(); + let word = + PlacedWord { text: "word".into(), origin: [20.0, 80.0], across: [30.0, 0.0], up: [0.0, 5.0], confidence: None, source: "test".into() }; + let page = |page, words| OcrPage { already_on_page: 0, page, words, skipped: None, notes: Vec::new() }; + let mut invalid = word.clone(); + invalid.origin[0] = f64::INFINITY; + let mut long = word.clone(); + long.text = "a".repeat(2 * 1024 * 1024 + 1); + for found in [ + vec![page(0, vec![invalid])], + vec![page(0, vec![long.clone()]), page(1, vec![long])], + vec![page(0, vec![word.clone(); 32768]), page(1, vec![word; 32769])], + ] { + assert!(s.editable_ocr_bytes(id, &found).is_err()); + assert_eq!(s.save_bytes(id).unwrap(), before); + } + } + + /// The editable output carries non-WinAnsi words as visible text that extraction reads + /// back, in a face beyond Helvetica when the build has one. + #[test] + fn editable_output_round_trips_non_win_ansi_words() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let w = PlacedWord { + text: "Привет".into(), + origin: [20.0, 250.0], + across: [80.0, 0.0], + up: [0.0, 10.0], + confidence: None, + source: "ocrs".into(), + }; + let bytes = s.editable_ocr_bytes(id, &[OcrPage { already_on_page: 0, page: 0, words: vec![w], skipped: None, notes: Vec::new() }]).unwrap(); + let mut out = Session::new(); + let oid = out.open("ocr.pdf", None, bytes, None).unwrap(); + let text = crate::tests::page_texts(&out, oid)[0].clone(); + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + assert!(text.contains("??????"), "{text}"); + } else { + assert!(text.contains("Привет"), "{text}"); + } + } + + /// The editable output's VISIBLE layer — a Type3 run for the Cyrillic word, Helvetica for + /// the Latin one — is redacted end to end: a mark over the Cyrillic word removes exactly + /// it (the renderer's extractor no longer reads it, before or after a save and reopen), and + /// the Latin word elsewhere on the page stays. + #[test] + fn the_editable_type3_layer_is_redacted_end_to_end() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let word = |text: &str, y: f64, w: f64| PlacedWord { + text: text.into(), + origin: [20.0, y], + across: [w, 0.0], + up: [0.0, 10.0], + confidence: None, + source: "ocrs".into(), + }; + let words = vec![word("Привет", 250.0, 80.0), word("Hello", 100.0, 50.0)]; + let bytes = s.editable_ocr_bytes(id, &[OcrPage { already_on_page: 0, page: 0, words, skipped: None, notes: Vec::new() }]).unwrap(); + let mut out = Session::new().with_clock(|| 1_700_000_000); + let oid = out.open("ocr.pdf", None, bytes, None).unwrap(); + let before = crate::tests::page_texts(&out, oid)[0].clone(); + assert!(before.contains("Привет") && before.contains("Hello"), "{before}"); + let shape = crate::Shape::Redact { + quads: vec![pdfcraft_annot::rect_quad([10.0, 240.0, 110.0, 270.0])], + overlay: String::new(), + look: Default::default(), + }; + let mark = Edit::AddAnnotation(crate::NewAnnotation { + page: 0, + style: crate::Style::default_for(&shape), + shape, + contents: String::new(), + author: "Ada".into(), + }); + out.apply(oid, mark).unwrap(); + out.apply(oid, Edit::ApplyRedactions { pages: None }).unwrap(); + let after = crate::tests::page_texts(&out, oid)[0].clone(); + assert!(!after.contains("Привет") && !after.contains("Пр"), "the Type3 word is gone: {after}"); + assert!(after.contains("Hello"), "the word outside the mark stays: {after}"); + // Reopened from the saved bytes, an independent read finds the same. + let saved = out.save_bytes(oid).unwrap(); + let mut again = Session::new(); + let aid = again.open("r.pdf", None, saved, None).unwrap(); + let reread = crate::tests::page_texts(&again, aid)[0].clone(); + assert!(!reread.contains("Привет") && reread.contains("Hello"), "{reread}"); + } + + /// A shared document plan serves a multi-page editable export: each page's words must be + /// written with their OWN encodings (the plan's offset for that page), so page 2 never + /// shows page 1's text. + #[test] + fn editable_output_keeps_each_pages_own_words() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(2)), None).unwrap(); + let word = |text: &str| PlacedWord { + text: text.into(), + origin: [20.0, 250.0], + across: [80.0, 0.0], + up: [0.0, 10.0], + confidence: None, + source: "ocrs".into(), + }; + // Different scripts, so each page's words live in different planned fonts (and page 2 + // would inherit page 1's codes under page-local indexing into the shared plan). + let found = vec![ + OcrPage { already_on_page: 0, page: 0, words: vec![word("日本語")], skipped: None, notes: Vec::new() }, + OcrPage { already_on_page: 0, page: 1, words: vec![word("Привет")], skipped: None, notes: Vec::new() }, + ]; + let bytes = s.editable_ocr_bytes(id, &found).unwrap(); + let mut out = Session::new(); + let oid = out.open("ocr.pdf", None, bytes, None).unwrap(); + let texts = crate::tests::page_texts(&out, oid); + assert!(texts[0].contains("日本語") && !texts[0].contains("Привет"), "{texts:?}"); + assert!(texts[1].contains("Привет") && !texts[1].contains("日本語"), "{texts:?}"); + } + + /// Redacting a recognized page takes the Unicode searchable layer away for good: the + /// apply's own verification passes over the Type3 font and an independent extractor finds + /// nothing, including in builds with no display fonts. + #[test] + fn redaction_removes_the_unicode_searchable_layer() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("src.pdf", None, Arc::new(fixture(1)), None).unwrap(); + let word = PlacedWord { + text: "日本語のテキスト".into(), + origin: [20.0, 250.0], + across: [100.0, 0.0], + up: [0.0, 10.0], + confidence: None, + source: "ocrs".into(), + }; + s.apply(id, Edit::AddOcrText { page: 0, words: vec![word] }).unwrap(); + assert!(crate::tests::page_texts(&s, id)[0].contains("日本語")); + let shape = pdfcraft_annot::Shape::Redact { + quads: vec![pdfcraft_annot::rect_quad([18.0, 248.0, 122.0, 262.0])], + overlay: String::new(), + look: Default::default(), + }; + let mark = Edit::AddAnnotation(pdfcraft_annot::NewAnnotation { + page: 0, + style: pdfcraft_annot::Style::default_for(&shape), + shape, + contents: String::new(), + author: "Ada".into(), + }); + s.apply(id, mark).unwrap(); + s.apply(id, Edit::ApplyRedactions { pages: None }).unwrap(); + assert!(!crate::tests::page_texts(&s, id)[0].contains("日本語"), "the layer is gone, not just hidden"); + } + + /// The job's render scale is what the run actually renders at: the chosen dpi, or the + /// page image's own resolution when that is lower. + #[test] + fn render_scale_matches_the_job_resolution() { + let mut s = Session::new(); + let text = s.create_from_text("t", "scale probe").unwrap(); + let id = s.open("t.pdf", None, text, None).unwrap(); + let job = s.ocr_job(id, &[0], OcrSettings { dpi: 144.0, ..Default::default() }).unwrap(); + assert!((job.render_scale(0) - 2.0).abs() < 1e-4, "144 dpi = 2 px/pt, got {}", job.render_scale(0)); + let job = s.ocr_job(id, &[0], OcrSettings { dpi: 600.0, ..Default::default() }).unwrap(); + assert!((job.render_scale(0) - 600.0 / 72.0).abs() < 1e-4, "a text page has no image, so the wanted dpi wins"); + let page = job.pages[0]; + assert_eq!(page, 0); + } + + /// #1848: an A4 scan at 300 dpi whose page box is its pixel size (2480 x 3508 pt, one pixel + /// per point) gets a smaller bitmap for each lower "Downsample to" choice, and a low-resolution + /// letter page is still never shrunk below one pixel per point. + #[test] + fn downsample_shrinks_a_page_box_given_in_pixels() { + let mut s = Session::new(); + let page = |w: u32, h: u32| export::encode_png(w, h, &vec![255; (w * h * 4) as usize]).unwrap(); + let bytes = s.create_from_images(&[("a4.png".into(), page(2480, 3508)), ("letter.png".into(), page(612, 792))]).unwrap(); + let id = s.open("scan.pdf", None, bytes, None).unwrap(); + let mut r = PageRenderer::new(s.get(id).unwrap().bytes.clone(), RenderConfig::default()); + for (dpi, a4) in [(72.0, [595, 842]), (150.0, [1240, 1754]), (300.0, [2480, 3508]), (600.0, [2480, 3508])] { + let job = s.ocr_job(id, &[], OcrSettings { dpi, ..Default::default() }).unwrap(); + for (page, want) in [(0, a4), (1, [612, 792])] { + let scale = job.render_scale(page); + let shot = r.render(RenderRequest { page, scale, ..Default::default() }); + assert!(shot.error.is_none(), "{:?}", shot.error); + let near = |got: u32, want: u32| got.abs_diff(want) <= 1; + assert!(near(shot.width, want[0]) && near(shot.height, want[1]), "{dpi} dpi, page {page}: {}x{}", shot.width, shot.height); + } + } + } } diff --git a/crates/engine/src/ocr/dictionary.rs b/crates/engine/src/ocr/dictionary.rs new file mode 100644 index 0000000000..cbefa9f7b5 --- /dev/null +++ b/crates/engine/src/ocr/dictionary.rs @@ -0,0 +1,80 @@ +//! The UI dictionary format: one case-insensitive word per UTF-8 line. +use std::collections::BTreeSet; + +pub const MAX_BYTES: usize = 4 * 1024 * 1024; +const MAX_WORDS: usize = 65536; + +fn word(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() || value.len() > 1024 || value.chars().any(char::is_control) { + return Err("dictionary words must be nonempty single lines of at most 1024 bytes".into()); + } + let normalized = value.to_lowercase(); + if normalized.len() > 1024 { + return Err("normalized dictionary words must be at most 1024 bytes".into()); + } + Ok(normalized) +} + +pub fn words(text: &str) -> Result, String> { + if text.len() > MAX_BYTES { + return Err("OCR dictionary exceeds 4 MiB".into()); + } + let mut words = BTreeSet::new(); + for line in text.lines().filter(|line| !line.trim().is_empty()) { + words.insert(word(line)?); + if words.len() > MAX_WORDS { + return Err("OCR dictionary exceeds 65536 words".into()); + } + } + Ok(words) +} + +/// Normalize, update and bound the complete dictionary before the caller writes it. +pub fn update(text: &str, add: &[&str], remove: &[&str]) -> Result { + if add.len() > MAX_WORDS || remove.len() > MAX_WORDS { + return Err("OCR dictionary exceeds 65536 words".into()); + } + let mut entries = words(text)?; + for value in remove { + entries.remove(&word(value)?); + } + for value in add { + entries.insert(word(value)?); + if entries.len() > MAX_WORDS { + return Err("OCR dictionary exceeds 65536 words".into()); + } + } + let mut out = String::new(); + for entry in entries { + if out.len().checked_add(entry.len()).and_then(|n| n.checked_add(1)).is_none_or(|n| n > MAX_BYTES) { + return Err("OCR dictionary exceeds 4 MiB".into()); + } + out.push_str(&entry); + out.push('\n'); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dictionary_updates_round_trip_and_refuse_expanding_normalization() { + let text = update("Quartz\n日本語\n", &["QUARTZ", "extra"], &["日本語"]).unwrap(); + assert_eq!(text, "extra\nquartz\n"); + assert_eq!(words(&text).unwrap(), BTreeSet::from(["extra".into(), "quartz".into()])); + assert!(update("original\n", &[&"İ".repeat(512)], &[]).unwrap_err().contains("normalized")); + assert!(update("original\n", &["two\nlines"], &[]).is_err()); + } + + #[test] + fn dictionary_limits_cover_existing_and_added_entries() { + let full: String = (0..65536).map(|i| format!("word{i}\n")).collect(); + assert_eq!(words(&full).unwrap().len(), 65536); + assert!(update(&full, &["extra"], &[]).is_err()); + assert_eq!(update(&full, &["extra"], &["word0"]).unwrap().lines().count(), 65536); + assert!(words(&"a".repeat(MAX_BYTES + 1)).is_err()); + } +} diff --git a/crates/engine/src/ocr/layer.rs b/crates/engine/src/ocr/layer.rs new file mode 100644 index 0000000000..ec2bac5fa8 --- /dev/null +++ b/crates/engine/src/ocr/layer.rs @@ -0,0 +1,145 @@ +//! Unicode search text has no visible glyphs. A generated Type 3 font supplies advance widths +//! and a ToUnicode map, with empty character programs; no display font is needed or embedded. +//! Adapted from the searchable-layer implementation in #433 (32e09fbe). + +use std::collections::BTreeMap; + +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_edit::EditError; +use pdfcraft_ocr::PlacedWord; + +/// Refuse oversized text or invalid geometry before cloning words or allocating fonts. +/// These limits apply to the complete input, including all editable export pages. +pub(crate) fn validate_words<'a>(words: impl IntoIterator) -> Result { + let mut count = 0usize; + let mut length = 0usize; + for w in words { + count += 1; + length = length.checked_add(w.text.len()).ok_or_else(|| EditError::Invalid("OCR text layer is too large".into()))?; + if count > 65536 || length > 4 * 1024 * 1024 { + return Err(EditError::Invalid("OCR text layer is too large".into())); + } + if !w.origin.iter().chain(&w.across).chain(&w.up).all(|v| v.is_finite()) + || !w.across[0].hypot(w.across[1]).is_finite() + || !w.up[0].hypot(w.up[1]).is_finite() + || (0..2).any(|i| !(w.origin[i] + 0.2 * w.up[i]).is_finite() || !(w.origin[i] + w.across[i] + 0.2 * w.up[i]).is_finite()) + { + return Err(EditError::Invalid("OCR text coordinates must be finite".into())); + } + } + Ok(count) +} + +pub(crate) fn add_text(doc: &mut Document, page: usize, words: &[PlacedWord]) -> Result<(), EditError> { + validate_words(words)?; + // Keep the existing Latin layout, including Helvetica's proportional advance widths. + if words.iter().all(|w| { + w.text + .chars() + .all(|c| matches!(c, '\u{20}'..='\u{7e}' | '\u{a0}'..='\u{ff}' | '€' | '‚' | '„' | '…' | '‘' | '’' | '“' | '”' | '•' | '–' | '—' | '™')) + }) { + pdfcraft_edit::stamp(doc, page, "OCR", pdfcraft_ocr::text_layer(words, &pdfcraft_fonts::TextPlan::default()))?; + return Ok(()); + } + let p = pdfcraft_model::pages(doc).get(page).cloned().ok_or_else(|| EditError::Invalid("OCR page does not exist".into()))?; + let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let mut fonts = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); + let chars: Vec = words.iter().flat_map(|w| w.text.chars()).collect::>().into_iter().collect(); + // Match the extractor's em box (-200..800). Baselines below are offset by 0.2 em, + // so search/selection rectangles span the detected image box. + let blank = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"1000 0 0 -200 1000 800 d1\n".to_vec()))); + let mut codes = BTreeMap::new(); + let mut names = Vec::new(); + for (index, chunk) in chars.chunks(255).enumerate() { + let mut name = format!("PCOcr{index}"); + while fonts.contains(name.as_bytes()) { + name.push('x'); + } + let mut differences = vec![Object::Int(1)]; + let mut procs = Dict::new(); + let mut cmap = String::from( + "/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CIDSystemInfo << /Registry (PdfCraft) /Ordering (Unicode) /Supplement 0 >> def\n/CMapName /PdfCraftOCR def\n/CMapType 2 def\n1 begincodespacerange\n<01> \nendcodespacerange\n", + ); + for (batch, part) in chunk.chunks(100).enumerate() { + cmap.push_str(&format!("{} beginbfchar\n", part.len())); + for (i, ch) in part.iter().enumerate() { + let code = (batch * 100 + i + 1) as u8; + let glyph = format!("g{code:02X}"); + differences.push(Object::name(&glyph)); + procs.set(glyph.into_bytes(), Object::Ref(blank)); + let mut units = [0u16; 2]; + let hex: String = ch.encode_utf16(&mut units).iter().map(|u| format!("{u:04X}")).collect(); + cmap.push_str(&format!("<{code:02X}> <{hex}>\n")); + codes.insert(*ch, (index, code)); + } + cmap.push_str("endbfchar\n"); + } + cmap.push_str("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"); + let map = doc.add(Object::Stream(Stream::flate(Dict::new(), cmap.as_bytes()))); + let mut encoding = Dict::new(); + encoding.set(b"Type".to_vec(), Object::name("Encoding")); + encoding.set(b"Differences".to_vec(), Object::Array(differences)); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type3")); + font.set(b"FontBBox".to_vec(), Object::Array(vec![0.into(), (-200).into(), 1000.into(), 800.into()])); + font.set(b"FontMatrix".to_vec(), Object::Array(vec![Object::Real(0.001), 0.into(), 0.into(), Object::Real(0.001), 0.into(), 0.into()])); + font.set(b"FirstChar".to_vec(), Object::Int(1)); + font.set(b"LastChar".to_vec(), Object::Int(chunk.len() as i64)); + font.set(b"Widths".to_vec(), Object::Array(vec![Object::Int(1000); chunk.len()])); + font.set(b"Encoding".to_vec(), Object::Dict(encoding)); + font.set(b"CharProcs".to_vec(), Object::Dict(procs)); + font.set(b"Resources".to_vec(), Object::Dict(Dict::new())); + font.set(b"ToUnicode".to_vec(), Object::Ref(map)); + fonts.set(name.as_bytes().to_vec(), Object::Ref(doc.add(Object::Dict(font)))); + names.push(name); + } + res.set(b"Font".to_vec(), Object::Dict(fonts)); + doc.update_dict(p.obj, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + // `3 Tr` and the font are graphics state, which `ET` doesn't reset: like + // `pdfcraft_ocr::text_layer` (#3029), the text is drawn inside `q … Q` so the invisible + // rendering mode cannot leak into later content streams on the page. + let mut content = String::from("/OCR BMC\nq\nBT\n3 Tr\n"); + for w in words { + let count = w.text.chars().count(); + if count == 0 || w.up[0].hypot(w.up[1]) <= 0.0 || w.across[0].hypot(w.across[1]) <= 0.0 { + continue; + } + let mut start = 0; + let mut run = Vec::new(); + let mut font_index = None; + let emit = |content: &mut String, font: usize, start: usize, run: &[u8]| { + let n = count as f64; + let at = start as f64 / n; + let hex: String = run.iter().map(|b| format!("{b:02X}")).collect(); + content.push_str(&format!( + "/{} 1 Tf\n{} {} {} {} {} {} Tm <{hex}> Tj\n", + names[font], + w.across[0] / n, + w.across[1] / n, + w.up[0], + w.up[1], + w.origin[0] + at * w.across[0] + 0.2 * w.up[0], + w.origin[1] + at * w.across[1] + 0.2 * w.up[1] + )); + }; + for (i, ch) in w.text.chars().enumerate() { + let Some(&(font, code)) = codes.get(&ch) else { continue }; + if let Some(previous) = font_index + && previous != font + { + emit(&mut content, previous, start, &run); + run.clear(); + start = i; + } + font_index = Some(font); + run.push(code); + } + if let Some(font) = font_index { + emit(&mut content, font, start, &run); + } + } + content.push_str("ET\nQ\nEMC\n"); + pdfcraft_edit::stamp(doc, page, "OCR", content.into_bytes())?; + Ok(()) +} diff --git a/crates/engine/src/open.rs b/crates/engine/src/open.rs index 32db07abee..80a5ad30c7 100644 --- a/crates/engine/src/open.rs +++ b/crates/engine/src/open.rs @@ -7,8 +7,8 @@ use pdfcraft_render::{DocInfo, OpenError, RenderConfig, RenderPool, inspect}; use crate::history::{Editor, Keys, TextBlocksCache}; use crate::{ - AppearanceCache, DocId, Document, Session, TrustStore, XFA_SCRIPTS_OFF, XfaLayout, display_bytes, display_config, guard, info_fields, js, - note_warnings, recovered_render_password, render_threads, same_pages, signatures_of, xfa, xfa_layout, xfa_values_from_datasets, + AppearanceCache, DocId, Document, Session, StatusCaptions, TrustStore, XFA_SCRIPTS_OFF, XfaLayout, display_bytes, display_config, guard, + info_fields, js, note_warnings, recovered_render_password, render_threads, same_pages, signatures_of, xfa, xfa_layout, xfa_values_from_datasets, }; /// Immutable session state needed while a document is prepared away from its owning thread. @@ -17,6 +17,7 @@ pub struct OpenContext { now: Option, js_off: bool, trust: Arc, + captions: Arc, xfa_time_scale: u32, } @@ -54,7 +55,13 @@ impl OpenContext { impl Session { /// Snapshot the session state that affects open-time scripts, dates and signature trust. pub fn open_context(&self) -> OpenContext { - OpenContext { now: self.now(), js_off: self.js_off, trust: self.trust.clone(), xfa_time_scale: self.xfa_time_scale.unwrap_or(1) } + OpenContext { + now: self.now(), + js_off: self.js_off, + trust: self.trust.clone(), + captions: self.captions.clone(), + xfa_time_scale: self.xfa_time_scale.unwrap_or(1), + } } /// Prepare a document without adding it to this session. @@ -74,6 +81,14 @@ impl Session { let id = DocId(self.next_id); let mut document = prepared.document; document.id = id; + // Trust or the interface language may have changed while it was prepared on another + // thread. + if !Arc::ptr_eq(&document.trust, &self.trust) { + document.captions = self.captions.clone(); + Self::revalidate(&mut document, self.trust.clone()); + } else if *document.captions != *self.captions { + Self::draw_captions(&mut document, self.captions.clone()); + } self.docs.push(document); id } @@ -253,7 +268,7 @@ impl Session { // XFA notes survive the document being re-read after edits. let xfa_warnings: Vec = if info.xfa.is_some() { info.warnings.clone() } else { Vec::new() }; let mut document = - Self::build_document(&context.trust, DocumentOpenData { name, path, bytes, info, structure: cos, render_password, password, xfa }); + Self::build_document(context, DocumentOpenData { name, path, bytes, info, structure: cos, render_password, password, xfa }); // A script that ran away on open turns the form's scripts off for this document. document.xfa_template = if ran_away { None } else { xfa_template }; document.js_output.append(script_output); @@ -266,7 +281,7 @@ impl Session { } /// The last step of opening: build the document record and register it. - fn build_document(trust: &Arc, data: DocumentOpenData<'_>) -> Document { + fn build_document(context: &OpenContext, data: DocumentOpenData<'_>) -> Document { let DocumentOpenData { name, path, bytes, mut info, structure: cos, render_password, password, xfa } = data; let config = RenderConfig { password: render_password.as_deref().map(Arc::from), ..Default::default() }; let (mut editor, read_only_reason) = match cos { @@ -293,7 +308,7 @@ impl Session { Err(_) => (None, Some("the document structure could not be read for editing".into())), }; let sig_cache = Arc::new(pdfcraft_sign::DigestCache::default()); - let signatures = editor.as_ref().map(|e| signatures_of(&e.cos, &bytes, trust, &sig_cache)).unwrap_or_default(); + let mut signatures = editor.as_ref().map(|e| signatures_of(&e.cos, &bytes, &context.trust, &sig_cache)).unwrap_or_default(); // The inspection reads only `/Info`; a file stating its title or author only in XMP still // shows them (#2373). if let Some(e) = &editor { @@ -303,7 +318,7 @@ impl Session { } } } - let display = display_bytes(editor.as_mut(), &bytes); + let display = display_bytes(editor.as_mut(), &bytes, &mut signatures, &context.captions); let renderer = RenderPool::new(display.clone(), render_threads(), display_config(&config)); let (form, adopted) = editor .as_ref() @@ -341,7 +356,8 @@ impl Session { marker_scan: OnceLock::new(), links: OnceLock::new(), signatures: Arc::new(signatures), - trust: trust.clone(), + trust: context.trust.clone(), + captions: context.captions.clone(), sig_cache, editor, config, diff --git a/crates/engine/src/pictures.rs b/crates/engine/src/pictures.rs index 49d76f07c0..d8b8835dc7 100644 --- a/crates/engine/src/pictures.rs +++ b/crates/engine/src/pictures.rs @@ -60,6 +60,18 @@ pub(crate) fn mark_source(doc: &mut pdfcraft_cos::Document, f: &MarkFile) -> Res } } +/// A 3D model's poster: the picture in `file` brought into `doc` and fitted inside `rect` +/// (user space), keeping its shape, and centred. It is drawn in user space and not counter-rotated +/// (unlike a stamp), so on a page with `/Rotate` it turns with the page, as the model does. +pub(crate) fn model_poster(doc: &mut pdfcraft_cos::Document, rect: [f64; 4], file: &MarkFile) -> Result { + let src = mark_source(doc, file)?; + let (sw, sh) = src.size; + if !(sw.is_finite() && sh.is_finite() && sw > 0.0 && sh > 0.0) { + return Err(EditError::Source(format!("{}: the poster has no area", file.name))); + } + Ok(pdfcraft_annot::PosterPicture { xobject: src.xobject, image: src.image, size: (sw, sh), at: fit_picture(rect, sw, sh) }) +} + /// The dimensions of an image from its bytes (JPEG, PNG, TIFF), without decoding the full raster. pub fn image_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { let format = image::guess_format(bytes).ok()?; diff --git a/crates/engine/src/rich_copy.rs b/crates/engine/src/rich_copy.rs new file mode 100644 index 0000000000..2e579adb19 --- /dev/null +++ b/crates/engine/src/rich_copy.rs @@ -0,0 +1,389 @@ +//! Copy with formatting: the selected glyphs of a page as paragraphs of styled spans and an +//! HTML fragment ([`pdfcraft_export::clip_html`]) for the clipboard, next to the plain text. +//! +//! Glyphs come from the text layer (view space); style comes from the page's reading blocks, +//! which never mix styles, mapped into view space. A glyph takes the style of the block its +//! centre lies in; a glyph in no block is copied unstyled. + +use pdfcraft_export::{ClipBuilder, ClipStyle, ClipText, clip_html}; +use pdfcraft_render::{PageInfo, PageText}; + +use crate::EditError; +use crate::compare::{display_unit, weight_and_slant}; + +/// Block tests spent matching glyphs to blocks for one copy; past it the rest is unstyled. +const MAX_MATCH_STEPS: usize = 20_000_000; +/// How far outside a block's box (points) a glyph centre still belongs to it. +const SLACK: f32 = 0.5; + +/// Selected text with its formatting. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RichCopy { + /// The paragraphs and spans. + pub text: ClipText, + /// The HTML fragment for the clipboard (empty when nothing was selected). + pub html: String, + /// Styles were left out: the document has no editor to read them, reading stopped at its + /// limits on this page, or matching glyphs to blocks ran out of steps. + pub styles_incomplete: bool, + /// A cap left part of the selection out of `text` and `html`; the plain text has all of it. + pub truncated: bool, +} + +struct StyledBlock { + rect: [f32; 4], + style: ClipStyle, +} + +impl crate::Document { + /// Selected `glyphs` of `page` (indices into `text`, in copy order) with their formatting. + /// Refused when the document does not allow copying. + pub fn rich_copy(&self, page: usize, text: &PageText, glyphs: &[usize]) -> Result { + self.rich_copy_within(page, text, glyphs, MAX_MATCH_STEPS) + } + + /// [`Self::rich_copy`] with `steps` block tests to match glyphs to blocks. + fn rich_copy_within(&self, page: usize, text: &PageText, glyphs: &[usize], mut steps: usize) -> Result { + self.check_copying()?; + let (blocks, mut incomplete) = self.styled_blocks(page); + let mut nested = vec![None; blocks.len()]; + let mut b = ClipBuilder::new(); + // The previous glyph's line, block and box, and whether it was a hyphen. + let mut prev: Option<(usize, Option, [f32; 4], bool)> = None; + for &i in glyphs { + let (Some(g), Some(&line)) = (text.glyphs.get(i), text.line_of.get(i)) else { continue }; + let hint = prev.and_then(|(_, b, _, _)| b).filter(|&k| !holds_smaller(&blocks, &mut nested, k, &mut steps)); + let found = block_at(&blocks, g.rect, hint, &mut steps).unwrap_or_else(|| { + incomplete = true; + None + }); + let line = usize::try_from(line).unwrap_or(usize::MAX); + if let Some((prev_line, prev_block, prev_rect, hyphen)) = prev { + if line != prev_line { + let style = |k: Option| k.and_then(|k| blocks.get(k)).map(|s| &s.style); + if found.is_none() && prev_block.is_none() { + b.line_break(); + } else if found == prev_block || wraps(style(prev_block), style(found), prev_rect, g.rect) { + // A paragraph wrapping: one space, none after a hyphen. + if !hyphen { + b.space(); + } + } else { + b.paragraph(); + } + } else if text.space_before.get(i).copied().unwrap_or(false) { + b.space(); + } + } + let style = found.and_then(|k| blocks.get(k)).map(|s| &s.style); + if !b.push(&g.text, style.unwrap_or(&ClipStyle::default())) { + break; + } + prev = Some((line, found, g.rect, g.text.ends_with('-'))); + } + Ok(finish(b, incomplete)) + } + + /// `page`'s reading blocks in view space with their style, and whether reading stopped at + /// its limits. + fn styled_blocks(&self, page: usize) -> (Vec, bool) { + let Some(info) = self.info.pages.get(page) else { return (Vec::new(), false) }; + let Some(editor) = self.editor.as_ref() else { return (Vec::new(), true) }; + let Ok(reading) = pdfcraft_edit::reading_blocks_checked(&editor.cos, page) else { return (Vec::new(), true) }; + let unit = display_unit(info); + let blocks = reading + .blocks + .into_iter() + .filter(|b| !b.text.trim().is_empty()) + .map(|b| { + let (bold, italic) = weight_and_slant(&b.base_font); + let style = ClipStyle { + families: pdfcraft_export::font_family(&b.base_font), + size: b.size * unit, + bold: b.bold || bold, + italic: b.italic || italic, + color: b.color, + }; + StyledBlock { rect: view_rect(info, b.rect), style } + }) + .collect(); + (blocks, reading.truncated) + } +} + +/// A user-space box as a view-space box (points from the displayed page's top-left, y down). +fn view_rect(info: &PageInfo, r: [f64; 4]) -> [f32; 4] { + let a = info.user_to_view(r[0] as f32, r[1] as f32); + let b = info.user_to_view(r[2] as f32, r[3] as f32); + [a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])] +} + +/// The copy `b` collected, with its HTML; `truncated` when a cap cut the text or the HTML. +fn finish(b: ClipBuilder, styles_incomplete: bool) -> RichCopy { + let text = b.finish(); + let (html, cut) = clip_html(&text); + RichCopy { html, truncated: text.truncated || cut, text, styles_incomplete } +} + +/// Whether the next line starts inside the same paragraph although its first glyph is in another +/// block: blocks split where the style changes, so a style change at a line's end is still a +/// wrap when the size stays and the lines sit no further apart than their own height allows. +/// Two blocks of the same style, a change of size, or a gap of over half a line are paragraphs. +fn wraps(prev: Option<&ClipStyle>, next: Option<&ClipStyle>, prev_rect: [f32; 4], rect: [f32; 4]) -> bool { + let (Some(prev), Some(next)) = (prev, next) else { return false }; + let diff = (prev.size - next.size).abs(); + if prev == next || diff.is_nan() || diff > 0.01 { + return false; + } + let height = (prev_rect[3] - prev_rect[1]).max(rect[3] - rect[1]); + // The next line starts just below the last one; a line far above it is another column. + let gap = rect[1] - prev_rect[3]; + height > 0.0 && gap.is_finite() && gap >= -height / 2.0 && gap <= height / 2.0 +} + +/// Whether a block smaller than block `k` overlaps it, so a glyph inside `k` may belong to that +/// one instead. Worked out once per block (`nested` keeps it), at a step per block tested; when +/// the steps run out the answer is yes, so the full search decides. +fn holds_smaller(blocks: &[StyledBlock], nested: &mut [Option], k: usize, steps: &mut usize) -> bool { + if let Some(Some(known)) = nested.get(k) { + return *known; + } + let Some(outer) = blocks.get(k) else { return true }; + let area = |r: &[f32; 4]| (r[2] - r[0]) * (r[3] - r[1]); + let overlaps = |r: &[f32; 4]| r[0] < outer.rect[2] && r[2] > outer.rect[0] && r[1] < outer.rect[3] && r[3] > outer.rect[1]; + let mut found = false; + for (j, b) in blocks.iter().enumerate() { + if *steps == 0 { + return true; + } + *steps -= 1; + if j != k && overlaps(&b.rect) && area(&b.rect) < area(&outer.rect) { + found = true; + break; + } + } + if let Some(slot) = nested.get_mut(k) { + *slot = Some(found); + } + found +} + +/// The block whose box holds the centre of `glyph` (view space): `hint` first (a block no +/// smaller one overlaps), then the smallest that holds it. Each block tested costs a step; +/// `None` when the steps ran out first. +fn block_at(blocks: &[StyledBlock], glyph: [f32; 4], hint: Option, steps: &mut usize) -> Option> { + let (x, y) = ((glyph[0] + glyph[2]) / 2.0, (glyph[1] + glyph[3]) / 2.0); + let holds = |r: &[f32; 4]| x >= r[0] - SLACK && x <= r[2] + SLACK && y >= r[1] - SLACK && y <= r[3] + SLACK; + let area = |r: &[f32; 4]| (r[2] - r[0]) * (r[3] - r[1]); + if let Some(k) = hint + && *steps > 0 + { + *steps -= 1; + if blocks.get(k).is_some_and(|b| holds(&b.rect)) { + return Some(Some(k)); + } + } + let mut best: Option<(usize, f32)> = None; + for (k, b) in blocks.iter().enumerate() { + if *steps == 0 { + return None; + } + *steps -= 1; + if holds(&b.rect) && best.is_none_or(|(_, a)| area(&b.rect) < a) { + best = Some((k, area(&b.rect))); + } + } + Some(best.map(|(k, _)| k)) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use pdfcraft_export::{ClipBuilder, ClipStyle}; + + use crate::Session; + + /// A heading in bold, a body paragraph of two lines with HTML-special characters, and a red + /// word on the body's first line. + fn styled_page() -> Vec { + let content = b"BT /F1 24 Tf 20 250 Td (Title) Tj ET \ + BT /F2 12 Tf 20 200 Td (Body text) Tj ET \ + BT 1 0 0 rg /F2 12 Tf 90 200 Td (red) Tj ET \ + BT /F2 12 Tf 20 186 Td (a Vec { + pdfcraft_testkit::Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 300] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> >>") + .stream("", content) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .build() + } + + fn page_text(s: &Session, id: crate::DocId) -> Arc { + let doc = s.get(id).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), Default::default()); + let request = pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; + r.render(request).text.unwrap() + } + + #[test] + fn copy_keeps_headings_body_and_colour_as_html() { + let mut s = Session::new(); + let id = s.open("styled.pdf", None, Arc::new(styled_page()), None).unwrap(); + let text = page_text(&s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + let copy = s.get(id).unwrap().rich_copy(0, &text, &all).unwrap(); + let html = ©.html; + assert!(!copy.styles_incomplete); + assert!( + html.starts_with("

Title

\n"), + "{html}" + ); + assert!(html.contains("font-size:12pt\">Body text "), "{html}"); + assert!(html.contains("font-size:12pt;color:#FF0000\">red"), "{html}"); + assert!(html.contains("a<b & c"), "{html}"); + assert!(!html.contains("= 2, "{html}"); + // The heading and the body are separate paragraphs; the red word shares the body's. + let body = copy.text.paragraphs.iter().find(|p| p.iter().any(|s| s.text.contains("Body"))).unwrap(); + assert!(body.iter().any(|s| s.text == "red" && s.style.color == [1.0, 0.0, 0.0]), "{body:?}"); + + // A part of the selection: only those glyphs. + let part = s.get(id).unwrap().rich_copy(0, &text, &all[..5]).unwrap(); + assert_eq!(part.text.paragraphs.len(), 1); + assert!(part.html.contains("Title"), "{}", part.html); + // Nothing selected, and indices past the text: an empty copy, not a panic. + let none = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap(); + assert!(none.html.is_empty() && none.text.paragraphs.is_empty()); + let past = s.get(id).unwrap().rich_copy(7, &text, &[usize::MAX, text.glyphs.len()]).unwrap(); + assert!(past.html.is_empty()); + } + + #[test] + fn running_out_of_match_steps_copies_the_rest_unstyled() { + let mut s = Session::new(); + let id = s.open("styled.pdf", None, Arc::new(styled_page()), None).unwrap(); + let text = page_text(&s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + let doc = s.get(id).unwrap(); + let full = doc.rich_copy(0, &text, &all).unwrap(); + // Enough for the first glyphs only: the text is all there, the rest has no style. + let short = doc.rich_copy_within(0, &text, &all, 8).unwrap(); + assert!(short.styles_incomplete); + let plain = |c: &super::RichCopy| c.text.paragraphs.iter().flatten().map(|s| s.text.clone()).collect::(); + assert_eq!(plain(&short).replace([' ', '\n'], ""), plain(&full).replace([' ', '\n'], "")); + let last = short.text.paragraphs.last().and_then(|p| p.last()).unwrap(); + assert_eq!(last.style, pdfcraft_export::ClipStyle::default(), "{last:?}"); + assert!(short.html.starts_with("

T"), "{}", short.html); + // No steps at all: nothing styled, still no panic. + let none = doc.rich_copy_within(0, &text, &all, 0).unwrap(); + assert!(none.styles_incomplete && !none.html.contains("font-size"), "{}", none.html); + // A page without blocks to match needs no steps. + let page_less = doc.rich_copy_within(5, &text, &all, 0).unwrap(); + assert!(!page_less.styles_incomplete); + } + + fn copy_all(s: &Session, id: crate::DocId) -> super::RichCopy { + let text = page_text(s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + s.get(id).unwrap().rich_copy(0, &text, &all).unwrap() + } + + #[test] + fn a_style_change_at_a_line_end_stays_in_its_paragraph() { + let content = b"BT /F2 12 Tf 20 200 Td (plain then) Tj ET \ + BT /F1 12 Tf 85 200 Td (bold) Tj ET \ + BT /F2 12 Tf 20 186 Td (continues here) Tj ET \ + BT /F2 12 Tf 20 120 Td (Another paragraph) Tj ET"; + let mut s = Session::new(); + let id = s.open("wrap.pdf", None, Arc::new(page_with(content)), None).unwrap(); + let copy = copy_all(&s, id); + let text = |p: &Vec| p.iter().map(|s| s.text.as_str()).collect::(); + let paragraphs: Vec = copy.text.paragraphs.iter().map(text).collect(); + assert!(paragraphs.iter().any(|p| p.contains("bold") && p.contains("continues")), "{paragraphs:?}"); + assert!(paragraphs.iter().any(|p| p.contains("Another") && !p.contains("continues")), "a wide gap is still a paragraph: {paragraphs:?}"); + } + + #[test] + fn a_style_change_into_the_next_column_starts_a_paragraph() { + let content = b"BT /F2 12 Tf 20 200 Td (left top) Tj ET \ + BT /F2 12 Tf 20 186 Td (left end) Tj ET \ + BT /F1 12 Tf 170 200 Td (Right head) Tj ET \ + BT /F2 12 Tf 170 186 Td (right body) Tj ET"; + let mut s = Session::new(); + let id = s.open("columns.pdf", None, Arc::new(page_with(content)), None).unwrap(); + let copy = copy_all(&s, id); + let text = |p: &Vec| p.iter().map(|s| s.text.as_str()).collect::(); + let paragraphs: Vec = copy.text.paragraphs.iter().map(text).collect(); + assert!(paragraphs.iter().any(|p| p.contains("Right")), "{paragraphs:?}"); + assert!(!paragraphs.iter().any(|p| p.contains("left end") && p.contains("Right")), "{paragraphs:?}"); + } + + #[test] + fn a_styled_word_inside_a_paragraph_keeps_its_style() { + let content = b"BT /F2 12 Tf 20 200 Td (first row of the text) Tj ET \ + BT /F2 12 Tf 20 186 Td (second) Tj ET \ + BT /F1 12 Tf 62 186 Td (bold) Tj ET \ + BT /F2 12 Tf 92 186 Td (row) Tj ET"; + let mut s = Session::new(); + let id = s.open("nested.pdf", None, Arc::new(page_with(content)), None).unwrap(); + let copy = copy_all(&s, id); + let spans: Vec<_> = copy.text.paragraphs.iter().flatten().collect(); + assert!(spans.iter().any(|s| s.text.contains("bold") && s.style.bold), "{spans:?}"); + assert!(spans.iter().any(|s| s.text.contains("second") && !s.style.bold), "{spans:?}"); + } + + #[test] + fn a_capped_copy_is_marked_truncated() { + let (plain, bold) = (ClipStyle::default(), ClipStyle { bold: true, ..ClipStyle::default() }); + let mut b = ClipBuilder::new(); + for k in 0..=pdfcraft_export::MAX_CLIP_SPANS { + b.push("x", if k % 2 == 0 { &plain } else { &bold }); + } + let copy = super::finish(b, false); + assert!(copy.truncated && copy.text.truncated); + let mut b = ClipBuilder::new(); + b.push("x", &plain); + assert!(!super::finish(b, false).truncated); + } + + #[test] + fn without_an_editor_the_styles_are_incomplete() { + let mut s = Session::new(); + let id = s.open("styled.pdf", None, Arc::new(styled_page()), None).unwrap(); + let text = page_text(&s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + s.doc_mut(id).unwrap().editor = None; + let copy = s.get(id).unwrap().rich_copy(0, &text, &all).unwrap(); + assert!(copy.styles_incomplete && !copy.html.contains("font-size"), "{}", copy.html); + assert!(!copy.truncated); + } + + #[test] + fn copy_with_formatting_is_refused_when_copying_is_withheld() { + let mut cos = pdfcraft_cos::Document::open(Arc::new(styled_page())).unwrap(); + cos.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0b0100, // print only + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(pdfcraft_cos::write_full(&cos, &Default::default()).unwrap()); + let mut s = Session::new(); + let id = s.open("locked.pdf", None, bytes, None).unwrap(); + let text = pdfcraft_render::PageText { glyphs: Vec::new(), line_of: Vec::new(), space_before: Vec::new() }; + let err = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap_err(); + assert!(matches!(err, crate::EditError::NotPermitted(_)), "{err:?}"); + } +} diff --git a/crates/engine/src/scan_clean.rs b/crates/engine/src/scan_clean.rs new file mode 100644 index 0000000000..cef7538309 --- /dev/null +++ b/crates/engine/src/scan_clean.rs @@ -0,0 +1,308 @@ +//! Scan & OCR ▸ Enhance ▸ Scanned document: clean up the pictures scanned pages are made of. +//! +//! Unlike Enhance ▸ Camera image, the page is not rendered and replaced: each scan image is +//! decoded, cleaned ([`crate::camera_clean::clean_samples`]) and written back **into the same +//! image object**, at the same size and in the same colour space. So everything else on the +//! page stays as it was: an invisible OCR text layer, comments, links, form fields, other +//! content, and the image's own soft mask, `/Interpolate` and other entries. A JPEG stays a JPEG +//! (at quality 90), any other grey or RGB image is stored losslessly (Flate), and a black and +//! white (1-bit) image stays 1-bit. An image several pages draw is cleaned once. +//! +//! A page is a scan when its pictures cover at least half of it (as for Recognize text); the +//! pictures each covering at least a tenth of the page are its scan. What is not cleaned is +//! reported, never skipped silently: pages that are no scan, and scan images in a form PdfCraft +//! can't rewrite yet (CMYK, indexed, JPEG 2000, JBIG2, CCITT fax, a stencil mask, a remapping +//! decode array, a scan drawn inside grouped Form artwork) or whose data is broken. +//! +//! Like Optimize, the work is a job: [`Session::scan_clean_job`] captures the document, +//! [`ScanCleanJob::run`] cleans anywhere (a worker thread), and [`Session::apply_scan_clean`] +//! makes the result one undoable step, "Enhance scanned file" (the name the Scan & OCR tool has). + +use pdfcraft_cos::{ObjRef, Object, Stream}; + +use crate::camera_clean::{CleanUp, MAX_LEVEL, clean_samples}; +use crate::edit::check_permission; +use crate::{DocId, Edit, EditError, Session, guard, optimize}; + +pub use crate::camera_clean::DESPECKLE_FROM; + +/// The JPEG quality a cleaned JPEG scan is written back at (as Enhance camera image). +const QUALITY: u8 = 90; +/// On a scanned page, an image covering at least this share of the page is part of the scan; +/// smaller ones (a logo or a signature pasted on) are left alone. +const PART_OF_SCAN: f64 = 0.1; +/// Why a scan drawn through grouped Form artwork stays as it is. +const IN_FORM: &str = "it is drawn inside grouped artwork (a form XObject), which can't be cleaned up yet"; + +/// How scanned pages are cleaned up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScanCleanSettings { + /// Background removal, 0 (off) … 100 (strongest), as Enhance camera image's clean-up level: + /// the paper's illumination is evened out (fully from 50), so a grey, yellowed or unevenly lit + /// background turns white; then the contrast is raised so faint strokes darken. Colour ink + /// stays colour. + pub level: u8, + /// Edge shadow removal: shadows along the scan's edges (a book's gutter, a copier's dark + /// border) are lightened to the paper next to them, line by line, before the level applies. + pub edge_shadows: bool, + /// Despeckle: isolated dark specks (dust, scanner noise), dark pixels with at most one dark + /// neighbour, turn white. The only change a black and white (1-bit) scan takes. + pub despeckle: bool, +} + +impl ScanCleanSettings { + /// The level `level` (capped at 100) with edge shadow removal (unless the level is 0), and + /// despeckling from [`DESPECKLE_FROM`] as the camera tool does. + pub fn at(level: u8) -> Self { + let level = level.min(MAX_LEVEL); + Self { level, edge_shadows: level > 0, despeckle: level >= DESPECKLE_FROM } + } + + /// Whether these settings change anything at all. + pub fn changes_something(self) -> bool { + self.level > 0 || self.edge_shadows || self.despeckle + } +} + +impl Default for ScanCleanSettings { + fn default() -> Self { + Self::at(50) + } +} + +/// What a clean-up did. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScanCleanReport { + /// Pages (0-based) whose scan was cleaned, in order. + pub pages: Vec, + /// Images rewritten. One drawn on several pages counts once. + pub images: usize, + /// Pages left alone because no picture covers most of them. + pub not_scanned: Vec, + /// Scan images left as they were, and why. + pub skipped: Vec, +} + +/// A scan image that was not cleaned. +#[derive(Clone, Debug, PartialEq)] +pub struct SkippedImage { + /// The pages (0-based) that draw it, among those asked for. + pub pages: Vec, + /// Its resource name on the first of them. + pub name: String, + /// Why, in a sentence that follows "skipped because". + pub reason: String, +} + +/// One image to clean: its object, its name and the pages that draw it. +struct Target { + object: ObjRef, + name: String, + pages: Vec, +} + +/// What cleaning scanned pages needs, independent of the session. +pub struct ScanCleanJob { + cos: pdfcraft_cos::Document, + targets: Vec, + settings: ScanCleanSettings, + /// The document's edit generation it was taken at: a result is for that state only. + generation: u64, + report: ScanCleanReport, +} + +/// The cleaned images, ready for [`Session::apply_scan_clean`], and the report. +pub struct ScanCleaned { + images: Vec<(ObjRef, Stream)>, + generation: u64, + pub report: ScanCleanReport, +} + +impl Session { + /// Capture what cleaning up the scans on `pages` (0-based; empty = all) of document `id` + /// needs. Errors when the document can't be changed (read-only, or its permissions forbid + /// changes) or a page doesn't exist. + pub fn scan_clean_job(&self, id: DocId, pages: &[usize], settings: ScanCleanSettings) -> Result { + let doc = self.get(id).ok_or(EditError::NoDocument)?; + if let Some(why) = &doc.read_only_reason { + return Err(EditError::ReadOnly(why.clone())); + } + let editor = doc.editor.as_ref().ok_or_else(|| EditError::ReadOnly(String::new()))?; + if let Some(p) = editor.cos.permissions() { + check_permission(&Edit::EnhanceScan { images: Vec::new() }, &p)?; + } + let all = doc.info.pages.len(); + let mut wanted: Vec = if pages.is_empty() { (0..all).collect() } else { pages.to_vec() }; + if let Some(p) = wanted.iter().find(|p| **p >= all) { + return Err(EditError::Invalid(format!("there is no page {}: the document has {all}", p.saturating_add(1)))); + } + wanted.sort_unstable(); + wanted.dedup(); + let mut report = ScanCleanReport::default(); + let mut targets: Vec = Vec::new(); + // Form artwork already reported, and where in `report.skipped`. + let mut forms_reported: Vec<(ObjRef, usize)> = Vec::new(); + for page in wanted { + let Some(info) = doc.info.pages.get(page) else { continue }; + // Ordinary images and scan-holding Forms are classified together, so a page that + // mixes them gets its images cleaned and its Forms reported. + let Some((images, forms, own_scan)) = crate::scan_fields::scan_parts(doc, page, info) else { + report.not_scanned.push(page); + continue; + }; + let forms: Vec<_> = forms.into_iter().filter(|(_, share)| *share >= PART_OF_SCAN).collect(); + if !own_scan && forms.is_empty() { + report.not_scanned.push(page); + continue; + } + // A scan drawn through a Form XObject is a scan all the same: say why it stays. + for (form, _) in forms { + let known = form.object.and_then(|o| forms_reported.iter().find(|(r, _)| *r == o)).and_then(|(_, k)| report.skipped.get_mut(*k)); + match known { + Some(s) if !s.pages.contains(&page) => s.pages.push(page), + Some(_) => {} + None => { + if let Some(o) = form.object { + forms_reported.push((o, report.skipped.len())); + } + report.skipped.push(SkippedImage { pages: vec![page], name: form.name, reason: IN_FORM.into() }); + } + } + } + for (image, _) in images.into_iter().filter(|(_, share)| *share >= PART_OF_SCAN) { + let Some(object) = image.object else { + report.skipped.push(SkippedImage { + pages: vec![page], + name: image.name, + reason: "it is stored inside the page's resources rather than as an object of its own".into(), + }); + continue; + }; + match targets.iter_mut().find(|t| t.object == object) { + Some(t) if !t.pages.contains(&page) => t.pages.push(page), + Some(_) => {} + None => targets.push(Target { object, name: image.name, pages: vec![page] }), + } + } + } + Ok(ScanCleanJob { cos: editor.cos.clone(), targets, settings, generation: doc.edit_generation(), report }) + } + + /// Write the images [`ScanCleanJob::run`] cleaned into document `id` as one undoable step, + /// "Enhance scanned file" (none when nothing was cleaned), and return the report. Refused + /// when the document changed since the job was taken. + pub fn apply_scan_clean(&mut self, id: DocId, cleaned: ScanCleaned) -> Result { + let doc = self.get(id).ok_or(EditError::NoDocument)?; + if doc.edit_generation() != cleaned.generation { + return Err(EditError::Invalid("the document changed while its scans were being cleaned up; nothing was changed, try again".into())); + } + if !cleaned.images.is_empty() { + self.apply(id, Edit::EnhanceScan { images: cleaned.images })?; + } + Ok(cleaned.report) + } + + /// Enhance ▸ Scanned document in one call: [`Self::scan_clean_job`], run, then + /// [`Self::apply_scan_clean`] (the CLI and agents use this). + pub fn enhance_scanned_pages(&mut self, id: DocId, pages: &[usize], settings: ScanCleanSettings) -> Result { + let job = self.scan_clean_job(id, pages, settings)?; + let cleaned = job.run(|_, _| true)?; + self.apply_scan_clean(id, cleaned) + } +} + +impl ScanCleanJob { + /// The images to clean (one drawn on several pages counts once). + pub fn total(&self) -> usize { + self.targets.len() + } + + /// Clean every scan image. `progress(done, total)` is called before each image; returning + /// `false` stops with [`EditError::Cancelled`]. A panic is returned as an error, so a worker + /// thread always reports back. + pub fn run(self, mut progress: impl FnMut(usize, usize) -> bool) -> Result { + guard(move || self.work(&mut progress)).map_err(|m| EditError::Invalid(format!("cleaning up the scans failed unexpectedly ({m})")))? + } + + fn work(self, progress: &mut dyn FnMut(usize, usize) -> bool) -> Result { + let ScanCleanJob { cos, targets, settings, generation, mut report } = self; + let total = targets.len(); + let mut images = Vec::new(); + let mut changed_pages: Vec = Vec::new(); + // One decoding allowance for the whole run, sized by the file. + let mut budget = pdfcraft_cos::ImageBudget::new(&cos); + for (done, target) in targets.into_iter().enumerate() { + if !progress(done, total) { + return Err(EditError::Cancelled); + } + let mut skip = |reason: String| report.skipped.push(SkippedImage { pages: target.pages.clone(), name: target.name.clone(), reason }); + match clean_image(&cos, target.object, settings, &mut budget) { + Ok(Some(stream)) => { + images.push((target.object, stream)); + changed_pages.extend(target.pages.iter().copied()); + } + Ok(None) => skip("it is already clean: nothing changes at these settings".into()), + Err(reason) => skip(reason), + } + } + changed_pages.sort_unstable(); + changed_pages.dedup(); + report.images = images.len(); + report.pages = changed_pages; + report.skipped.sort_by_key(|s| s.pages.first().copied()); + Ok(ScanCleaned { images, generation, report }) + } +} + +/// Image `object` cleaned with `settings`: the new stream, `None` when nothing changes, or why it +/// can't be cleaned. +fn clean_image( + cos: &pdfcraft_cos::Document, + object: ObjRef, + settings: ScanCleanSettings, + budget: &mut pdfcraft_cos::ImageBudget, +) -> Result, String> { + let obj = cos.get(object); + let Object::Stream(s) = &*obj else { return Err("it is not an image".into()) }; + if s.dict.name(b"Subtype") != Some(b"Image") { + return Err("it is not an image".into()); + } + // A colour-key mask names sample values to leave unpainted: cleaned samples would match others. + if matches!(s.dict.get(b"Mask").map(|m| cos.resolve(m)).as_deref(), Some(Object::Array(_))) { + return Err("it has a colour-key mask, which cleaning would change".into()); + } + let samples = optimize::decode_samples(cos, s, budget).map_err(|e| e.to_string())?; + let cleaned = if samples.bilevel { + if !settings.despeckle { + return Err("it is black and white, which only Despeckle changes, and Despeckle is off".into()); + } + clean_samples(&samples.data, 1, samples.width, samples.height, CleanUp { level: 0, edge_shadows: false, despeckle: true }) + } else { + if !settings.changes_something() { + return Ok(None); + } + let how = CleanUp { level: settings.level, edge_shadows: settings.edge_shadows, despeckle: settings.despeckle }; + clean_samples(&samples.data, samples.components, samples.width, samples.height, how) + } + .ok_or("its samples don't match its size")?; + if cleaned == samples.data { + return Ok(None); + } + let cleaned = optimize::Samples { data: cleaned, ..samples }; + optimize::encode_samples(s, &cleaned, QUALITY).map(Some).ok_or_else(|| "it could not be written back".into()) +} + +/// [`Edit::EnhanceScan`]: put the cleaned images in place. Each object must still be an image. +pub(crate) fn write_back(doc: &mut pdfcraft_cos::Document, images: &[(ObjRef, Stream)]) -> Result<(), EditError> { + for (object, stream) in images { + let is_image = matches!(&*doc.get(*object), Object::Stream(s) if s.dict.name(b"Subtype") == Some(b"Image")); + if !is_image { + return Err(EditError::Invalid(format!("object {} is no longer an image; the scan was not changed", object.num))); + } + doc.set(*object, Object::Stream(stream.clone())); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/scan_clean/tests.rs b/crates/engine/src/scan_clean/tests.rs new file mode 100644 index 0000000000..e58ab3e5ef --- /dev/null +++ b/crates/engine/src/scan_clean/tests.rs @@ -0,0 +1,463 @@ +use std::sync::Arc; + +use pdfcraft_testkit::Pdf; + +use super::*; + +const W: usize = 300; +const H: usize = 400; +/// Isolated specks on the paper, away from the text rows. +const SPECKS: [(usize, usize); 3] = [(150, 25), (200, 85), (120, 125)]; + +/// The light falling on the paper at `x`: full in the middle, darker towards a shadowed left +/// edge (as a book's gutter) and a little at the right edge. +fn light(x: usize) -> f32 { + let left = (x as f32 / 60.0).min(1.0); + let right = ((W - 1 - x) as f32 / 30.0).min(1.0); + (0.55 + 0.45 * left) * (0.8 + 0.2 * right) +} + +fn is_stroke(x: usize, y: usize) -> bool { + (60..360).contains(&y) && (y - 60) % 40 < 6 && (60..240).contains(&x) +} + +/// A synthetic scan, `n` components (1 grey, 3 RGB): greyish yellowed paper, a shadowed gutter, +/// dark text strokes and a few specks. +fn scan_pixels(n: usize) -> Vec { + let mut out = Vec::with_capacity(W * H * n); + for y in 0..H { + for x in 0..W { + let ink = if is_stroke(x, y) || SPECKS.contains(&(x, y)) { 0.12 } else { 0.8 }; + let v = ink * light(x); + let px = [v, v, v * 0.85]; + for c in px.iter().take(n) { + out.push((c * 255.0).round().clamp(0.0, 255.0) as u8); + } + } + } + out +} + +fn jpeg(px: &[u8], n: usize) -> Vec { + let mut out = Vec::new(); + let ty = if n == 1 { image::ExtendedColorType::L8 } else { image::ExtendedColorType::Rgb8 }; + image::ImageEncoder::write_image(image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 95), px, W as u32, H as u32, ty).unwrap(); + out +} + +/// Three pages: 1 a scan (the image `image_dict`/`data` as object 4) with an invisible text +/// layer and a comment, 2 a page of real text, 3 the same scan image drawn again. +fn scan_pdf(image_dict: &str, data: &[u8]) -> Vec { + let resources = "/Resources << /XObject << /Im0 4 0 R >> /Font << /F1 5 0 R >> >>"; + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 8 0 R 9 0 R] /Count 3 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 6 0 R /Annots [7 0 R] >>")) + .stream(&format!("/Type /XObject /Subtype /Image {image_dict}"), data) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .stream("", format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q BT 3 Tr /F1 12 Tf 20 380 Td (Invisible OCR words) Tj ET")) + .obj("<< /Type /Annot /Subtype /Square /Rect [10 10 50 50] /C [1 0 0] /Contents (A comment) >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 10 0 R >>")) + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 11 0 R >>")) + .stream("", "BT /F1 12 Tf 20 380 Td (Real text) Tj ET") + .stream("", format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q")) + .build() +} + +fn grey_scan() -> Vec { + scan_pdf(&format!("/Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8 /Interpolate true"), &scan_pixels(1)) +} + +fn open(bytes: Vec) -> (Session, DocId) { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("scan.pdf", None, Arc::new(bytes), None).unwrap(); + (s, id) +} + +fn object(s: &Session, id: DocId, num: u32) -> Arc { + s.get(id).unwrap().editor.as_ref().unwrap().cos.get(ObjRef::new(num, 0)) +} + +/// The samples of the image page 1 draws, now. +fn samples(s: &Session, id: DocId) -> optimize::Samples { + let doc = s.get(id).unwrap(); + let object = doc.page_images(0)[0].object.unwrap(); + let cos = &doc.editor.as_ref().unwrap().cos; + let Object::Stream(img) = &*cos.get(object) else { panic!("the image") }; + optimize::decode_samples(cos, img, &mut pdfcraft_cos::ImageBudget::with_work(u64::MAX)).unwrap() +} + +fn grey_at(px: &optimize::Samples, x: usize, y: usize) -> u8 { + px.data[(y * W + x) * px.components] +} + +#[test] +fn a_scan_gets_white_paper_even_edges_dark_strokes_and_no_specks_in_place() { + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + let kept: Vec> = [3, 5, 6, 7, 8, 10, 11].iter().map(|&n| object(&s, id, n)).collect(); + let text_before = s.get(id).unwrap().text_lines(0); + let original = s.get(id).unwrap().bytes.clone(); + + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.pages, vec![0, 2], "the shared image counts for both its pages"); + assert_eq!(report.images, 1); + assert_eq!(report.not_scanned, vec![1], "the text page is no scan"); + assert!(report.skipped.is_empty(), "{:?}", report.skipped); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Enhance scanned file")); + + let after = samples(&s, id); + assert_eq!((after.width, after.height, after.components, after.jpeg), (W as u32, H as u32, 1, false)); + // Paper, also in the shadowed gutter and at the right edge, turns (nearly) white. + for (x, y) in [(150, 40), (150, 380), (3, 40), (3, 380), (30, 200), (W - 2, 40)] { + assert!(grey_at(&before, x, y) < 210, "the fixture's paper is grey at ({x}, {y})"); + assert!(grey_at(&after, x, y) >= 235, "paper at ({x}, {y}): {} → {}", grey_at(&before, x, y), grey_at(&after, x, y)); + } + // The edge is evened out: the gutter is as white as the middle. + assert!(grey_at(&after, 150, 40).abs_diff(grey_at(&after, 3, 40)) <= 12); + // Strokes stay dark (darker, with the contrast), also in the gutter's light. + for (x, y) in [(150, 62), (61, 102), (238, 342)] { + assert!(grey_at(&after, x, y) <= 40, "stroke at ({x}, {y}): {}", grey_at(&after, x, y)); + } + for (x, y) in SPECKS { + assert!(grey_at(&before, x, y) < 60); + assert!(grey_at(&after, x, y) >= 200, "speck at ({x}, {y}): {}", grey_at(&after, x, y)); + } + // Everything but the image is as it was: the page, its invisible text layer, the comment, the + // font, the other pages. The image keeps its entries. + let now: Vec> = [3, 5, 6, 7, 8, 10, 11].iter().map(|&n| object(&s, id, n)).collect(); + assert_eq!(kept, now); + assert_eq!(s.get(id).unwrap().info.pages.len(), 3); + let text_after = s.get(id).unwrap().text_lines(0); + assert!(text_after.iter().any(|l| l.text.contains("Invisible OCR words")), "{text_after:?}"); + assert_eq!(text_before.len(), text_after.len()); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.get(b"Interpolate"), Some(&Object::Bool(true))); + assert_eq!(img.dict.name(b"ColorSpace"), Some(&b"DeviceGray"[..])); + + // Undo brings the original image, and the original file, back. + s.undo(id).unwrap(); + assert_eq!(samples(&s, id), before); + assert_eq!(s.get(id).unwrap().bytes, original); + assert_eq!(s.get(id).unwrap().can_undo(), None); +} + +#[test] +fn a_jpeg_scan_stays_a_jpeg_in_colour() { + let px = scan_pixels(3); + let (mut s, id) = open(scan_pdf(&format!("/Width {W} /Height {H} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode"), &jpeg(&px, 3))); + let report = s.enhance_scanned_pages(id, &[0], ScanCleanSettings::at(80)).unwrap(); + // Only page 1 was asked for; the image it shares with page 3 is changed for both. + assert_eq!((report.pages.as_slice(), report.images), (&[0][..], 1)); + let after = samples(&s, id); + assert!(after.jpeg && after.components == 3); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.name(b"Filter"), Some(&b"DCTDecode"[..])); + // Yellowed paper becomes white; the strokes stay dark. + let at = |x: usize, y: usize| after.data[(y * W + x) * 3..][..3].to_vec(); + for (x, y) in [(150, 40), (5, 300)] { + assert!(at(x, y).iter().all(|&v| v >= 230), "paper at ({x}, {y}): {:?}", at(x, y)); + } + assert!(at(150, 63).iter().all(|&v| v <= 60), "stroke: {:?}", at(150, 63)); + // The file still renders. + let doc = s.get(id).unwrap(); + let shot = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), doc.config.clone()).render(pdfcraft_render::RenderRequest { + page: 0, + scale: 1.0, + ..Default::default() + }); + assert!(shot.error.is_none(), "{:?}", shot.error); +} + +#[test] +fn despeckle_is_its_own_switch() { + let speck = |settings: ScanCleanSettings| { + let (mut s, id) = open(grey_scan()); + s.enhance_scanned_pages(id, &[0], settings).unwrap(); + let after = samples(&s, id); + SPECKS.iter().map(|&(x, y)| grey_at(&after, x, y)).max().unwrap_or(0) + }; + assert!(speck(ScanCleanSettings { despeckle: false, ..ScanCleanSettings::at(80) }) < 100, "kept without despeckle"); + assert!(speck(ScanCleanSettings::at(80)) >= 200); + // Despeckle alone, at level 0, changes nothing but the specks. + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + s.enhance_scanned_pages(id, &[], ScanCleanSettings { level: 0, edge_shadows: false, despeckle: true }).unwrap(); + let after = samples(&s, id); + let changed: Vec = before.data.iter().zip(&after.data).enumerate().filter(|(_, (a, b))| a != b).map(|(i, _)| i).collect(); + let mut specks: Vec = SPECKS.iter().map(|&(x, y)| y * W + x).collect(); + specks.sort_unstable(); + assert_eq!(changed, specks); + // Nothing to do: no undo step, and the report says why. + let (mut s, id) = open(grey_scan()); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings { level: 0, edge_shadows: false, despeckle: false }).unwrap(); + assert_eq!(report.images, 0); + assert!(report.pages.is_empty()); + assert_eq!(report.skipped.len(), 1); + assert!(report.skipped[0].reason.contains("already clean"), "{:?}", report.skipped); + assert_eq!(s.get(id).unwrap().can_undo(), None); + assert!(!s.get(id).unwrap().dirty); +} + +#[test] +fn a_black_and_white_scan_stays_one_bit_and_loses_its_specks() { + // 64 × 64, one bit per pixel, 1 = white: a black bar and two specks. + let (w, h) = (64usize, 64usize); + let mut bits = vec![0xFFu8; w / 8 * h]; + let mut black = |x: usize, y: usize| bits[y * w / 8 + x / 8] &= !(0x80 >> (x % 8)); + for y in 20..30 { + for x in 10..50 { + black(x, y); + } + } + black(5, 5); + black(58, 58); + let bytes = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 64 64] /Resources << /XObject << /Im0 4 0 R >> >> /Contents 5 0 R >>") + .stream("/Type /XObject /Subtype /Image /Width 64 /Height 64 /ColorSpace /DeviceGray /BitsPerComponent 1", &bits) + .stream("", "q 64 0 0 64 0 0 cm /Im0 Do Q") + .build(); + let (mut s, id) = open(bytes); + // Only Despeckle changes a black and white scan: without it the image is reported. + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings { despeckle: false, ..ScanCleanSettings::at(100) }).unwrap(); + assert_eq!(report.images, 0); + assert!(report.skipped.first().is_some_and(|k| k.reason.contains("black and white")), "{:?}", report.skipped); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.images, 1); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.int(b"BitsPerComponent"), Some(1)); + let px = samples(&s, id); + assert!(px.bilevel); + let at = |x: usize, y: usize| px.data[y * w + x]; + assert_eq!((at(5, 5), at(58, 58)), (255, 255), "specks gone"); + assert_eq!((at(10, 20), at(30, 25), at(49, 29)), (0, 0, 0), "the bar stays"); + assert_eq!(px.data.iter().filter(|&&v| v == 0).count(), 40 * 10); +} + +#[test] +fn hostile_images_are_reported_not_a_panic() { + // Each page draws one image over the whole page. + let images: [(&str, Vec); 6] = [ + ("/Width 100000 /Height 100000 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![0; 64]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /DCTDecode", vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 3, 9]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![200; 1000]), + ("/Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![128]), + ("/Width 8 /Height 8 /ColorSpace /DeviceCMYK /BitsPerComponent 8", vec![0; 256]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8 /Mask [250 255]", vec![200; 300 * 400]), + ]; + let n = images.len(); + let kids: String = (0..n).map(|i| format!("{} 0 R ", 3 + 3 * i)).collect(); + let mut pdf = Pdf::new().obj("<< /Type /Catalog /Pages 2 0 R >>").obj(format!("<< /Type /Pages /Kids [{kids}] /Count {n} >>")); + for (i, (dict, data)) in images.iter().enumerate() { + let (img, contents) = (4 + 3 * i, 5 + 3 * i); + pdf = pdf + .obj(format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /XObject << /Im0 {img} 0 R >> >> /Contents {contents} 0 R >>" + )) + .stream(&format!("/Type /XObject /Subtype /Image {dict}"), data) + .stream("", "q 300 0 0 400 0 0 cm /Im0 Do Q"); + } + let (mut s, id) = open(pdf.build()); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.images, 0); + assert_eq!(report.not_scanned, vec![3], "a 1 × 1 image is no scan"); + let why: Vec<(usize, &str)> = report.skipped.iter().map(|k| (k.pages[0], k.reason.as_str())).collect(); + assert_eq!(why.len(), 5, "{why:?}"); + assert!(why[0].0 == 0 && why[0].1.contains("80 million"), "{why:?}"); + assert!(why[1].0 == 1 && why[1].1.contains("could not be decoded"), "{why:?}"); + assert!(why[2].0 == 2 && why[2].1.contains("shorter"), "{why:?}"); + assert!(why[3].0 == 4 && why[3].1.contains("colour space"), "{why:?}"); + assert!(why[4].0 == 5 && why[4].1.contains("colour-key mask"), "{why:?}"); + assert_eq!(s.get(id).unwrap().can_undo(), None); + // Pages that don't exist are an error, not a panic. + assert!(matches!(s.enhance_scanned_pages(id, &[99], ScanCleanSettings::default()), Err(EditError::Invalid(_)))); +} + +#[test] +fn a_scan_drawn_inside_a_form_xobject_is_reported_not_called_no_scan() { + // Pages 1 and 3 draw the scan through Form Fm0 (as some scanners write it), page 2 is text. + let page = |contents: usize| { + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Fm0 4 0 R >> /Font << /F1 9 0 R >> >> /Contents {contents} 0 R >>" + ) + }; + let bytes = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 7 0 R 10 0 R] /Count 3 >>") + .obj(page(6)) + .stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 {W} {H}] /Resources << /XObject << /Im0 5 0 R >> >>"), + format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q"), + ) + .stream(&format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8"), scan_pixels(1)) + .stream("", "/Fm0 Do") + .obj(page(8)) + .stream("", "BT /F1 12 Tf 20 380 Td (Real text) Tj ET") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .obj(page(11)) + .stream("", "q /Fm0 Do Q") + .build(); + let (mut s, id) = open(bytes); + let before = object(&s, id, 5); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert!(report.pages.is_empty() && report.images == 0, "{report:?}"); + assert_eq!(report.not_scanned, vec![1], "only the text page is no scan"); + assert_eq!(report.skipped.len(), 1, "the form is reported once: {report:?}"); + let k = &report.skipped[0]; + assert_eq!((k.pages.as_slice(), k.name.as_str()), (&[0, 2][..], "Fm0")); + assert!(k.reason.contains("form XObject"), "{k:?}"); + assert_eq!(object(&s, id, 5), before, "the picture is left as it was"); + assert_eq!(s.get(id).unwrap().can_undo(), None); + + // A page of vector artwork in a Form, holding only a small logo, is no scan. + let logo = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Fm0 4 0 R >> >> /Contents 6 0 R >>")) + .stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 {W} {H}] /Resources << /XObject << /Im0 5 0 R >> >>"), + format!("0 0 {W} {H} re f q 40 0 0 20 10 370 cm /Im0 Do Q"), + ) + .stream("/Type /XObject /Subtype /Image /Width 40 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", [128; 800]) + .stream("", "/Fm0 Do") + .build(); + let (mut s, id) = open(logo); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!((report.not_scanned.as_slice(), report.skipped.len()), (&[0][..], 0), "{report:?}"); +} + +#[test] +fn a_page_mixing_its_own_image_and_a_scan_in_a_form_cleans_the_one_and_reports_the_other() { + // Page 1: its own image covers 40% and a Form holding a scan 20%: a scan only together. + // Page 2: its own image covers the page and the Form 20% more. Both pages clean Im1 and + // report Fm0. + let page = |contents: usize| { + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Fm0 4 0 R /Im1 6 0 R >> >> /Contents {contents} 0 R >>" + ) + }; + let gray = format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8"); + let bytes = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 8 0 R] /Count 2 >>") + .obj(page(7)) + .stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 {W} {H}] /Resources << /XObject << /Im0 5 0 R >> >>"), + format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q"), + ) + .stream(&gray, scan_pixels(1)) + .stream(&gray, scan_pixels(1)) + .stream("", format!("q {W} 0 0 160 0 240 cm /Im1 Do Q q 1 0 0 0.2 0 0 cm /Fm0 Do Q")) + .obj(page(9)) + .stream("", format!("q {W} 0 0 {H} 0 0 cm /Im1 Do Q q 1 0 0 0.2 0 0 cm /Fm0 Do Q")) + .build(); + let (mut s, id) = open(bytes); + let (form_picture, own) = (object(&s, id, 5), object(&s, id, 6)); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!((report.pages.as_slice(), report.images), (&[0, 1][..], 1), "Im1 is cleaned on both pages: {report:?}"); + assert!(report.not_scanned.is_empty(), "{report:?}"); + assert_eq!(report.skipped.len(), 1, "the form is reported once: {report:?}"); + let k = &report.skipped[0]; + assert_eq!((k.pages.as_slice(), k.name.as_str()), (&[0, 1][..], "Fm0")); + assert!(k.reason.contains("form XObject"), "{k:?}"); + assert_ne!(object(&s, id, 6), own, "the page's own image is cleaned"); + assert_eq!(object(&s, id, 5), form_picture, "the picture in the form is left as it was"); +} + +#[test] +fn protected_documents_keep_their_encryption_and_restrictions() { + let protect = |user: &str, permissions: i32| { + let mut doc = pdfcraft_cos::Document::open(Arc::new(grey_scan())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: user, + owner_password: "owner", + permissions, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + Arc::new(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap()) + }; + // Opened with the password: cleaned, saved still encrypted, and the clean scan reads back. + let mut s = Session::new(); + let id = s.open("x.pdf", None, protect("pw", -1), Some("pw")).unwrap(); + assert_eq!(s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap().images, 1); + let saved = s.save_bytes(id).unwrap(); + assert!(pdfcraft_cos::Document::open(saved.clone()).is_err(), "still protected"); + let id2 = s.open("y.pdf", None, saved, Some("pw")).unwrap(); + assert!(grey_at(&samples(&s, id2), 3, 40) >= 235); + // Changes withheld: refused before any work. + let id = s.open("r.pdf", None, protect("", 0b0100), None).unwrap(); + assert_eq!(s.enhance_scanned_pages(id, &[], ScanCleanSettings::default()).unwrap_err(), EditError::NotPermitted("changes to the document")); + assert!(!s.get(id).unwrap().dirty); +} + +#[test] +fn a_result_for_an_older_state_is_refused() { + let (mut s, id) = open(grey_scan()); + let job = s.scan_clean_job(id, &[], ScanCleanSettings::at(70)).unwrap(); + assert_eq!(job.total(), 1); + let cleaned = job.run(|_, _| true).unwrap(); + s.apply(id, Edit::RotatePages { pages: vec![1], degrees: 90 }).unwrap(); + assert!(matches!(s.apply_scan_clean(id, cleaned), Err(EditError::Invalid(_)))); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Rotate page")); + // Cancelled from the progress callback. + let job = s.scan_clean_job(id, &[], ScanCleanSettings::at(70)).unwrap(); + assert!(matches!(job.run(|_, _| false), Err(EditError::Cancelled))); +} + +#[test] +fn edge_shadow_removal_whitens_the_gutter_background_removal_alone_leaves() { + let gutter = |edge_shadows: bool| { + let (mut s, id) = open(grey_scan()); + s.enhance_scanned_pages(id, &[0], ScanCleanSettings { edge_shadows, ..ScanCleanSettings::at(100) }).unwrap(); + let after = samples(&s, id); + [(3, 40), (3, 380), (10, 200)].map(|(x, y)| grey_at(&after, x, y)) + }; + assert!(gutter(false).iter().any(|&v| v < 200), "{:?}", gutter(false)); + assert!(gutter(true).iter().all(|&v| v >= 235), "{:?}", gutter(true)); + // On its own, at level 0, it lightens the gutter's paper to the paper beside it. + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + s.enhance_scanned_pages(id, &[0], ScanCleanSettings { level: 0, edge_shadows: true, despeckle: false }).unwrap(); + let after = samples(&s, id); + assert!(grey_at(&after, 3, 40).abs_diff(grey_at(&before, 150, 40)) <= 4, "{} vs {}", grey_at(&after, 3, 40), grey_at(&before, 150, 40)); + assert_eq!(grey_at(&after, 150, 40), grey_at(&before, 150, 40), "the middle is left alone"); +} + +/// Two pages, each with its own scan image (objects 4 and 7), so cleaning one page can be told +/// apart from cleaning the other. +fn two_scans_pdf() -> Vec { + let dict = format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8"); + let draw = format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q"); + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Im0 4 0 R >> >> /Contents 5 0 R >>")) + .stream(&dict, scan_pixels(1)) + .stream("", &draw) + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Im0 7 0 R >> >> /Contents 8 0 R >>")) + .stream(&dict, scan_pixels(1)) + .stream("", &draw) + .build() +} + +#[test] +fn only_the_chosen_pages_scan_is_cleaned() { + // Clean page 1, then page 2 of a fresh copy: the other page's image stays byte-identical. + for (chosen, image, other) in [(0usize, 4u32, 7u32), (1, 7, 4)] { + let (mut s, id) = open(two_scans_pdf()); + let (image_before, other_before) = (object(&s, id, image), object(&s, id, other)); + assert_eq!(s.get(id).unwrap().page_images(chosen)[0].object, Some(ObjRef::new(image, 0)), "the fixture"); + let report = s.enhance_scanned_pages(id, &[chosen], ScanCleanSettings::at(100)).unwrap(); + assert_eq!((report.pages.as_slice(), report.images), (&[chosen][..], 1), "page {}", chosen + 1); + assert_ne!(object(&s, id, image), image_before, "page {}'s image was not cleaned", chosen + 1); + assert_eq!(object(&s, id, other), other_before, "cleaning page {} changed the other page's image", chosen + 1); + let (Object::Stream(was), Object::Stream(now)) = (&*other_before, &*object(&s, id, other)) else { panic!("the images") }; + assert_eq!(was.raw, now.raw, "the other image's stream bytes"); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Enhance scanned file")); + } +} diff --git a/crates/engine/src/scan_fields.rs b/crates/engine/src/scan_fields.rs index 8287743fc5..05906d75b2 100644 --- a/crates/engine/src/scan_fields.rs +++ b/crates/engine/src/scan_fields.rs @@ -19,30 +19,94 @@ const SCALE: f32 = 2.0; const MAX_SIDE: f32 = 3000.0; /// A page is a scan when its images cover at least this share of it. const COVERED: f64 = 0.5; +/// A picture inside Form artwork is taken for a scan when it has at least this resolution over +/// the area the artwork covers (a quarter of the pixels a 72 dpi scan would have). +const MIN_FORM_DPI: f64 = 36.0; /// Recognised "words" less tall than this (points) are specks read as letters, and would make /// the box they sit on look filled in. const MIN_WORD_HEIGHT: f64 = 4.0; /// Whether `page` is a picture of a page: images cover most of it. pub(crate) fn is_scan(doc: &Document, page: usize, info: &PageInfo) -> bool { - let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); - let page_area = (cx1 - cx0) * (cy1 - cy0); - if !page_area.is_finite() || page_area <= 0.0 { - return false; - } - let covered: f64 = doc + scan_images(doc, page, info).is_some() +} + +/// The raster images (larger than one pixel) `page` draws, each with the share of the page (its +/// crop box) it covers, when together they cover most of it ([`is_scan`]); `None` otherwise. +/// Enhance ▸ Scanned document cleans these up ([`crate::scan_clean`]). +pub(crate) fn scan_images(doc: &Document, page: usize, info: &PageInfo) -> Option> { + let area = page_area(info)?; + let images: Vec<(pdfcraft_edit::PageImage, f64)> = doc .page_images(page) - .iter() + .into_iter() .filter(|i| !i.is_form && i.width > 1 && i.height > 1) - .map(|i| { - let r = i.visible_rect.unwrap_or(i.rect); - let w = (r[2].min(cx1) - r[0].max(cx0)).max(0.0); - let h = (r[3].min(cy1) - r[1].max(cy0)).max(0.0); - w * h - }) - .filter(|a| a.is_finite()) - .sum(); - covered >= COVERED * page_area + .filter_map(|i| Some((share(info, area, &i)?, i))) + .map(|(share, i)| (i, share)) + .collect(); + let covered: f64 = images.iter().map(|(_, share)| share).sum(); + (covered >= COVERED).then_some(images) +} + +/// A page that is a scan, classified in one pass: its raster images (larger than one pixel) and +/// the grouped Form artwork that holds a raster picture, each with the share of the page it +/// covers, when together they cover most of the page; `None` otherwise. The flag says whether the +/// raster images alone do ([`scan_images`]). Enhance ▸ Scanned document cleans the images and +/// reports the Forms, which it can't clean yet, so a page mixing both gets both. +#[allow(clippy::type_complexity)] +pub(crate) fn scan_parts( + doc: &Document, + page: usize, + info: &PageInfo, +) -> Option<(Vec<(pdfcraft_edit::PageImage, f64)>, Vec<(pdfcraft_edit::PageImage, f64)>, bool)> { + let area = page_area(info)?; + let editor = doc.editor.as_ref(); + let (mut images, mut forms) = (Vec::new(), Vec::new()); + let (mut own, mut covered) = (0.0, 0.0); + for i in doc.page_images(page) { + let Some(share) = share(info, area, &i) else { continue }; + if !i.is_form { + if i.width > 1 && i.height > 1 { + own += share; + covered += share; + images.push((i, share)); + } + } else if let (Some(form), Some(editor)) = (i.object, editor) + && holds_scan(&editor.cos, form, share * area) + { + covered += share; + forms.push((i, share)); + } + } + (covered >= COVERED).then_some((images, forms, own >= COVERED)) +} + +/// Whether Form `form`, covering `points` square points of the page, draws a picture that could +/// be a scan of it: one with at least [`MIN_FORM_DPI`] over that area, not a logo inside a page +/// of vector artwork. +fn holds_scan(cos: &pdfcraft_cos::Document, form: pdfcraft_cos::ObjRef, points: f64) -> bool { + let Some(picture) = crate::copy_image::picture_in_form(cos, form) else { return false }; + let obj = cos.get(picture); + let pdfcraft_cos::Object::Stream(s) = &*obj else { return false }; + let px = |k: &[u8]| s.dict.int(k).unwrap_or(0).max(0) as f64; + let per_point = MIN_FORM_DPI / 72.0; + px(b"Width") * px(b"Height") >= points * per_point * per_point +} + +/// The area of `info`'s crop box, when it is a usable one. +fn page_area(info: &PageInfo) -> Option { + let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); + let area = (cx1 - cx0) * (cy1 - cy0); + (area.is_finite() && area > 0.0).then_some(area) +} + +/// The share of the crop box (of area `area`) that `image` covers, when it is a number. +fn share(info: &PageInfo, area: f64, image: &pdfcraft_edit::PageImage) -> Option { + let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); + let r = image.visible_rect.unwrap_or(image.rect); + let w = (r[2].min(cx1) - r[0].max(cx0)).max(0.0); + let h = (r[3].min(cy1) - r[1].max(cy0)).max(0.0); + let share = w * h / area; + share.is_finite().then_some(share) } /// The page as displayed, in points with y up: where detection on a scan works. @@ -122,10 +186,14 @@ impl Session { if !ocr::available() { return Vec::new(); } - let (Ok(engine), Some(job)) = (ocr::engine(), self.ocr_job(id, &[page], ocr::OcrSettings::default())) else { + let settings = ocr::OcrSettings::default(); + let Some(job) = self.ocr_job(id, &[page], settings.clone()) else { return Vec::new(); }; - job.run(&engine, |_, _| true) + let Ok(recognizers) = ocr::recognizers(&settings) else { + return Vec::new(); + }; + job.run(&recognizers, |_, _| true) .into_iter() .flat_map(|p| p.words) .filter_map(|w| { diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index 9a4cb04dac..e5029ccbd6 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -11,11 +11,11 @@ use crate::edit::{EditCtx, check_permission, run_edit}; use crate::history::{Keys, MAX_UNDO, Scope, keys_after, scope_of, uses_scripts}; use crate::{ BookmarkPage, BookmarkRow, CombineSource, Combined, DataFormat, DocId, Document, Edit, EditError, Hidden, ImageResolution, - LOW_RESOLUTION_PRINT_DPI, Label, ObjectKind, OptimizeReport, RecoverySnapshot, SignOptions, SourceKind, SplitPart, SummarySort, TrustStore, - XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, apply_layer_state, comment_list, comment_summary, decode_text, display_bytes, display_config, guard, - info_fields, js, local_utc_offset, note_warnings, open_source, open_source_with, optimize, outline_rows, parse_ymd, print, print_raster, - render_threads, scan_colours, set_default_layer_state, shown_under_redactions, signatures_of, source_kind, sync_xfa_state, undrawn_notices, - use_layer_choices, writable, xfa, xfa_sync_datasets, + LOW_RESOLUTION_PRINT_DPI, Label, ObjectKind, OptimizeReport, RecoverySnapshot, SignOptions, SourceKind, SplitPart, StatusCaptions, SummarySort, + TrustStore, XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, apply_layer_state, carry_verdict_marks, comment_list, comment_summary, decode_text, + display_bytes, display_config, guard, info_fields, js, local_utc_offset, note_warnings, open_source, open_source_with, optimize, outline_rows, + parse_ymd, print, print_raster, render_threads, scan_colours, set_default_layer_state, shown_under_redactions, signatures_of, source_kind, + sync_xfa_state, undrawn_notices, use_layer_choices, writable, xfa, xfa_sync_datasets, }; mod editing; @@ -36,6 +36,8 @@ pub struct Session { utc_offset: Option i64>, /// Certificates trusted for signing (Acrobat: Trusted Certificates). pub(crate) trust: Arc, + /// The words signature status layers are drawn with, in the interface language. + pub(crate) captions: Arc, /// Preferences ▸ JavaScript ▸ Enable Acrobat JavaScript, inverted (on by default). pub(crate) js_off: bool, /// Preferences ▸ Date format, when not the default (see [`dates`]). diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index be7fd5aca5..8ab3ca22fa 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -2,9 +2,9 @@ use super::check_refresh_panic; use super::{ Arc, DocId, Document, Edit, EditCtx, EditError, Label, MAX_UNDO, ObjectKind, OnceLock, RenderPool, SaveOptions, Scope, Session, - XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, check_permission, comment_list, display_bytes, display_config, guard, info_fields, inspect, js, keys_after, - note_warnings, render_threads, run_edit, scan_colours, scope_of, shown_under_redactions, signatures_of, sync_xfa_state, undrawn_notices, - uses_scripts, write_incremental, xfa, xfa_sync_datasets, + XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, carry_verdict_marks, check_permission, comment_list, display_bytes, display_config, guard, info_fields, + inspect, js, keys_after, note_warnings, render_threads, run_edit, scan_colours, scope_of, shown_under_redactions, signatures_of, sync_xfa_state, + undrawn_notices, uses_scripts, write_incremental, xfa, xfa_sync_datasets, }; use crate::{EditNotice, MarkKind}; @@ -342,7 +342,14 @@ impl Session { #[cfg(test)] check_refresh_panic(); if !doc.signatures.is_empty() { - let signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + let mut signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + // The display copy is kept for a metadata edit, and so are its marks, unless the + // edit changed a verdict (a certification can forbid it): the page then draws the + // new one. + if !carry_verdict_marks(&doc.signatures, &mut signatures) { + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } doc.signatures = Arc::new(signatures); } doc.bytes = bytes; @@ -386,8 +393,8 @@ impl Session { } doc.info.file_size = bytes.len(); doc.form = form; - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; @@ -444,8 +451,8 @@ impl Session { doc.marker_scan = OnceLock::new(); } doc.info.file_size = bytes.len(); - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache) }; + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; @@ -467,6 +474,13 @@ impl Session { doc.renderer = RenderPool::new_inline(Arc::new(Vec::new()), doc.config.clone()); doc.display = original.clone(); doc.bytes = original; + // The page shows the original's own artwork again, so nothing counts as drawn, even if + // the refresh that follows fails. + if doc.signatures.iter().any(|s| s.verdict_shown) { + for sig in Arc::make_mut(&mut doc.signatures) { + sig.verdict_shown = false; + } + } #[cfg(test)] check_refresh_panic(); } @@ -528,8 +542,8 @@ impl Session { if invalidate_links { doc.links = OnceLock::new(); } - let signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); - doc.display = display_bytes(Some(editor), &bytes); + let mut signatures = signatures_of(&editor.cos, &bytes, &doc.trust, &doc.sig_cache); + doc.display = display_bytes(Some(editor), &bytes, &mut signatures, &doc.captions); doc.signatures = Arc::new(signatures); doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); doc.bytes = bytes; diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index 553ddd6aa3..f4e25bc2f6 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -114,6 +114,19 @@ impl Session { .map_err(EditError::Write) } + /// XFDF with these comments (0-based page, position in the page's `/Annots`) and their + /// replies: what Copy puts on the clipboard; [`Edit::PasteComments`] takes it back in. + pub fn export_comments(&self, id: DocId, which: &[(usize, usize)]) -> Result { + let doc = self.get(id).ok_or(EditError::NoDocument)?; + let cos = match doc.editor.as_ref() { + Some(e) => e.cos.clone(), + None => { + pdfcraft_cos::Document::open_with_password(doc.bytes.clone(), doc.password.as_deref()).map_err(|e| EditError::Write(e.to_string()))? + } + }; + guard(|| pdfcraft_xfdf::export_xfdf_comments(&cos, which, &doc.name)).map_err(EditError::Write) + } + /// The print-ready PDF for `settings` (sheets laid out for the paper; see `pdfcraft-print`). pub fn print_pdf(&self, id: DocId, settings: &print::Settings) -> Result, EditError> { let doc = self.get(id).ok_or(EditError::NoDocument)?; @@ -157,6 +170,11 @@ impl Session { if !doc.config.layers.is_empty() { set_default_layer_state(&mut cos, &doc.info.layers); } + // Signature states as the screen shows them, so not when its display copy could not be + // written. This copy is only printed, never saved. + if doc.signatures.iter().any(|s| s.verdict_shown) { + let _ = pdfcraft_sign::show_status_layers(&mut cos, &mut doc.signatures.to_vec(), &doc.captions); + } pdfcraft_print::impose(&cos, settings).map_err(|e| EditError::Print(e.to_string())) } @@ -384,7 +402,7 @@ impl Session { /// A new file (Combine, Extract, Split, Create) as bytes, checked like a save before anyone /// writes it (#1698, #2166): it reopens, and shows the pages it was built from. - fn write_new(&self, doc: &pdfcraft_cos::Document) -> Result>, EditError> { + pub(crate) fn write_new(&self, doc: &pdfcraft_cos::Document) -> Result>, EditError> { let bytes = self.write_unchecked(doc)?; let expected = pdfcraft_organize::page_count(doc)?; pdfcraft_cos::Document::open(bytes.clone()).map_err(|e| EditError::Reopen(e.to_string()))?; diff --git a/crates/engine/src/session/save.rs b/crates/engine/src/session/save.rs index 1e6918f6a6..c67cd391b9 100644 --- a/crates/engine/src/session/save.rs +++ b/crates/engine/src/session/save.rs @@ -1,6 +1,6 @@ use super::{ - Arc, DocId, EditError, Keys, OpenError, RecoverySnapshot, RenderPool, SaveOptions, Session, SignOptions, TrustStore, display_bytes, - display_config, guard, render_threads, signatures_of, writable, write_full, write_incremental, xfa_sync_datasets, + Arc, DocId, Document, EditError, Keys, OpenError, RecoverySnapshot, RenderPool, SaveOptions, Session, SignOptions, StatusCaptions, TrustStore, + display_bytes, display_config, guard, render_threads, signatures_of, writable, write_full, write_incremental, xfa_sync_datasets, }; impl Session { @@ -242,17 +242,55 @@ impl Session { pub fn set_trusted_certificates(&mut self, certs: Vec) { self.trust = Arc::new(TrustStore { certs }); for doc in &mut self.docs { - doc.trust = self.trust.clone(); - let Some(editor) = doc.editor.as_mut() else { continue }; - // Keep verdict refresh and renderer refresh together. Viewer caches are - // invalidated by the UI trust path; trust alone changes no document artwork. - let signatures = if doc.signatures.is_empty() { Vec::new() } else { signatures_of(&editor.cos, &doc.bytes, &doc.trust, &doc.sig_cache) }; - doc.display = display_bytes(Some(editor), &doc.bytes); - doc.signatures = Arc::new(signatures); - doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + Self::revalidate(doc, self.trust.clone()); } } + /// Validate `doc`'s signatures against `trust`, and rebuild its display copy and renderer + /// with them: the verdicts choose what the copy draws. An unsigned document has nothing to + /// validate and keeps both. Viewer caches are invalidated by the UI trust path. + pub(crate) fn revalidate(doc: &mut Document, trust: Arc) { + doc.trust = trust; + if doc.signatures.is_empty() { + return; + } + let Some(editor) = doc.editor.as_mut() else { return }; + let mut signatures = signatures_of(&editor.cos, &doc.bytes, &doc.trust, &doc.sig_cache); + doc.display = display_bytes(Some(editor), &doc.bytes, &mut signatures, &doc.captions); + doc.signatures = Arc::new(signatures); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } + + /// The words the caption of a named-layer signature appearance is drawn with + /// ([`pdfcraft_sign::show_status_layers`]), in the interface language. When they change, every + /// open document's display copy and renderer are rebuilt; like trust, this is not an edit. + /// Returns whether they changed. + pub fn set_signature_captions(&mut self, captions: StatusCaptions) -> bool { + if *self.captions == captions { + return false; + } + self.captions = Arc::new(captions); + for doc in &mut self.docs { + Self::draw_captions(doc, self.captions.clone()); + } + true + } + + /// Draw `doc`'s signature status layers with `captions`: the display copy and renderer are + /// rebuilt when it draws a signature's state (working file, generation and dirty unchanged). + /// Whether a state is drawn does not depend on the words: a caption that can't be drawn in + /// them is drawn in English. + pub(crate) fn draw_captions(doc: &mut Document, captions: Arc) { + doc.captions = captions; + if !doc.signatures.iter().any(|s| s.verdict_shown) { + return; + } + let Some(editor) = doc.editor.as_mut() else { return }; + let signatures = Arc::make_mut(&mut doc.signatures); + doc.display = display_bytes(Some(editor), &doc.bytes, signatures, &doc.captions); + doc.renderer = RenderPool::new(doc.display.clone(), render_threads(), display_config(&doc.config)); + } + /// The signing time as a PDF date in local time with its offset (`D:…+02'00'`). pub fn signing_date(&self) -> String { let offset = self.local_offset(); diff --git a/crates/engine/src/session/view.rs b/crates/engine/src/session/view.rs index c0e7417c8e..5ef135d6c8 100644 --- a/crates/engine/src/session/view.rs +++ b/crates/engine/src/session/view.rs @@ -168,6 +168,14 @@ impl Session { Some(BookmarkPage { rows, next: (end < all.len()).then_some(end), truncated: doc.info.outline_more }) } + /// The bookmark at `path` with everything under it, or the whole outline for `[]`, as a tree + /// another document can take ([`crate::Edit::ImportBookmarks`]). `None` when no document has + /// `id`, or its structure could not be read for editing. + pub fn outline_tree(&self, id: DocId, path: &[usize]) -> Option> { + let editor = self.get(id)?.editor.as_ref()?; + Some(pdfcraft_organize::outline_tree(&editor.cos, path)) + } + /// Top-level bookmarks as split points: (first page of each part, its bookmark's title). pub fn bookmark_splits(&self, id: DocId) -> Vec<(usize, String)> { let Some(doc) = self.get(id) else { return Vec::new() }; diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a77e471a7d..d230d098de 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use super::document::is_placeholder_title; +use super::document::{is_placeholder_title, write_display_copy}; use super::edit::{EditCtx, check_permission, run_edit}; use super::*; use pdfcraft_render::{Layer, LayerOp, OpenError, ReadingColors}; @@ -549,7 +549,86 @@ fn pdf_page_stamps_fit_inside_an_explicit_box() { assert!(content.contains("/Pic Do") && !content.contains("BI"), "{content}"); } -fn page_texts(s: &Session, id: DocId) -> Vec { +/// #1297: a 3D model on a page keeps the whole box as its rectangle and draws a PDF poster as a +/// form XObject fitted inside it, keeping its shape and centred. +#[test] +fn a_3d_model_gets_a_fitted_vector_poster_and_a_file_that_is_not_one_is_refused() { + let mut session = Session::new().with_clock(|| 1_700_000_000); + let blank = session.create_blank(595.0, 842.0, 1).unwrap(); + let id = session.open_new("model.pdf", blank).unwrap(); + let mut prc = b"PRC".to_vec(); + prc.extend_from_slice(&[1; 64]); + let add = |model: Vec| Edit::AddModel3d { + page: 0, + rect: [56.0, 400.0, 539.0, 700.0], + model: Arc::new(model), + poster: Some(MarkFile { name: "poster.pdf".into(), bytes: Arc::new(half_red_page()), page: 0 }), + activation: Activation3d::default(), + name: "Bracket".into(), + }; + assert!(session.apply(id, add(b"%PDF-1.7 not a model".to_vec())).is_err()); + session.apply(id, add(prc)).unwrap(); + + let doc = pdfcraft_cos::Document::open(session.save_bytes(id).unwrap()).unwrap(); + let page = &pdfcraft_model::pages(&doc)[0]; + let annots = doc.resolve(page.dict.get(b"Annots").unwrap()); + let [only] = annots.as_array().unwrap().as_slice() else { panic!("the refused model added nothing: {annots:?}") }; + let annot = doc.get(only.as_ref().unwrap()); + let annot = annot.as_dict().unwrap(); + assert_eq!(annot.name(b"Subtype"), Some(&b"3D"[..])); + let rect: Vec = annot.get(b"Rect").unwrap().as_array().unwrap().iter().filter_map(pdfcraft_cos::Object::as_f64).collect(); + assert_eq!(rect, [56.0, 400.0, 539.0, 700.0], "the whole box, not the fitted picture"); + assert_eq!(doc.resolve(annot.get(b"3DA").unwrap()).as_dict().unwrap().name(b"A"), Some(&b"XA"[..])); + let ap = doc.resolve(annot.get(b"AP").unwrap()); + let pdfcraft_cos::Object::Stream(normal) = doc.get(ap.as_dict().unwrap().reference(b"N").unwrap()).as_ref().clone() else { + panic!("an appearance stream") + }; + let pic = doc.resolve(normal.dict.get(b"Resources").unwrap()).as_dict().unwrap().clone(); + let pic = doc.resolve(pic.get(b"XObject").unwrap()).as_dict().unwrap().reference(b"Pic").unwrap(); + let pdfcraft_cos::Object::Stream(picture) = doc.get(pic).as_ref().clone() else { panic!("the picture is a stream") }; + assert_eq!(picture.dict.name(b"Subtype"), Some(&b"Form"[..]), "the page is kept as a form XObject"); + // The 80 × 40 page in the 483 × 300 box: 483 × 241.5 from y 429.25. + let content = String::from_utf8(normal.decoded().unwrap().to_vec()).unwrap(); + assert!(content.contains(" 56 429.25 cm /Pic Do"), "{content}"); +} + +/// A PDF poster whose page has no area is refused rather than drawn as nothing. +#[test] +fn a_3d_poster_page_without_area_is_refused() { + let mut session = Session::new().with_clock(|| 1_700_000_000); + let blank = session.create_blank(595.0, 842.0, 1).unwrap(); + let id = session.open_new("model.pdf", blank).unwrap(); + let flat = String::from_utf8(half_red_page()).unwrap().replace("[0 0 80 40]", "[0 0 80 0 ]"); + let add = Edit::AddModel3d { + page: 0, + rect: [56.0, 400.0, 539.0, 700.0], + model: Arc::new(b"PRC rest".to_vec()), + poster: Some(MarkFile { name: "flat.pdf".into(), bytes: Arc::new(flat.into_bytes()), page: 0 }), + activation: Activation3d::default(), + name: "Bracket".into(), + }; + assert!(matches!(session.apply(id, add), Err(EditError::Source(_)))); + assert!(session.get(id).unwrap().info.annotations.is_empty(), "nothing was added"); +} + +/// Adding a 3D model is a comment change: refused without the comments permission, and only the +/// comments are rebuilt after it. +#[test] +fn a_3d_model_needs_the_comments_permission_and_changes_the_comments() { + let edit = Edit::AddModel3d { + page: 0, + rect: [0.0, 0.0, 100.0, 100.0], + model: Arc::new(b"PRC rest".to_vec()), + poster: None, + activation: Activation3d::default(), + name: String::new(), + }; + let p = pdfcraft_cos::Permissions { bits: 0, owner: false, revision: 3 }; + assert_eq!(check_permission(&edit, &p), Err(EditError::NotPermitted("comments"))); + assert_eq!(crate::history::scope_of(&edit), crate::history::Scope::Comments); +} + +pub(crate) fn page_texts(s: &Session, id: DocId) -> Vec { let doc = s.get(id).unwrap(); let config = pdfcraft_render::RenderConfig { password: doc.password.as_deref().map(Arc::from), ..Default::default() }; let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), config); @@ -2132,7 +2211,7 @@ fn rect_comment(page: usize, rect: [f64; 4]) -> Edit { Edit::AddAnnotation(NewAnnotation { page, shape: Shape::Rectangle { rect }, - style: Style { color: [1.0, 0.0, 0.0], opacity: 1.0, width: 2.0, fill: Some([1.0, 0.0, 0.0]) }, + style: Style { color: [1.0, 0.0, 0.0], opacity: 1.0, width: 2.0, fill: Some([1.0, 0.0, 0.0]), ..Style::default() }, contents: "Look here".into(), author: "Reviewer".into(), }) @@ -3037,6 +3116,26 @@ fn a_word_in_a_font_set_by_an_extgstate_is_redacted() { ); } +/// A word drawn as outlines reads through its /ActualText. A mark over part of it applies (the +/// outlines are clipped), and the name that would still read the whole word is gone from the page +/// and from the saved file. +#[test] +fn a_word_drawn_as_outlines_is_redacted_with_its_name() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page("", "/Span <> BDC 20 150 m 140 150 l 140 170 l 20 170 l h f EMC"); + let id = s.open("outlines.pdf", None, Arc::new(page), None).unwrap(); + assert_eq!(page_texts(&s, id), ["SECRET42"]); + let shape = + Shape::Redact { quads: vec![pdfcraft_annot::rect_quad([10.0, 140.0, 90.0, 180.0])], overlay: String::new(), look: Default::default() }; + let mark = + Edit::AddAnnotation(NewAnnotation { page: 0, style: Style::default_for(&shape), shape, contents: String::new(), author: "Ada".into() }); + s.apply(id, mark).unwrap(); + s.apply(id, Edit::ApplyRedactions { pages: None }).unwrap(); + assert_eq!(page_texts(&s, id), [""]); + let saved = s.save_bytes(id).unwrap(); + assert!(!saved.windows(b"SECRET42".len()).any(|w| w == b"SECRET42"), "the name is not left in the file"); +} + #[test] fn redaction_marks_apply_for_good_and_undo() { let (mut s, id) = session_with(2); @@ -4103,6 +4202,7 @@ fn pasted_added_content_keeps_its_style_and_picture() { italic: true, size: 12.0, color: [0.8, 0.1, 0.2], + ink: None, align: TextAlign::Center, aligns: Vec::new(), char_spacing: 0.0, @@ -4775,6 +4875,17 @@ fn bytes_has(haystack: &[u8], needle: &[u8]) -> bool { /// ranges and embed a detached CMS over them. The signed bytes keep the /// prepared appearance, so they validate exactly like a product signer's file. fn sign_prepared(doc: &mut pdfcraft_cos::Document, id: &sign::pkcs12::DigitalId, date: &str) -> Vec { + sign_prepared_with(doc, id, date, |_| {}) +} + +/// [`sign_prepared`], with `value` adding to the signature dictionary first (a certification's +/// `/Reference`, say). +fn sign_prepared_with( + doc: &mut pdfcraft_cos::Document, + id: &sign::pkcs12::DigitalId, + date: &str, + value: impl FnOnce(&mut pdfcraft_cos::Dict), +) -> Vec { use pdfcraft_cos::{Dict, Object, PdfString}; let reserve = 8192 + id.certificate.raw.len() + id.chain.iter().map(|c| c.raw.len()).sum::(); let mut v = Dict::new(); @@ -4784,6 +4895,7 @@ fn sign_prepared(doc: &mut pdfcraft_cos::Document, id: &sign::pkcs12::DigitalId, v.set(b"ByteRange".to_vec(), Object::Array([0].iter().chain(LAYERED_BR_MARK.iter()).map(|n| Object::Int(*n)).collect())); v.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0; reserve], hex: true })); v.set(b"M".to_vec(), PdfString::literal(date.as_bytes().to_vec())); + value(&mut v); let sig = doc.add(Object::Dict(v)); // The field by name, never by number: a full save (encryption, repairs) is // free to renumber objects, and a hardcoded reference would stamp the value @@ -5024,6 +5136,442 @@ fn real_layered_signatures_round_trip_behind_a_user_password() { assert_eq!(t.get(tid).unwrap().signatures[0].status, SignatureStatus::Valid); } +/// A one-page file whose unsigned signature field carries a named-layer appearance, in the +/// geometry of a real one scaled by 4 (every word and drawing synthetic): /N draws /FRM, which +/// draws n0–n4; n1 a yellow question mark and n4 "Signature Not Verified". Object numbers: +/// 1 catalog, 2 pages, 3 page (400×300), 4 field, 5 AcroForm, 6 /N, 7 /FRM, 8–12 n0–n4, +/// 13 Helvetica, 14 page content. `sign_prepared` adds the value. +fn named_layer_signable_fixture() -> Vec { + named_layer_signable_fixture_with("[0 0 100 100]") +} + +/// [`named_layer_signable_fixture`] with `n1_bbox` as its n1 layer's `/BBox`. +fn named_layer_signable_fixture_with(n1_bbox: &str) -> Vec { + fn form(bbox: &str, resources: &str, ops: &str) -> String { + format!("<< /Type /XObject /Subtype /Form /BBox {bbox} /Resources {resources} /Length {} >>\nstream\n{ops}\nendstream", ops.len()) + } + let helv = "<< /Font << /F1 13 0 R >> >>"; + let glyph = "1.08 0 0 1.08 46 6 cm"; + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 5 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_string(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 300] /Contents 14 0 R /Resources << /Font << /F1 13 0 R >> >> /Annots [4 0 R] >>" + .to_string(), + "<< /FT /Sig /T (Signature1) /Type /Annot /Subtype /Widget /F 132 /Rect [100 100 300 220] /P 3 0 R /AP << /N 6 0 R >> >>".to_string(), + "<< /Fields [4 0 R] /SigFlags 3 >>".to_string(), + form("[0 0 200 120]", "<< /XObject << /FRM 7 0 R >> >>", "q 1 0 0 1 0 0 cm /FRM Do Q"), + form( + "[0 0 200 120]", + "<< /XObject << /n0 8 0 R /n1 9 0 R /n2 10 0 R /n3 11 0 R /n4 12 0 R >> >>", + &format!("q /n0 Do Q q {glyph} /n1 Do Q q /n2 Do Q q {glyph} /n3 Do Q q /n4 Do Q"), + ), + form("[0 0 100 100]", "<< >>", "% DSBlank"), + form( + n1_bbox, + "<< >>", + "q 1 0.84 0.2 rg 0.55 0.4 0 RG 3 w 30 72 m 30 88 42 98 56 98 c 72 98 84 88 84 74 c 66 50 l 58 28 l 44 28 l 44 48 l h B 44 6 14 14 re B Q", + ), + form("[0 0 200 120]", helv, "BT /F1 13.6 Tf 1 0 0 1 8 62.4 Tm (Digitally signed by) Tj 1 0 0 1 8 48.8 Tm (Test Signer) Tj ET"), + form("[0 0 100 100]", "<< >>", "% DSBlank"), + form("[0 84 200 120]", helv, "BT /F1 18.8 Tf 1 0 0 1 8 93.2 Tm (Signature Not Verified) Tj ET"), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_string(), + "<< /Length 44 >>\nstream\nBT /F1 14 Tf 20 270 Td (Contract text) Tj ET\nendstream".to_string(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (k, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", k + 1).as_bytes()); + } + let start = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in &offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{start}\n%%EOF", objs.len() + 1).as_bytes()); + out +} + +/// The status colours as the regenerated icons draw them. +const VALID_GREEN: [u8; 3] = [0x2D, 0x9D, 0x78]; +const UNKNOWN_AMBER: [u8; 3] = [0xE6, 0x86, 0x19]; +const INVALID_RED: [u8; 3] = [0xD7, 0x37, 0x3F]; + +/// How many pixels of `rgba` are within 40 of `rgb` on every channel. A regenerated icon's +/// stroke covers thousands at 1 px/pt; the fixture holds none of these colours otherwise. +fn pixels_near(rgba: &[u8], rgb: [u8; 3]) -> usize { + rgba.as_chunks::<4>().0.iter().filter(|p| p.iter().zip(rgb).all(|(a, b)| a.abs_diff(b) <= 40)).count() +} + +/// The displayed page at 1 px/pt. +fn displayed_pixels(s: &Session, id: DocId) -> Vec { + let doc = s.get(id).unwrap(); + let mut renderer = pdfcraft_render::PageRenderer::new(doc.display.clone(), doc.config.clone()); + let r = renderer.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(r.error.is_none(), "{:?}", r.error); + r.rgba.to_vec() +} + +fn exported_text(s: &Session, id: DocId) -> String { + export::Exporter::new(s.get(id).unwrap()).text_of(&[0]).unwrap() +} + +/// The test identity and the certificate that makes it trusted. +fn test_identity() -> (sign::pkcs12::DigitalId, sign::x509::Certificate) { + let p12 = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/../sign/tests/data/ec-p256.p12")).unwrap(); + let id = sign::pkcs12::open(&p12, "test").unwrap(); + let anchor = id.chain.first().cloned().unwrap_or_else(|| id.certificate.clone()); + (id, anchor) +} + +/// A named-layer signature through `Session`: the unknown state drawn on opening, the valid one +/// once trusted, the signed bytes saved unchanged and still valid on reopening. +fn check_named_layer_states(signed: Arc>, password: Option<&str>, anchor: &sign::x509::Certificate) { + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("named.pdf", None, signed.clone(), password).unwrap(); + let sig = &s.get(id).unwrap().signatures[0]; + assert_eq!(sig.status, SignatureStatus::Unknown, "untrusted at first"); + assert!(sig.verdict_shown, "the appearance shows the unknown state"); + let text = exported_text(&s, id); + assert!(text.contains("Validity unknown") && !text.contains("Not Verified"), "{text:?}"); + let amber = pixels_near(&displayed_pixels(&s, id), UNKNOWN_AMBER); + assert!(amber > 500, "{amber} amber pixels"); + s.set_trusted_certificates(vec![anchor.clone()]); + let doc = s.get(id).unwrap(); + assert_eq!(doc.signatures[0].status, SignatureStatus::Valid); + assert!(doc.signatures[0].verdict_shown); + assert!(!doc.dirty, "trust is not an edit"); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); + let page = displayed_pixels(&s, id); + assert!(pixels_near(&page, VALID_GREEN) > 500 && pixels_near(&page, UNKNOWN_AMBER) < 50, "the page redraws for the new state"); + // Saving writes the working file, which never held the drawn state. + let saved = s.save_bytes(id).unwrap(); + assert!(saved == signed, "an unchanged save is the signed file exactly"); + let mut t = Session::new().with_clock(|| 1_800_000_000); + t.set_trusted_certificates(vec![anchor.clone()]); + let tid = t.open("named-saved.pdf", None, saved, password).unwrap(); + let sig = &t.get(tid).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Valid && sig.verdict_shown); + let exported = exported_text(&t, tid); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn named_layer_signatures_show_their_state_through_trust_save_and_reopen() { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + check_named_layer_states(signed, None, &anchor); +} + +#[test] +fn named_layer_signatures_round_trip_behind_a_user_password() { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes128, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let encrypted = Arc::new(pdfcraft_cos::write_full(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap()); + let mut doc = pdfcraft_cos::Document::open_with_password(encrypted, Some("user")).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + check_named_layer_states(signed, Some("user"), &anchor); +} + +#[test] +fn a_tampered_named_layer_signature_shows_the_invalid_state() { + let (id, _) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let mut signed = sign_prepared(&mut doc, &id, "D:20261002120000+01'00'"); + let at = signed.windows(15).position(|w| w == b"(Contract text)").unwrap(); + signed[at + 10] = b'X'; + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("tampered.pdf", None, Arc::new(signed), None).unwrap(); + let sig = &s.get(id).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Invalid && sig.verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature invalid"), "{exported}"); + let red = pixels_near(&displayed_pixels(&s, id), INVALID_RED); + assert!(red > 500, "{red} red pixels"); +} + +/// A trusted named-layer signature. +fn trusted_named_layer_session() -> (Session, DocId) { + let (id, anchor) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let vid = s.open("named.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + assert_eq!(s.get(vid).unwrap().signatures[0].status, SignatureStatus::Valid); + (s, vid) +} + +#[test] +fn named_layer_states_reach_exports_and_both_print_modes() { + // What is drawn on the page is what leaves it. + let (s, id) = trusted_named_layer_session(); + let doc = s.get(id).unwrap(); + let source = doc.export_source(); + let mut file = export::Exporter::from_source(export::ExportSource { bytes: doc.bytes.clone(), ..source.clone() }); + let mut shown = export::Exporter::from_source(source); + assert_ne!(shown.png(0, 72.0).unwrap(), file.png(0, 72.0).unwrap(), "image export follows the display copy"); + let text = shown.text_of(&[0]).unwrap(); + assert!(text.contains("Signature valid") && !text.contains("Not Verified"), "{text:?}"); + let mut settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; + let (_, _, vector) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + settings.raster = Some(print::Raster { dpi: 72.0, grayscale: false }); + let (_, _, raster) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + for (mode, pixels) in [("vector", &vector), ("raster", &raster)] { + assert!(pixels_near(pixels, VALID_GREEN) > 500, "{mode} print draws the valid state"); + assert!(pixels_near(pixels, [255, 214, 51]) < 50, "{mode} print drops the stored question mark"); + } +} + +#[test] +fn named_layers_show_beside_comments_that_need_appearances() { + // A comment without an appearance makes the display copy for its own sake; the signature's + // state is drawn into that same copy. + use pdfcraft_cos::{ObjRef, Object}; + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let square = cos.add(pdfcraft_cos::Lexer::new(b"<< /Type /Annot /Subtype /Square /Rect [10 10 60 60] /C [1 0 0] >>", 0).object().unwrap()); + cos.update_dict(ObjRef::new(3, 0), |page| page.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(square)]))) + .unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("commented.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let shown = pdfcraft_cos::Document::open(s.get(id).unwrap().display.clone()).unwrap(); + assert!(shown.get(square).as_dict().is_some_and(|d| d.contains(b"AP")), "the comment's appearance is in the same copy"); + assert!(s.get(id).unwrap().signatures[0].verdict_shown); + let green = pixels_near(&displayed_pixels(&s, id), VALID_GREEN); + assert!(green > 500, "{green} green pixels"); +} + +#[test] +fn named_layers_show_beside_fields_the_viewer_draws() { + // A text field without an appearance makes the display copy for its own sake; the + // signature's state is drawn into that same copy. + use pdfcraft_cos::{Lexer, ObjRef, Object}; + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let field = cos.add( + Lexer::new( + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /V (Test Signer) /DA (/F1 12 Tf 0 g) /Rect [20 20 380 40] /F 4 /P 3 0 R >>", + 0, + ) + .object() + .unwrap(), + ); + let both = Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(field)]); + cos.update_dict(ObjRef::new(3, 0), |page| page.set(b"Annots".to_vec(), both.clone())).unwrap(); + let resources = Lexer::new(b"<< /Font << /F1 13 0 R >> >>", 0).object().unwrap(); + cos.update_dict(ObjRef::new(5, 0), |form| { + form.set(b"Fields".to_vec(), both); + form.set(b"DR".to_vec(), resources); + }) + .unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("filled.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let shown = pdfcraft_cos::Document::open(s.get(id).unwrap().display.clone()).unwrap(); + assert!(shown.get(field).as_dict().is_some_and(|d| d.contains(b"AP")), "the field's appearance is in the same copy"); + assert!(s.get(id).unwrap().signatures[0].verdict_shown); + let green = pixels_near(&displayed_pixels(&s, id), VALID_GREEN); + assert!(green > 500, "{green} green pixels"); +} + +#[test] +fn a_metadata_edit_keeps_the_regenerated_state() { + let (mut s, id) = trusted_named_layer_session(); + s.apply(id, Edit::SetInfo { key: "Title".into(), value: "Contract".into() }).unwrap(); + let doc = s.get(id).unwrap(); + assert!(doc.signatures[0].verdict_shown, "the page still draws the state"); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn a_metadata_edit_that_changes_a_verdict_redraws_it() { + // A certification that allows no changes: setting the title invalidates the signature, and + // the page has to say so rather than keep the valid state it drew. + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = sign_prepared_with(&mut cos, &signer, "D:20261002120000+01'00'", |v| { + let reference = + pdfcraft_cos::Lexer::new(b"[<< /Type /SigRef /TransformMethod /DocMDP /TransformParams << /Type /TransformParams /P 1 /V /1.2 >> >>]", 0) + .object() + .unwrap(); + v.set(b"Reference".to_vec(), reference); + }); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let id = s.open("certified.pdf", None, Arc::new(signed), None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + assert_eq!(s.get(id).unwrap().signatures[0].status, SignatureStatus::Valid); + s.apply(id, Edit::SetInfo { key: "Title".into(), value: "Changed".into() }).unwrap(); + let doc = s.get(id).unwrap(); + assert_eq!(doc.signatures[0].status, SignatureStatus::Invalid); + assert!(doc.signatures[0].verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature invalid"), "{exported}"); + let red = pixels_near(&displayed_pixels(&s, id), INVALID_RED); + assert!(red > 500, "{red} red pixels"); +} + +#[test] +fn verdict_marks_carry_over_only_while_every_verdict_is_unchanged() { + let (s, id) = trusted_named_layer_session(); + let previous = s.get(id).unwrap().signatures.to_vec(); + assert!(previous[0].verdict_shown); + let mut same = previous.clone(); + same[0].verdict_shown = false; + assert!(carry_verdict_marks(&previous, &mut same) && same[0].verdict_shown); + let mut changed = previous.clone(); + (changed[0].verdict_shown, changed[0].status) = (false, SignatureStatus::Invalid); + assert!(!carry_verdict_marks(&previous, &mut changed) && !changed[0].verdict_shown); + let mut later = previous.clone(); + (later[0].verdict_shown, later[0].revision) = (false, previous[0].revision + 1); + assert!(!carry_verdict_marks(&previous, &mut later) && !later[0].verdict_shown); +} + +#[test] +fn releasing_the_working_file_drops_the_marks() { + // The page shows the original's own artwork until the refresh finishes. + let (mut s, id) = trusted_named_layer_session(); + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + assert!(doc.signatures[0].verdict_shown); + Session::release_working(doc); + assert!(!doc.signatures[0].verdict_shown); +} + +#[test] +fn background_open_uses_the_current_trust_at_commit() { + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let prepared = s.open_context().prepare_open("background.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + let id = s.commit_open(prepared); + let sig = &s.get(id).unwrap().signatures[0]; + assert!(sig.status == SignatureStatus::Valid && sig.verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signature valid"), "{exported}"); +} + +#[test] +fn background_open_uses_the_current_trust_and_language_at_commit() { + // Both changed while the document was prepared: the state and its words are both the new ones. + let (signer, anchor) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")); + let mut s = Session::new().with_clock(|| 1_800_000_000); + let prepared = s.open_context().prepare_open("background.pdf", None, signed, None).unwrap(); + s.set_trusted_certificates(vec![anchor]); + s.set_signature_captions(StatusCaptions { + valid: "Signatur gültig".into(), + unknown: "Gültigkeit unbekannt".into(), + invalid: "Signatur ungültig".into(), + }); + let id = s.commit_open(prepared); + assert_eq!(s.get(id).unwrap().signatures[0].status, SignatureStatus::Valid); + let exported = exported_text(&s, id); + assert!(exported.contains("Signatur gültig"), "{exported}"); +} + +#[test] +fn printing_draws_signature_states_only_where_the_screen_does() { + let (mut s, id) = trusted_named_layer_session(); + let settings = print::Settings { pages: vec![0], paper: (400.0, 300.0), ..Default::default() }; + // The screen falls back to the file's artwork when its display copy cannot be written, and + // takes the marks with it; print follows. + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + let mut cleared = doc.signatures.to_vec(); + cleared[0].verdict_shown = false; + doc.signatures = Arc::new(cleared); + let (_, _, printed) = printed_pixels(s.print_pdf(id, &settings).unwrap()); + let green = pixels_near(&printed, VALID_GREEN); + assert!(green < 50, "{green} green pixels"); +} + +#[test] +fn a_failed_display_write_drops_the_marks() { + let (mut s, id) = trusted_named_layer_session(); + let doc = s.docs.iter_mut().find(|d| d.id == id).unwrap(); + let mut signatures = doc.signatures.to_vec(); + assert!(signatures[0].verdict_shown); + // A stream-limited document is read-only: the writer refuses it. + let unwritable = pdfcraft_cos::Document::open_with_stream_limit(doc.bytes.clone(), None, 1).unwrap(); + let editor = doc.editor.as_mut().unwrap(); + let shown = write_display_copy(editor, &doc.bytes, &unwritable, &mut signatures); + assert!(Arc::ptr_eq(&shown, &doc.bytes), "the page falls back to the file's own artwork"); + assert!(!signatures[0].verdict_shown, "so nothing counts as drawn"); + // Logged once, however many refreshes lose the copy again. + write_display_copy(editor, &doc.bytes, &unwritable, &mut signatures); + let logged = editor.cos.repair_log().iter().filter(|l| l.contains("the display copy could not be written")).count(); + assert_eq!(logged, 1); +} + +#[test] +fn background_open_uses_the_current_caption_language_at_commit() { + let (mut s, id) = trusted_named_layer_session(); + let bytes = s.get(id).unwrap().bytes.clone(); + let prepared = s.open_context().prepare_open("background.pdf", None, bytes, None).unwrap(); + s.set_signature_captions(StatusCaptions { + valid: "Signatur gültig".into(), + unknown: "Gültigkeit unbekannt".into(), + invalid: "Signatur ungültig".into(), + }); + let later = s.commit_open(prepared); + let exported = exported_text(&s, later); + assert!(exported.contains("Signatur gültig"), "{exported}"); + assert!(s.get(later).unwrap().signatures[0].verdict_shown); + assert!(!s.get(later).unwrap().dirty); +} + +#[test] +fn caption_language_follows_the_session_captions() { + let (mut s, id) = trusted_named_layer_session(); + let generation = s.get(id).unwrap().generation; + let german = StatusCaptions { valid: "Signatur gültig".into(), unknown: "Gültigkeit unbekannt".into(), invalid: "Signatur ungültig".into() }; + s.set_signature_captions(german); + let doc = s.get(id).unwrap(); + assert!(!doc.dirty && doc.generation == generation, "captions are not an edit"); + assert!(doc.signatures[0].verdict_shown); + let exported = exported_text(&s, id); + assert!(exported.contains("Signatur gültig"), "{exported}"); + // Documents opened later draw with them too. + let (signer, _) = test_identity(); + let mut cos = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let later = s.open("later.pdf", None, Arc::new(sign_prepared(&mut cos, &signer, "D:20261002120000+01'00'")), None).unwrap(); + let exported = exported_text(&s, later); + assert!(exported.contains("Signatur gültig"), "{exported}"); +} + +#[test] +fn non_win_ansi_captions_embed_inter_in_the_display_copy_only() { + // Words WinAnsi cannot encode are drawn in Inter, embedded into the display copy only. + let (mut s, id) = trusted_named_layer_session(); + let russian = StatusCaptions { + valid: "Подпись действительна".into(), + unknown: "Действительность неизвестна".into(), + invalid: "Подпись недействительна".into(), + }; + s.set_signature_captions(russian); + let exported = exported_text(&s, id); + assert!(exported.contains("Подпись действительна"), "{exported}"); + let doc = s.get(id).unwrap(); + assert!(bytes_has(&doc.display, b"/BaseFont /Inter") && !bytes_has(&doc.bytes, b"/BaseFont /Inter"), "Inter goes into the display copy only"); +} + #[test] #[ignore = "timing probe"] fn probe_edit_latency_on_a_large_signed_document() { @@ -5762,6 +6310,10 @@ fn comparing_two_versions_of_a_document() { let got: Vec<(compare::Kind, &str, &str)> = c.changes.iter().map(|x| (x.kind, x.old.text.as_str(), x.new.text.as_str())).collect(); assert_eq!(got, [(compare::Kind::Replaced, "Monday.", "Tuesday."), (compare::Kind::Replaced, "days.", "days of invoice.")], "{got:?}"); assert!(s.compare(old, old).unwrap().identical()); + // limited to pages of the new file: both changes are on the first + assert_eq!(s.compare_pages(old, new, &[]).unwrap().changes.len(), 2); + assert_eq!(s.compare_pages(old, new, &[0]).unwrap().changes.len(), 2); + assert!(s.compare_pages(old, new, &[1]).unwrap().identical()); let report = s.compare_report(old, new).unwrap(); let rid = s.open("report.pdf", None, report, None).unwrap(); @@ -5969,7 +6521,7 @@ fn measurement_edits_are_transactional_and_honour_annotation_permissions() { assert!(session.get(id).unwrap().measurements().unwrap().measurements.is_empty()); session.redo(id).unwrap(); assert_eq!(session.get(id).unwrap().bytes, before); - let p = pdfcraft_cos::Permissions { bits: 0, owner: false }; + let p = pdfcraft_cos::Permissions { bits: 0, owner: false, revision: 3 }; assert_eq!(check_permission(&Edit::AddMeasurement(m), &p), Err(EditError::NotPermitted("comments"))); assert_eq!( check_permission( @@ -7243,12 +7795,13 @@ fn datasets_streams(bytes: &Arc>) -> usize { /// and noted. #[test] fn hostile_xfa_open_scripts_are_merged_capped_kept_from_printing_and_noted() { - // On open: a calculate setting its own value 50 000 times, an initialize script showing a - // thousand messages and asking to print, open a link and save. + // On open: a calculate setting its own value 20 000 times (50 000 sat right at the 1 s + // unasked-script timeout and abandoned whenever the test machine was loaded), an + // initialize script showing a thousand messages and asking to print, open a link and save. let tpl = pdfcraft_xfa::fixtures::scripted_template() .replace( r#""#, - r#""#, + r#""#, ) .replace( r#"if (qty.rawValue === null) qty.rawValue = 2;"#, @@ -7258,9 +7811,11 @@ fn hostile_xfa_open_scripts_are_merged_capped_kept_from_printing_and_noted() { let mut s = Session::new().with_clock(|| 1_700_000_000); let started = std::time::Instant::now(); let id = s.open("hostile.pdf", None, bytes, None).expect("opens"); - assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + // 30 s and not 5 s: the run is wall-clock timed and a loaded test machine (all suites in + // parallel) can take several seconds for the 20 000 iterations; a real hang still fails. + assert!(started.elapsed() < std::time::Duration::from_secs(30), "{:?}", started.elapsed()); let doc = s.get(id).unwrap(); - assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["49999".to_string()]); + assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["19999".to_string()]); // One datasets stream for the scripts' revision, not one per value set. assert!(datasets_streams(&doc.bytes) <= 3, "{} datasets streams", datasets_streams(&doc.bytes)); // What opening changed is noted with the form's other XFA notes. @@ -7567,6 +8122,37 @@ fn free_text_without_appearance() -> Arc> { Arc::new(bytes) } +/// The CLI renders what the app shows with no trusted certificates: a named-layer signature in +/// the state it has then (validity unknown), not the "not verified" its file stores. The source +/// is not changed. +#[test] +fn cli_display_draws_signature_states_as_the_app_does_without_trust() { + let (id, _) = test_identity(); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture())).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let display = display_bytes_for(signed.clone(), None); + assert!(!Arc::ptr_eq(&display, &signed), "the state is drawn into a copy"); + let mut renderer = pdfcraft_render::PageRenderer::new(display, pdfcraft_render::RenderConfig::default()); + let page = renderer.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(pixels_near(&page.rgba, UNKNOWN_AMBER) > 500, "the unknown state is drawn"); + assert!(pixels_near(&page.rgba, [255, 214, 51]) < 50, "the stored question mark is not"); +} + +/// A layer recognised but too small to draw an icon in (a side so small the icon's scale rounds to +/// zero) changes nothing, so the CLI renders the source, as the app does. +#[test] +fn cli_display_makes_no_copy_when_no_state_could_be_drawn() { + let (id, _) = test_identity(); + let n1 = format!("[0 0 0.{}6 100]", "0".repeat(322)); + let mut doc = pdfcraft_cos::Document::open(Arc::new(named_layer_signable_fixture_with(&n1))).unwrap(); + let signed = Arc::new(sign_prepared(&mut doc, &id, "D:20261002120000+01'00'")); + let cos = pdfcraft_cos::Document::open(signed.clone()).unwrap(); + let trust = pdfcraft_sign::TrustStore::default(); + let signatures = crate::doc_state::signatures_of(&cos, &signed, &trust, &pdfcraft_sign::DigestCache::default()); + assert!(pdfcraft_sign::has_status_layers(&cos, &signatures), "the layers are recognised"); + assert!(Arc::ptr_eq(&display_bytes_for(signed.clone(), None), &signed), "nothing was drawn, so there is no copy"); +} + /// #2371: a comment without an appearance stream is displayed by the app, so the CLI's page image /// must show it too. The helper's copy is for rendering only; documents that need no copy, and /// files it can't display, keep exactly the original source bytes. @@ -8026,6 +8612,7 @@ fn content_edit_keeps_pages_annotations_fields_and_links() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, @@ -8067,6 +8654,7 @@ fn content_refresh_lists_match_the_storages_an_edit_touches() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, @@ -8425,6 +9013,141 @@ fn a_signed_document_rebuilt_from_object_headers_refuses_edits() { assert!(!s.get(id).unwrap().dirty); } +/// A one-page document whose only signature field is signed and named with `len` bytes. The +/// signature field walk reads names of up to 4096 bytes and passes longer ones over. +fn signed_field_named(len: usize) -> Vec { + pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Sig /T ({}) /V 5 0 R /Rect [20 20 180 60] /P 3 0 R >>", "a".repeat(len)), + "<< /Type /Sig /Filter /Adobe.PPKLite /SubFilter /adbe.pkcs7.detached /ByteRange [0 1 2 1] /Contents <00> >>".into(), + ]) +} + +/// A PDF of `objs`, numbered from 1, with a cross-reference table. +fn pdf_of(objs: &[String]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn open_signed_field_named(len: usize) -> (Session, DocId) { + let mut s = Session::new(); + let id = s.open("form.pdf", None, Arc::new(signed_field_named(len)), None).expect("opens"); + (s, id) +} + +/// The cap's boundary as the engine meets it: a 4096-byte name is read, and its signature blocks a +/// full rewrite. +#[test] +fn a_signature_named_at_the_cap_is_listed_and_blocks_a_full_rewrite() { + let (s, id) = open_signed_field_named(4096); + let doc = s.get(id).unwrap(); + let listed: Vec<_> = doc.signatures.iter().map(|r| (r.field.len(), r.signed)).collect(); + assert_eq!(listed, [(4096, true)], "the field is read, and no row stands for unread ones"); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +/// One byte past the cap the field goes unread: the list says so, and a full rewrite is still +/// refused. +#[test] +fn a_signature_named_one_byte_past_the_cap_still_blocks_a_full_rewrite() { + let (s, id) = open_signed_field_named(4097); + let doc = s.get(id).unwrap(); + let unread = doc.signatures.iter().find(|r| r.field == "UnreadFields"); + let said = unread.map(|r| (r.signed, r.status, r.details.first().is_some_and(|d| d.message.starts_with("This form has too many fields")))); + assert_eq!(said, Some((true, SignatureStatus::Unknown, true)), "{:?}", doc.signatures); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +/// A signed field behind a kid with no /T: one merged with its widget under a named field that +/// isn't a signature field (the walk can't list it, so its row is the unread one), and one under a +/// node with no /T (listed by its full name). Either way the document is signed, and neither a full +/// save nor Optimize may rewrite it. +#[test] +fn signed_fields_behind_kids_without_a_name_block_a_full_rewrite() { + let page = |annot: u32| format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Annots [{annot} 0 R] >>"); + let head = |annot: u32| { + vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + page(annot), + ] + }; + let sig = "<< /Type /Sig /SubFilter /ETSI.CAdES.detached /ByteRange [0 1 2 1] /Contents <00> >>".to_string(); + let mut merged = head(5); + merged.extend([ + "<< /T (parent) /Kids [5 0 R] >>".to_string(), + "<< /Type /Annot /Subtype /Widget /FT /Sig /V 6 0 R /Parent 4 0 R /Rect [20 20 180 60] /P 3 0 R >>".into(), + sig.clone(), + ]); + let mut nested = head(6); + nested.extend([ + "<< /T (p) /Kids [5 0 R] >>".to_string(), + "<< /Kids [6 0 R] /Parent 4 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Sig /T (s) /V 7 0 R /Parent 5 0 R /Rect [20 20 180 60] /P 3 0 R >>".into(), + sig, + ]); + for (shape, objs, row) in [("merged", merged, "UnreadFields"), ("nested", nested, "p.s")] { + let mut s = Session::new(); + let id = s.open("form.pdf", None, Arc::new(pdf_of(&objs)), None).expect("opens"); + let doc = s.get(id).unwrap(); + assert!(doc.is_signed() && doc.signatures.iter().any(|r| r.field == row), "{shape}: {:?}", doc.signatures); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed, "{shape}"); + assert!(matches!(s.optimize_job(id, &optimize::Settings::default(), &[]), Err(EditError::Signed)), "{shape}"); + } +} + +/// The walk stops short of a signed field, so a signature may go unlisted; the document still +/// counts as signed, and each guard against rewriting a signed document holds. +#[test] +fn a_signature_the_field_walk_passes_over_still_counts_as_signed() { + let (s, id) = open_signed_field_named(64 << 10); + let doc = s.get(id).unwrap(); + assert!(doc.is_signed(), "{:?}", doc.signatures); +} + +#[test] +fn a_full_rewrite_is_refused_when_the_field_walk_passes_a_signature_over() { + let (s, id) = open_signed_field_named(64 << 10); + assert_eq!(s.save_full_bytes(id).unwrap_err(), EditError::Signed); +} + +#[test] +fn optimizing_is_refused_when_the_field_walk_passes_a_signature_over() { + let (s, id) = open_signed_field_named(64 << 10); + assert!(matches!(s.optimize_job(id, &optimize::Settings::default(), &[]), Err(EditError::Signed))); +} + +#[test] +fn a_rewriting_edit_is_refused_when_the_field_walk_passes_a_signature_over() { + let (mut s, id) = open_signed_field_named(64 << 10); + assert!(matches!(s.apply(id, Edit::Sanitize), Err(EditError::SignedRewrite(_)))); +} + +#[test] +fn an_action_appends_to_a_file_whose_field_walk_passes_a_signature_over() { + let bytes = Arc::new(signed_field_named(64 << 10)); + let steps = vec![actions::Step::from_id("set_title", "Checked").unwrap()]; + let title = actions::Action { name: "Title".into(), description: String::new(), steps, builtin: false }; + let r = actions::run_on(&title, "form.pdf", bytes.clone(), |_, _| {}).unwrap(); + assert!(r.bytes.starts_with(&bytes[..]), "the action rewrote the file instead of appending to it"); + let mut s = Session::new(); + let id = s.open("out.pdf", None, r.bytes, None).unwrap(); + assert_eq!(s.get(id).unwrap().info.title.as_deref(), Some("Checked"), "the title is in what was appended"); +} + /// #1324: marking a figure decorative rewrites that figure only. The header, added text and an /// unknown entry on the original stream stay attached to the page. This lives in the engine tests /// because edit and a11y are the same layer and cannot depend on each other. @@ -10106,6 +10829,42 @@ fn need_appearances_buttons_are_displayed_from_their_captions() { } } +/// Copy and paste: the copies of a comment and its reply land on the asked-for page of another +/// document under names of their own (nothing replaced; a second paste gives a second set), +/// undo takes a paste away as one step, and a clipboard without comments is refused. +#[test] +fn pasting_copied_comments_adds_copies_that_undo_as_one_step() { + let (mut s, id) = session_with(2); + s.apply(id, rect_comment(1, [80.0, 80.0, 120.0, 120.0])).unwrap(); + s.apply(id, Edit::ReplyToAnnotation { page: 1, index: 0, text: "Done".into(), author: "Ada".into() }).unwrap(); + let xfdf = s.export_comments(id, &[(1, 0)]).unwrap(); + assert!(xfdf.contains("> 7; + seed ^= seed << 17; + seed as usize + }; + let letters: Vec> = (0..26).map(|_| (0..16).map(|_| 1 + next() % 12).collect()).collect(); + let stride = w.div_ceil(8); + let mut bits = vec![0xFFu8; stride * h]; + for top in (8..h - 40).step_by(44) { + let mut x = 8 + next() % 8; + while x + 32 < w { + let letter = &letters[next() % 26]; + for (row, run) in letter.iter().enumerate() { + for (px, py) in (x..x + 2 * run).flat_map(|px| [(px, top + 2 * row), (px, top + 2 * row + 1)]) { + bits[py * stride + px / 8] &= !(0x80 >> (px % 8)); + } + } + x += 2 * letter.iter().max().unwrap_or(&1) + 4 + next() % 5; + } + } + let mut doc = Document::new_empty(); + let pages = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|d| d.reference(b"Pages"))).unwrap(); + let device_n = Object::Array(vec![Object::name("DeviceN"), Object::Array(vec![Object::name("Black")]), Object::name("DeviceGray"), { + let mut tint = Dict::new(); + tint.set(b"FunctionType".to_vec(), Object::Int(2)); + tint.set(b"Domain".to_vec(), Object::Array(vec![Object::Int(0), Object::Int(1)])); + tint.set(b"N".to_vec(), Object::Int(1)); + Object::Dict(tint) + }]); + let mut images: Vec = Vec::new(); + let grey = || ("ColorSpace", Object::name("DeviceGray")); + for (k, (key, value)) in [grey(), grey(), ("ColorSpace", device_n), ("ImageMask", Object::Bool(true))].into_iter().enumerate() { + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("XObject")); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(w as i64)); + d.set(b"Height".to_vec(), Object::Int(h as i64)); + d.set(b"BitsPerComponent".to_vec(), Object::Int(1)); + d.set(key.as_bytes().to_vec(), value); + let image = doc.add(Object::Stream(Stream::flate(d, &bits))); + images.push(image); + let content = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"q 144 0 0 115.2 36 600 cm /Im Do Q".to_vec()))); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Parent".to_vec(), Object::Ref(pages)); + page.set(b"MediaBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 612.into(), 792.into()])); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + if k == 0 { + let mut mark = Dict::new(); + mark.set(b"Type".to_vec(), Object::name("Annot")); + mark.set(b"Subtype".to_vec(), Object::name("Redact")); + mark.set(b"Rect".to_vec(), Object::Array(vec![40.into(), 610.into(), 100.into(), 650.into()])); + let mark = doc.add(Object::Dict(mark)); + page.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(mark)])); + } + let page = doc.add(Object::Dict(page)); + doc.update_dict(pages, |d| { + let mut kids = d.get(b"Kids").and_then(|k| k.as_array().cloned()).unwrap_or_default(); + kids.push(Object::Ref(page)); + d.set(b"Count".to_vec(), Object::Int(kids.len() as i64)); + d.set(b"Kids".to_vec(), Object::Array(kids)); + }) + .unwrap(); + } + let mono = pdfcraft_optimize::MonoSettings { + compression: pdfcraft_optimize::MonoCompression::Jbig2, + downsample: false, + ..pdfcraft_optimize::MonoSettings::REDUCE + }; + pdfcraft_optimize::optimize(&mut doc, &pdfcraft_optimize::Settings { mono, ..Default::default() }).unwrap(); + let globals = |doc: &Document, image: ObjRef| match &*doc.get(image) { + Object::Stream(s) => s.dict.get(b"DecodeParms").and_then(Object::as_dict).and_then(|d| d.reference(b"JBIG2Globals")), + _ => None, + }; + let shared = globals(&doc, images[0]); + assert!(shared.is_some() && globals(&doc, images[1]) == shared && globals(&doc, images[3]) == shared, "DeviceGray and the mask share"); + assert_eq!(globals(&doc, images[2]), None, "the DeviceN page is coded alone"); + pdfcraft_redact::apply(&mut doc, None).unwrap(); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let named = saved.scan_objects().filter(|(_, o)| matches!(&**o, Object::Stream(s) if saved.jbig2_globals_ref(s).is_some())).count(); + assert_eq!(named, 0, "no image names the dictionary, which left with them"); +} + #[test] fn fill_sign_multiline_text_survives_engine_save_reopen_and_render() { let (mut session, id) = rotated_blank(0); @@ -10268,3 +11128,35 @@ fn fill_sign_multiline_text_survives_engine_save_reopen_and_render() { assert_eq!(reopened.get(again).unwrap().info.annotations[0].contents.as_deref(), Some(text), "reopening preserves all blank lines"); assert_eq!(before.rgba, render_page(&reopened, again).rgba, "the saved appearance is identical"); } + +/// A whole tree goes in as one undoable step, and a refused tree changes nothing. +#[test] +fn imported_bookmarks_undo_in_one_step() { + let (mut s, id) = session_with(2); + let node = |title: &str, page, children| pdfcraft_organize::OutlineNode { + title: title.into(), + page, + view: None, + color: None, + italic: false, + bold: false, + open: true, + children, + }; + let bookmarks = vec![node("Intro", Some(0), vec![node("Detail", Some(5), vec![])]), node("End", Some(1), vec![])]; + s.apply(id, Edit::ImportBookmarks { parent: vec![], index: 0, bookmarks: bookmarks.clone(), replace: false }).unwrap(); + assert_eq!(outline_titles(&s.get(id).unwrap().info.outline), ["Intro→1[Detail→0]", "End→2"]); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Import bookmarks")); + assert_eq!(s.outline_tree(id, &[]).unwrap().unwrap().nodes[0].title, "Intro"); + s.undo(id).unwrap(); + assert!(s.get(id).unwrap().info.outline.is_empty()); + s.apply(id, Edit::ImportBookmarks { parent: vec![], index: 0, bookmarks: bookmarks.clone(), replace: false }).unwrap(); + s.apply(id, Edit::ImportBookmarks { parent: vec![], index: 0, bookmarks: vec![node("New", Some(1), vec![])], replace: true }).unwrap(); + assert_eq!(outline_titles(&s.get(id).unwrap().info.outline), ["New→2"]); + s.undo(id).unwrap(); + assert_eq!(outline_titles(&s.get(id).unwrap().info.outline), ["Intro→1[Detail→0]", "End→2"]); + s.undo(id).unwrap(); + let err = s.apply(id, Edit::ImportBookmarks { parent: vec![4], index: 0, bookmarks, replace: false }).unwrap_err(); + assert!(err.to_string().contains("no bookmark"), "{err}"); + assert!(s.get(id).unwrap().info.outline.is_empty()); +} diff --git a/crates/engine/src/word_count.rs b/crates/engine/src/word_count.rs index feeb0a066f..c20dc8325f 100644 --- a/crates/engine/src/word_count.rs +++ b/crates/engine/src/word_count.rs @@ -78,6 +78,7 @@ fn count_inner(source: ExportSource, pages: &[usize], options: Options, mut prog tables: options.exclude_tables, code: options.exclude_code, images: options.exclude_images, + painted: false, mcids: excluded.remove(&page).unwrap_or_default(), }; let text = renderer.filtered_text(page, filter).map_err(|e| format!("page {}: {e}", page.saturating_add(1)))?.plain_text(); diff --git a/crates/engine/tests/form_date_language.rs b/crates/engine/tests/form_date_language.rs new file mode 100644 index 0000000000..1d4293e12c --- /dev/null +++ b/crates/engine/tests/form_date_language.rs @@ -0,0 +1,453 @@ +//! Form date fields name months in the app's date language (#5303). Its own test binary: the +//! names are process-wide, so other tests must not see them change, and its tests take turns. + +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use pdfcraft_cos::{Document, Object}; +use pdfcraft_engine::dates::use_form_date_language; +use pdfcraft_engine::js::JsRunner; +use pdfcraft_forms::FieldValue; +use pdfcraft_forms::af::{DateTime, Format, format_date, format_value, keystroke, parse_date}; +use pdfcraft_js::{DocInfo, Event, Limits, run}; + +/// Held by each test while it changes the process-wide date names, which start as English (a +/// failed test may have left others). +fn names_lock() -> MutexGuard<'static, ()> { + static LOCK: Mutex<()> = Mutex::new(()); + let lock = LOCK.lock().unwrap_or_else(PoisonError::into_inner); + use_form_date_language("en"); + lock +} + +/// `util.printd(fmt)` of 18 October 2026, then of what `util.scand(fmt, typed)` reads. +fn printd(fmt: &str, typed: &str) -> String { + let doc = DocInfo { file_name: "form.pdf".into(), num_pages: 1, page: 0, info: Vec::new(), xfa: false }; + let script = format!("event.value = util.printd('{fmt}', new Date(2026, 9, 18)) + '|' + util.printd('{fmt}', util.scand('{fmt}', '{typed}'));"); + let o = run(&script, &Event::field("Format", "date", ""), &doc, &[], &[], Limits::default()); + assert_eq!(o.error, None); + o.value +} + +#[test] +fn form_dates_follow_the_date_language() { + let _names = names_lock(); + let oct18 = DateTime { y: 2026, m: 10, d: 18, hh: 0, mm: 0, ss: 0 }; + let dd_mmm_yy = Format::Date("dd-mmm-yy".into()); + // English until the app says otherwise. + assert_eq!(format_date(oct18, "dd-mmm-yy dddd ddd mmmm"), "18-Oct-26 Sunday Sun October"); + use_form_date_language("it"); + // AFDate_FormatEx("dd-mmm-yy"), what the issue's form used. + assert_eq!(format_value(&dd_mmm_yy, "18-10-26"), "18-ott-26"); + assert_eq!(format_date(oct18, "dddd d mmmm yyyy, ddd"), "domenica 18 ottobre 2026, dom"); + // What it writes reads back, and so does English typing. + assert_eq!(parse_date("18-ott-26", "dd-mmm-yy"), Some(oct18)); + assert_eq!(parse_date("18 ottobre 2026", "dd mmmm yyyy"), Some(oct18)); + assert_eq!(parse_date("18-Oct-26", "dd-mmm-yy"), Some(oct18)); + assert!(keystroke(&dd_mmm_yy, "date", "18-ott-26").is_ok()); + // util.printd writes Italian names; util.scand reads them. + assert_eq!(printd("dd-mmm-yy", "18-ott-26"), "18-ott-26|18-ott-26"); + assert_eq!(printd("dd-mmm-yy", "18-Oct-26"), "18-ott-26|18-ott-26", "English typing reads too"); + assert_eq!(printd("dddd d mmmm yyyy", "domenica 18 ottobre 2026"), "domenica 18 ottobre 2026|domenica 18 ottobre 2026"); + // German short names end in a dot; they are written and read whole. + use_form_date_language("de"); + assert_eq!(format_value(&dd_mmm_yy, "18-10-26"), "18-Okt.-26"); + assert_eq!(parse_date("18-Okt.-26", "dd-mmm-yy"), Some(oct18)); + // An unknown language is English again. + use_form_date_language("xx"); + assert_eq!(format_value(&dd_mmm_yy, "18-10-26"), "18-Oct-26"); + assert_eq!(printd("dd-mmm-yy", "18-Oct-26"), "18-Oct-26|18-Oct-26"); + every_date_language_reads_back_what_it_writes(); + // Hostile typing never panics and stays quick: long, odd and mixed-script text. + use_form_date_language("th"); + let junk = ["", " ", "ม.", "ม.ค.ม.ค.ม.ค.", "\u{0}", "1月1月1月", "tháng tháng", "Ottobre", "🙂ott🙂", "İstanbul ott 26"]; + for t in junk.iter().map(|s| s.to_string()).chain(["ม.ค.1".repeat(5000), "9".repeat(100_000)]) { + let started = std::time::Instant::now(); + let _ = parse_date(&t, "dd-mmm-yy"); + let _ = keystroke(&dd_mmm_yy, "date", &t); + assert!(started.elapsed() < std::time::Duration::from_secs(2), "{} chars took {:?}", t.len(), started.elapsed()); + } + use_form_date_language("en"); +} + +/// An English weekday typed before the month fills the format's weekday, so a month name that +/// the date language shares with a weekday (Italian, Spanish, Romanian and Albanian "mar") is +/// still read as the month: "Tue 17 Mar 2026" read as "Tue 17 Tuesday 2026" (#5303). +#[test] +fn an_english_weekday_before_a_shared_month_name_reads_as_english() { + let _names = names_lock(); + let doc = DocInfo { file_name: "form.pdf".into(), num_pages: 1, page: 0, info: Vec::new(), xfa: false }; + let mar17 = DateTime { y: 2026, m: 3, d: 17, hh: 0, mm: 0, ss: 0 }; + let mut bad = Vec::new(); + for language in ["it", "es", "ro", "sq"] { + use_form_date_language(language); + let ddd_d_mmm = "ddd d mmm yyyy"; + let typed = [ + // What the language writes, then English, then English and localized mixed. + (ddd_d_mmm, format_date(mar17, ddd_d_mmm)), + (ddd_d_mmm, "Tue 17 Mar 2026".into()), + (ddd_d_mmm, "Tuesday 17 Mar 2026".into()), + (ddd_d_mmm, "tue 17 mar 2026".into()), + (ddd_d_mmm, format!("Tue 17 {}", format_date(mar17, "mmm yyyy"))), + (ddd_d_mmm, format!("{} 17 Mar 2026", format_date(mar17, "ddd"))), + ("mmm d yyyy ddd", "Mar 17 2026 Tue".into()), + ("dddd d mmmm yyyy", "Tuesday 17 March 2026".into()), + ]; + for (fmt, text) in typed { + if parse_date(&text, fmt) != Some(mar17) { + bad.push(format!("{language} forms {fmt}: {text:?} -> {:?}", parse_date(&text, fmt))); + } + let script = format!("var d = util.scand('{fmt}', {text:?}); event.value = d ? util.printd('yyyy-mm-dd', d) : 'null';"); + let o = run(&script, &Event::field("Format", "date", ""), &doc, &[], &[], Limits::default()); + if o.value != "2026-03-17" { + bad.push(format!("{language} scand {fmt}: {text:?} -> {:?} {:?}", o.value, o.error)); + } + } + } + use_form_date_language("en"); + assert!(bad.is_empty(), "{}", bad.join("; ")); +} + +/// Every date language: what a date field writes reads back as the same date, for each date +/// letter that writes a name, through the form path and through `util.scand`. Called from the +/// one test, which owns the process-wide names. +fn every_date_language_reads_back_what_it_writes() { + let doc = DocInfo { file_name: "form.pdf".into(), num_pages: 1, page: 0, info: Vec::new(), xfa: false }; + let mut bad = Vec::new(); + for l in pdfcraft_engine::dates::DATE_LANGUAGES.iter() { + use_form_date_language(l.code); + for m in 1..=12u32 { + let dt = DateTime { y: 2026, m, d: 18, hh: 0, mm: 0, ss: 0 }; + for fmt in ["dd-mmm-yy", "d mmmm yyyy", "dddd, mmmm d, yyyy", "ddd d mmm yyyy", "mmm d, yyyy ddd", "yyyymmmdd"] { + let text = format_date(dt, fmt); + // A short month ending in a number ("Thg 1") glued to the day is ambiguous + // ("Thg 118": January 18 or November 8): nothing can read it back. + if fmt == "yyyymmmdd" && l.months_short.iter().any(|n| n.ends_with(|c: char| c.is_ascii_digit())) { + continue; + } + if parse_date(&text, fmt).map(|p| (p.y, p.m, p.d)) != Some((2026, m, 18)) { + bad.push(format!("{} forms {fmt}: {text:?}", l.code)); + } + let script = format!("var d = util.scand('{fmt}', {text:?}); event.value = d ? util.printd('yyyy-mm-dd', d) : 'null';"); + let o = run(&script, &Event::field("Format", "date", ""), &doc, &[], &[], Limits::default()); + if o.value != format!("2026-{m:02}-18") { + bad.push(format!("{} scand {fmt}: {text:?} -> {:?} {:?}", l.code, o.value, o.error)); + } + } + } + } + use_form_date_language("en"); + assert!(bad.is_empty(), "{} failures:\n{}", bad.len(), bad.iter().take(40).cloned().collect::>().join("\n")); +} + +/// A form whose text fields are each `(name, font, format script)`: `/Helv` is Helvetica with +/// WinAnsi, `/Song` a Chinese font with a Unicode CMap, which also holds Cyrillic. +fn form(fields: &[(&str, &str, &str)]) -> Document { + let n = fields.len(); + let first = 6; + let refs = |from: usize| (from..from + n).map(|i| format!("{i} 0 R")).collect::>().join(" "); + let mut objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{}] >>", refs(first)), + format!("<< /Fields [{}] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R /Song {} 0 R >> >> >>", refs(first), first + n), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + ]; + for (i, (name, font, js)) in fields.iter().enumerate() { + let js = js.replace('\\', "\\\\").replace('(', "\\(").replace(')', "\\)"); + let y = 700 - 30 * i; + objs.push(format!( + "<< /Type /Annot /Subtype /Widget /FT /Tx /T ({name}) /DA (/{font} 10 Tf 0 g) /Rect [50 {y} 300 {}] /P 3 0 R /AA << /F << /S /JavaScript /JS ({js}) >> >> >>", + y + 20 + )); + } + objs.push("<< /Type /Font /Subtype /Type0 /BaseFont /STSong-Light /Encoding /UniGB-UCS2-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType0 /BaseFont /STSong-Light /CIDSystemInfo << /Registry (Adobe) /Ordering (GB1) /Supplement 4 >> >>] >>".into()); + let mut out = b"%PDF-1.7 +" + .to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice( + format!( + "{} 0 obj +{o} +endobj +", + i + 1 + ) + .as_bytes(), + ); + } + let xref = out.len(); + out.extend_from_slice( + format!( + "xref +0 {} +0000000000 65535 f +", + objs.len() + 1 + ) + .as_bytes(), + ); + for o in offsets { + out.extend_from_slice( + format!( + "{o:010} 00000 n +" + ) + .as_bytes(), + ); + } + out.extend_from_slice( + format!( + "trailer +<< /Size {} /Root 1 0 R >> +startxref +{xref} +%%EOF +", + objs.len() + 1 + ) + .as_bytes(), + ); + Document::open(Arc::new(out)).expect("opens") +} + +/// The content of field `name`'s normal appearance. +fn appearance(doc: &Document, name: &str) -> Vec { + let all = pdfcraft_forms::fields(doc); + let f = all.iter().find(|f| f.name == name).expect("field"); + let widget = doc.get(f.widgets[0].obj); + let normal = widget.as_dict().and_then(|d| d.get(b"AP")).and_then(|a| doc.resolve(a).as_dict().and_then(|a| a.reference(b"N"))).expect("/AP /N"); + match &*doc.get(normal) { + Object::Stream(s) => s.decoded().expect("decodes"), + other => panic!("not a stream: {other:?}"), + } +} + +/// What a Unicode-CMap (UCS-2) font is given for `text`. +fn ucs2(text: &str) -> Vec { + text.encode_utf16().flat_map(u16::to_be_bytes).collect() +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()).any(|w| w == needle) +} + +/// Fills `form`'s field `name` with 18 October 2026 through the JavaScript runner. +fn fill(doc: &mut Document, name: &str) { + fill_with(doc, name, "18-10-26"); +} + +/// Fills `form`'s field `name` with `value` through the JavaScript runner. +fn fill_with(doc: &mut Document, name: &str, value: &str) { + let mut js = JsRunner::new(doc, "form.pdf"); + pdfcraft_forms::set_value_with(doc, name, &FieldValue::Text(value.into()), &mut js).expect("fills"); +} + +/// Month names in the date language that no font of the field can draw are drawn in English, +/// never as `?` ("18-????.-26" in Russian); a font that holds them keeps them (#5303). +#[test] +fn date_names_no_field_font_can_draw_are_drawn_in_english() { + let _names = names_lock(); + let date = "AFDate_FormatEx(\"dd-mmm-yy\");"; + let fields = [("helv", "Helv", date), ("song", "Song", date)]; + for (language, local) in [("ru", "окт."), ("cs", "říj"), ("ja", "10月"), ("el", "Οκτ")] { + use_form_date_language(language); + let mut doc = form(&fields); + fill(&mut doc, "helv"); + fill(&mut doc, "song"); + let helv = appearance(&doc, "helv"); + assert!(contains(&helv, b"(18-Oct-26) Tj"), "{language}: {}", String::from_utf8_lossy(&helv)); + assert!(!helv.contains(&b'?'), "{language}: {}", String::from_utf8_lossy(&helv)); + assert!(pdfcraft_forms::undrawn_fields(&doc).is_empty(), "{language}"); + let song = appearance(&doc, "song"); + assert!(contains(&song, &ucs2(&format!("18-{local}-26"))), "{language} keeps its name: {song:?}"); + } + // Names Helvetica holds are kept: Italian "ott". + use_form_date_language("it"); + let mut doc = form(&fields); + fill(&mut doc, "helv"); + assert!(contains(&appearance(&doc, "helv"), b"(18-ott-26) Tj")); + use_form_date_language("en"); +} + +/// A Format script's `util.printd` is run again with English names for a field whose fonts +/// can't draw the date language's, and kept where they can (#5303). +#[test] +fn format_scripts_print_dates_in_english_where_the_field_cannot_draw_them() { + let _names = names_lock(); + let script = "event.value = event.value ? util.printd(\"dd-mmm-yy\", util.scand(\"dd-mm-yy\", event.value)) + \" !\" : \"\";"; + let fields = [("helv", "Helv", script), ("song", "Song", script)]; + use_form_date_language("ru"); + let mut doc = form(&fields); + fill(&mut doc, "helv"); + fill(&mut doc, "song"); + let helv = appearance(&doc, "helv"); + assert!(contains(&helv, b"(18-Oct-26 !) Tj"), "{}", String::from_utf8_lossy(&helv)); + assert!(!helv.contains(&b'?'), "{}", String::from_utf8_lossy(&helv)); + assert!(pdfcraft_forms::undrawn_fields(&doc).is_empty()); + assert!(contains(&appearance(&doc, "song"), &ucs2("18-окт.-26 !"))); + // A script whose text has no English either keeps it, reported as undrawn. + let greeting = [("hi", "Helv", "event.value = \"Привет \" + util.printd(\"mmm\", new Date(2026, 9, 18));")]; + let mut doc = form(&greeting); + fill(&mut doc, "hi"); + assert_eq!(pdfcraft_forms::undrawn_fields(&doc), [("hi".to_string(), 'П')]); + // The English run is the appearance's alone: scripts go on printing the date language. + assert_eq!(printd("dd-mmm-yy", "18-окт.-26"), "18-окт.-26|18-окт.-26"); + use_form_date_language("en"); +} + +/// A Format script's English run is kept when it ran, even where it prints the stored value +/// again: a Helvetica field storing "18-Oct-26", printed again as `dd-mmm-yy`, showed +/// "18-????.-26" in Russian. A script that fails keeps what it showed, reported undrawn (#5303). +#[test] +fn an_english_run_that_prints_the_stored_value_is_drawn() { + let _names = names_lock(); + let script = "event.value = util.printd(\"dd-mmm-yy\", util.scand(\"dd-mmm-yy\", event.value));"; + use_form_date_language("ru"); + let mut doc = form(&[("helv", "Helv", script), ("fails", "Helv", "throw new Error(\"no\");")]); + fill_with(&mut doc, "helv", "18-Oct-26"); + let helv = appearance(&doc, "helv"); + assert!(contains(&helv, b"(18-Oct-26) Tj"), "{}", String::from_utf8_lossy(&helv)); + assert!(!helv.contains(&b'?'), "{}", String::from_utf8_lossy(&helv)); + assert!(pdfcraft_forms::undrawn_fields(&doc).is_empty()); + fill_with(&mut doc, "fails", "18-окт.-26"); + assert_eq!(pdfcraft_forms::undrawn_fields(&doc), [("fails".to_string(), 'о')]); + use_form_date_language("en"); +} + +/// What `util.scand(fmt, text)` reads, as `yyyy-mm-dd`, or `null`. +fn scanned(fmt: &str, text: &str) -> String { + let doc = DocInfo { file_name: "form.pdf".into(), num_pages: 1, page: 0, info: Vec::new(), xfa: false }; + let script = format!("var d = util.scand('{fmt}', {text:?}); event.value = d ? util.printd('yyyy-mm-dd', d) : 'null';"); + let o = run(&script, &Event::field("Format", "date", ""), &doc, &[], &[], Limits::default()); + assert_eq!(o.error, None); + o.value +} + +/// A short weekday typed without its dot ("mar" for French "mar.", Tuesday) is the weekday where +/// the format has one, not the English "Mar" (March) that is the only other name of its length: +/// "mar 6 janv 2026" read as no date in French and Romanian (#5303). +#[test] +fn a_short_weekday_typed_without_its_dot_is_the_weekday() { + let _names = names_lock(); + let mut bad = Vec::new(); + let cases = [ + ("fr", "ddd d mmm yyyy", "Mar 6 janv 2026", "2026-01-06"), + ("fr", "ddd d mmm yyyy", "mar 17 mars 2026", "2026-03-17"), + ("fr", "mmm d yyyy ddd", "janv 6 2026 mar", "2026-01-06"), + ("fr", "ddd dd/mm/yyyy", "mar 06/01/2026", "2026-01-06"), + ("ro", "ddd d mmm yyyy", "mar 6 ian 2026", "2026-01-06"), + ("ro", "ddd dd/mm/yyyy", "mar 06/01/2026", "2026-01-06"), + ]; + for (language, fmt, text, want) in cases { + use_form_date_language(language); + let got = scanned(fmt, text); + if got != want { + bad.push(format!("{language} scand {fmt}: {text:?} -> {got}")); + } + let read = parse_date(text, fmt).map(|d| format!("{:04}-{:02}-{:02}", d.y, d.m, d.d)); + if read.as_deref() != Some(want) { + bad.push(format!("{language} forms {fmt}: {text:?} -> {read:?}")); + } + } + use_form_date_language("en"); + assert!(bad.is_empty(), "{}", bad.join("; ")); +} + +/// A weekday left out of a format with one ("17 mar 2026" for "ddd d mmm yyyy") lets a name the +/// date language shares between a weekday and a month be the month: Italian, Spanish and +/// Albanian "mar" (Tuesday, March) and Albanian "sht" (Saturday, September) were read as the +/// weekday, leaving no month (#5303). Typed weekdays still go first. +#[test] +fn a_shared_name_is_the_month_when_the_weekday_is_left_out() { + let _names = names_lock(); + let mut bad = Vec::new(); + let cases = [ + ("it", "ddd d mmm yyyy", "17 Mar 2026", (2026, 3, 17)), + ("es", "ddd d mmm yyyy", "17 Mar 2026", (2026, 3, 17)), + ("sq", "ddd d mmm yyyy", "17 Mar 2026", (2026, 3, 17)), + ("it", "ddd d mmm yyyy", "17 mar 2026", (2026, 3, 17)), + ("it", "dddd, mmmm d, yyyy", "mar 17, 2026", (2026, 3, 17)), + ("it", "dddd, mmm d, yyyy", "mar 17 2026", (2026, 3, 17)), + ("sq", "ddd d mmm yyyy", "5 sht 2026", (2026, 9, 5)), + ("it", "ddd d mmm yyyy", "mar 17 Mar 2026", (2026, 3, 17)), + ("it", "ddd d mmm yyyy", "mar 17 mar 2026", (2026, 3, 17)), + ]; + for (language, fmt, text, want) in cases { + use_form_date_language(language); + let read = parse_date(text, fmt).map(|d| (d.y, d.m, d.d)); + if read != Some(want) { + bad.push(format!("{language} forms {fmt}: {text:?} -> {read:?}")); + } + if keystroke(&Format::Date(fmt.into()), "date", text).is_err() { + bad.push(format!("{language} keystroke {fmt}: {text:?} refused")); + } + } + use_form_date_language("en"); + assert!(bad.is_empty(), "{}", bad.join("; ")); +} + +/// Turkish names typed in capitals are read: lower-casing letter by letter made "MAYIS" "mayis" +/// (not "mayıs"), "SALI" "sali" and "EKİM" "eki̇m" (an i with a dot above), so no name matched. +/// Dotted, dotless and capital i are one letter when names are read (#5303). +#[test] +fn turkish_names_typed_in_capitals_are_read() { + let _names = names_lock(); + use_form_date_language("tr"); + let mut bad = Vec::new(); + for (fmt, text, want) in [ + ("d mmmm yyyy", "1 MAYIS 2026", (2026, 5, 1)), + ("d mmmm yyyy", "1 EKİM 2026", (2026, 10, 1)), + ("d mmmm yyyy", "1 Ekim 2026", (2026, 10, 1)), + ("d mmm yyyy", "1 EKİ 2026", (2026, 10, 1)), + ("dddd d mmmm yyyy", "SALI 6 OCAK 2026", (2026, 1, 6)), + ("dddd d mmmm yyyy", "Salı 6 Ocak 2026", (2026, 1, 6)), + ] { + let read = parse_date(text, fmt).map(|d| (d.y, d.m, d.d)); + if read != Some(want) { + bad.push(format!("forms {fmt}: {text:?} -> {read:?}")); + } + let (y, m, d) = want; + let got = scanned(fmt, text); + if got != format!("{y:04}-{m:02}-{d:02}") { + bad.push(format!("scand {fmt}: {text:?} -> {got}")); + } + } + // English is read as before, capitals and all. + use_form_date_language("en"); + for (fmt, text) in [("d mmmm yyyy", "1 APRIL 2026"), ("dddd d mmmm yyyy", "FRIDAY 1 MAY 2026"), ("d mmm yyyy", "1 MAY 2026")] { + if parse_date(text, fmt).is_none() || scanned(fmt, text) == "null" { + bad.push(format!("en {fmt}: {text:?}")); + } + } + assert!(bad.is_empty(), "{}", bad.join("; ")); +} + +/// #5464 review: Greek names typed in capitals end in a capital Σ, which is the final ς of the +/// name ("ΙΑΝΟΥΆΡΙΟΣ" for "Ιανουάριος"); they read as they did before names were folded. +#[test] +fn greek_names_typed_in_capitals_are_read() { + let _names = names_lock(); + use_form_date_language("el"); + let mut bad = Vec::new(); + for (fmt, text, want) in [ + ("d mmmm yyyy", "1 ΙΑΝΟΥΆΡΙΟΣ 2026", (2026, 1, 1)), + ("d mmmm yyyy", "1 Ιανουάριος 2026", (2026, 1, 1)), + ("d mmmm yyyy", "1 ιανουάριοσ 2026", (2026, 1, 1)), + ("mmmm d, yyyy", "ΜΆΙΟΣ 5, 2026", (2026, 5, 5)), + ("dddd, mmmm d, yyyy", "FRIDAY, ΙΑΝΟΥΆΡΙΟΣ 2, 2026", (2026, 1, 2)), + ] { + let read = parse_date(text, fmt).map(|d| (d.y, d.m, d.d)); + if read != Some(want) { + bad.push(format!("forms {fmt}: {text:?} -> {read:?}")); + } + let (y, m, d) = want; + let got = scanned(fmt, text); + if got != format!("{y:04}-{m:02}-{d:02}") { + bad.push(format!("scand {fmt}: {text:?} -> {got}")); + } + } + assert!(bad.is_empty(), "{}", bad.join("; ")); +} diff --git a/crates/engine/tests/it/ocr_edit.rs b/crates/engine/tests/it/ocr_edit.rs index 8009ef1bbf..2d358e327e 100644 --- a/crates/engine/tests/it/ocr_edit.rs +++ b/crates/engine/tests/it/ocr_edit.rs @@ -12,7 +12,14 @@ const WORD: [u32; 4] = [40, 40, 100, 60]; /// A 200 × 100 pt "scan" on tinted paper, with OCR text over its word. fn scanned() -> (Session, pdfcraft_engine::DocId) { - scanned_with(vec![PlacedWord { text: "John".into(), origin: [40.0, 40.0], across: [60.0, 0.0], up: [0.0, 20.0] }]) + scanned_with(vec![PlacedWord { + text: "John".into(), + origin: [40.0, 40.0], + across: [60.0, 0.0], + up: [0.0, 20.0], + confidence: None, + source: "ocrs".into(), + }]) } fn scanned_with(words: Vec) -> (Session, pdfcraft_engine::DocId) { @@ -86,11 +93,18 @@ fn an_explicit_colour_is_kept_for_the_new_text() { /// descender line when it has descenders, up to the cap height. fn scanned_word(text: &str, x0: f64, x1: f64, baseline: f64, size: f64) -> PlacedWord { let low = if text.chars().any(|c| "gjpqy".contains(c)) { 0.21 * size } else { 0.0 }; - PlacedWord { text: text.into(), origin: [x0, baseline - low], across: [x1 - x0, 0.0], up: [0.0, 0.718 * size + low] } + PlacedWord { + text: text.into(), + origin: [x0, baseline - low], + across: [x1 - x0, 0.0], + up: [0.0, 0.718 * size + low], + confidence: None, + source: "ocrs".into(), + } } fn word(text: &str, x0: f64, x1: f64, baseline: f64) -> PlacedWord { - PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0] } + PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0], confidence: None, source: "ocrs".into() } } /// OCR words keep the scan's layout: a phrase is one paragraph, a value in the next table cell and @@ -207,3 +221,192 @@ fn visible_stretched_text_rewrites_unchanged() { let i = ((y * out.width + x) * 4) as usize; assert_eq!(&out.rgba[i..i + 3], &[255, 255, 255]); } + +/// A scan whose OCR layer is drawn first and the opaque page image over it, as some OCR tools +/// write it (#3973): invisible Helvetica `text` on the scanned word, then the image. +fn ocr_under_scan(text: &str) -> std::sync::Arc> { + under_scan(&format!("BT /F1 20 Tf 3 Tr 42 44 Td ({text}) Tj ET")) +} + +/// A page whose `text` content is followed by the opaque 200 × 100 scan image. +fn under_scan(text: &str) -> std::sync::Arc> { + under_scan_with(text, |_| pdfcraft_cos::Dict::new()) +} + +/// As [`under_scan`], with the page's ExtGState resources made by `ext_gstates`. +fn under_scan_with(text: &str, ext_gstates: impl FnOnce(&mut pdfcraft_cos::Document) -> pdfcraft_cos::Dict) -> std::sync::Arc> { + use pdfcraft_cos::{Dict, Document, Object, SaveOptions, Stream}; + let (w, h) = (200u32, 100u32); + let mut rgb = Vec::new(); + for y in 0..h { + for x in 0..w { + let ink = (WORD[0]..WORD[2]).contains(&x) && (WORD[1]..WORD[3]).contains(&y); + rgb.extend(if ink { INK } else { PAPER }); + } + } + let mut d = Document::new_empty(); + let pages = d.add(Object::Dict(Dict::new())); + let content = format!("{text} q 200 0 0 100 0 0 cm /Im Do Q"); + let contents = d.add(Object::Stream(Stream::flate(Dict::new(), content.as_bytes()))); + let mut image = Dict::new(); + image.set(b"Type".to_vec(), Object::name("XObject")); + image.set(b"Subtype".to_vec(), Object::name("Image")); + image.set(b"Width".to_vec(), Object::Int(i64::from(w))); + image.set(b"Height".to_vec(), Object::Int(i64::from(h))); + image.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + image.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let image = d.add(Object::Stream(Stream::flate(image, &rgb))); + let mut font = Dict::new(); + font.set(b"Type".to_vec(), Object::name("Font")); + font.set(b"Subtype".to_vec(), Object::name("Type1")); + font.set(b"BaseFont".to_vec(), Object::name("Helvetica")); + let mut fonts = Dict::new(); + fonts.set(b"F1".to_vec(), Object::Dict(font)); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut res = Dict::new(); + res.set(b"Font".to_vec(), Object::Dict(fonts)); + res.set(b"XObject".to_vec(), Object::Dict(xobjects)); + res.set(b"ExtGState".to_vec(), Object::Dict(ext_gstates(&mut d))); + let mut p = Dict::new(); + p.set(b"Type".to_vec(), Object::name("Page")); + p.set(b"Parent".to_vec(), Object::Ref(pages)); + p.set(b"Contents".to_vec(), Object::Ref(contents)); + p.set(b"Resources".to_vec(), Object::Dict(res)); + p.set(b"MediaBox".to_vec(), Object::Array([0, 0, 200, 100].map(Object::Int).to_vec())); + let page = d.add(Object::Dict(p)); + d.update_dict(pages, |p| { + p.set(b"Type".to_vec(), Object::name("Pages")); + p.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(page)])); + p.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + let mut root = Dict::new(); + root.set(b"Type".to_vec(), Object::name("Catalog")); + root.set(b"Pages".to_vec(), Object::Ref(pages)); + let root = d.add(Object::Dict(root)); + d.trailer_mut().set(b"Root".to_vec(), Object::Ref(root)); + std::sync::Arc::new(pdfcraft_cos::write_full(&d, &SaveOptions::default()).unwrap()) +} + +/// The scanned word's box shows the cover and new ink, not the scan. +fn assert_edit_shows(s: &Session, id: pdfcraft_engine::DocId, label: &str) { + let px = word_pixels(s, id); + let paper = px.iter().filter(|p| near(**p, PAPER)).count(); + let ink = px.iter().filter(|p| near(**p, INK)).count(); + assert!(paper * 10 > px.len() * 4, "{label}: the cover is drawn over the scan: {paper} of {} paper", px.len()); + assert!(ink * 100 > px.len() * 3, "{label}: the new text shows: {ink} of {} ink", px.len()); +} + +fn texts(s: &Session, id: pdfcraft_engine::DocId) -> Vec { + s.get(id).unwrap().text_blocks(0).iter().map(|b| b.text.replace('\n', " ")).collect() +} + +fn edit_block(s: &mut Session, id: pdfcraft_engine::DocId, block: usize, text: &str) { + s.apply(id, Edit::EditTextBlock { page: 0, block, text: text.into(), style: Default::default() }).unwrap(); +} + +/// The OCR layer is under the scan image: the rewritten paragraph is drawn on top of the image, +/// not in the OCR text's place where the image would hide it (#3973). One line takes the +/// single-style rewrite, a typed line break the styled one. Undo restores the hidden layer, and +/// the edit survives redo, a save and reopen, and being edited again. +#[test] +fn an_ocr_layer_under_the_scan_image_shows_its_edit() { + for edit in ["Jane", "Jane\nDoe"] { + let mut s = Session::new(); + let id = s.open("ocr-under-scan.pdf", None, ocr_under_scan("John"), None).unwrap(); + let blocks = s.get(id).unwrap().text_blocks(0); + assert_eq!(blocks.len(), 1); + assert!(blocks[0].invisible, "the OCR layer is invisible text"); + assert!(word_pixels(&s, id).iter().all(|p| near(*p, INK)), "the scanned word fills its box"); + + edit_block(&mut s, id, 0, edit); + assert_eq!(texts(&s, id), [edit.replace('\n', " ")], "{edit:?}"); + assert_edit_shows(&s, id, edit); + + s.undo(id).unwrap(); + assert_eq!(texts(&s, id), ["John"], "{edit:?}: undo"); + assert!(s.get(id).unwrap().text_blocks(0)[0].invisible, "{edit:?}: undo hides the layer again"); + assert!(word_pixels(&s, id).iter().all(|p| near(*p, INK)), "{edit:?}: undo shows the scan again"); + s.redo(id).unwrap(); + assert_edit_shows(&s, id, &format!("{edit:?} redo")); + + let saved = s.save_bytes(id).unwrap(); + let mut reopened = Session::new(); + let id = reopened.open("saved.pdf", None, saved, None).unwrap(); + assert_eq!(texts(&reopened, id), [edit.replace('\n', " ")], "{edit:?}: reopened"); + assert_edit_shows(&reopened, id, &format!("{edit:?} reopened")); + + edit_block(&mut reopened, id, 0, "Ann"); + assert_eq!(texts(&reopened, id), ["Ann"], "{edit:?}: edited again"); + assert_edit_shows(&reopened, id, &format!("{edit:?} edited again")); + } +} + +/// Visible text earlier in the same text object makes the rewrite split it; the OCR phrase still +/// goes on top of the scan, and the visible text stays. +#[test] +fn an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit() { + for edit in ["Jane", "Jane\nDoe"] { + let mut s = Session::new(); + let page = under_scan("BT /F1 20 Tf 10 80 Td (Head) Tj 3 Tr 32 -36 Td (John) Tj ET"); + let id = s.open("split.pdf", None, page, None).unwrap(); + let blocks = s.get(id).unwrap().text_blocks(0); + let john = blocks.iter().position(|b| b.text == "John").unwrap(); + assert!(blocks[john].invisible && blocks.iter().any(|b| b.text == "Head" && !b.invisible), "{blocks:?}"); + edit_block(&mut s, id, john, edit); + let mut after = texts(&s, id); + after.sort(); + let mut want = vec!["Head".to_string(), edit.replace('\n', " ")]; + want.sort(); + assert_eq!(after, want, "{edit:?}"); + assert_edit_shows(&s, id, &format!("{edit:?} split")); + } +} + +/// `/SM`: a luminosity soft mask that is white over 30..110 × 30..70 in the space it is selected +/// in, black elsewhere. +fn soft_mask_state(d: &mut pdfcraft_cos::Document) -> pdfcraft_cos::Dict { + use pdfcraft_cos::{Dict, Object, Stream}; + let mut group = Dict::new(); + group.set(b"S".to_vec(), Object::name("Transparency")); + group.set(b"CS".to_vec(), Object::name("DeviceRGB")); + let mut form = Dict::new(); + form.set(b"Type".to_vec(), Object::name("XObject")); + form.set(b"Subtype".to_vec(), Object::name("Form")); + form.set(b"BBox".to_vec(), Object::Array([0, 0, 200, 100].map(Object::Int).to_vec())); + form.set(b"Group".to_vec(), Object::Dict(group)); + let mut mask = Dict::new(); + mask.set(b"Type".to_vec(), Object::name("Mask")); + mask.set(b"S".to_vec(), Object::name("Luminosity")); + mask.set(b"G".to_vec(), Object::Ref(d.add(Object::Stream(Stream::flate(form, b"1 g 30 30 80 40 re f"))))); + let mut state = Dict::new(); + state.set(b"Type".to_vec(), Object::name("ExtGState")); + state.set(b"SMask".to_vec(), Object::Dict(mask)); + let mut states = Dict::new(); + states.set(b"SM".to_vec(), Object::Dict(state)); + states +} + +/// The soft mask is selected before a translation, so it is placed by the CTM before it. The +/// paragraph lifted over the scan selects it again after its own `cm`: there it must be placed +/// as it was, not 100 pt to the right where it would hide the new text, so the edit renders +/// exactly as the same edit on the page without the mask. Both rewrite paths. +#[test] +fn a_soft_mask_selected_before_a_cm_stays_put_when_the_edit_is_lifted() { + for edit in ["Jane", "Jane\nDoe"] { + let mut plain = Session::new(); + let page = under_scan("q 1 0 0 1 100 0 cm BT /F1 20 Tf 3 Tr -58 44 Td (John) Tj ET Q"); + let plain_id = plain.open("plain.pdf", None, page, None).unwrap(); + edit_block(&mut plain, plain_id, 0, edit); + let mut s = Session::new(); + let page = under_scan_with("q /SM gs 1 0 0 1 100 0 cm BT /F1 20 Tf 3 Tr -58 44 Td (John) Tj ET Q", soft_mask_state); + let id = s.open("masked.pdf", None, page, None).unwrap(); + assert_eq!(texts(&s, id), ["John"]); + edit_block(&mut s, id, 0, edit); + assert_eq!(texts(&s, id), [edit.replace('\n', " ")], "{edit:?}"); + assert_edit_shows(&s, id, &format!("{edit:?} masked")); + let ink = |s: &Session, id| word_pixels(s, id).iter().filter(|p| near(**p, INK)).count(); + assert_eq!(ink(&s, id), ink(&plain, plain_id), "{edit:?}: the mask does not hide the new text"); + } +} diff --git a/crates/export/src/clip.rs b/crates/export/src/clip.rs new file mode 100644 index 0000000000..eeac6d2ddb --- /dev/null +++ b/crates/export/src/clip.rs @@ -0,0 +1,433 @@ +//! Copy with formatting: selected text as paragraphs of styled spans, written as an HTML +//! fragment for the clipboard (inline styles, so a word processor keeps font, size, weight, +//! slant and colour when it pastes). + +use crate::as_html::allowed; +use crate::{esc, hex}; + +/// Spans kept for one copy; text past it is left out and [`ClipText::truncated`] is set. +pub const MAX_CLIP_SPANS: usize = 20_000; +/// Text bytes kept for one copy (before escaping). +pub const MAX_CLIP_TEXT_BYTES: usize = 2 << 20; +/// The HTML written for one copy never passes this many bytes. +pub const MAX_CLIP_HTML_BYTES: usize = 16 << 20; +/// Font sizes are written within this range (points), so a hostile size stays a font size. +const CLIP_SIZE_PT: std::ops::RangeInclusive = 1.0..=1000.0; +/// A family name longer than this is not a family name. +const MAX_FAMILY_CHARS: usize = 64; + +/// How a span looks. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipStyle { + /// Family names to try in order ([`font_family`]); empty keeps the target's own. + pub families: Vec, + /// Size in points; zero or not finite leaves the target's size, and any other is written + /// within 1 to 1000. + pub size: f64, + pub bold: bool, + pub italic: bool, + /// Fill colour as RGB in 0..=1. + pub color: [f64; 3], +} + +/// A run of text in one style. +#[derive(Clone, Debug, PartialEq)] +pub struct ClipSpan { + pub text: String, + pub style: ClipStyle, +} + +/// The copied text: paragraphs of spans, and whether a cap left some out. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipText { + pub paragraphs: Vec>, + pub truncated: bool, +} + +/// Builds a [`ClipText`] piece by piece, merging pieces of equal style into one span and +/// stopping at the caps. +#[derive(Debug)] +pub struct ClipBuilder { + out: ClipText, + /// The separator owed before the next piece in this paragraph (a space or a line break). + pending: Option, + new_paragraph: bool, + spans: usize, + bytes: usize, +} + +impl Default for ClipBuilder { + fn default() -> Self { + Self { out: ClipText::default(), pending: None, new_paragraph: true, spans: 0, bytes: 0 } + } +} + +impl ClipBuilder { + pub fn new() -> Self { + Self::default() + } + + /// A word space before the next piece (dropped at the start of a paragraph). + pub fn space(&mut self) { + self.pending.get_or_insert(' '); + } + + /// A line break inside the paragraph before the next piece. + pub fn line_break(&mut self) { + self.pending = Some('\n'); + } + + /// The next piece starts a new paragraph. + pub fn paragraph(&mut self) { + self.new_paragraph = true; + self.pending = None; + } + + /// Append `text` in `style`. Returns `false` once a cap is reached; nothing more is kept. + pub fn push(&mut self, text: &str, style: &ClipStyle) -> bool { + if self.out.truncated { + return false; + } + if text.is_empty() { + return true; + } + let sep = if self.new_paragraph { None } else { self.pending }; + let need = text.len().saturating_add(sep.map_or(0, char::len_utf8)); + let Some(bytes) = self.bytes.checked_add(need).filter(|b| *b <= MAX_CLIP_TEXT_BYTES) else { + self.out.truncated = true; + return false; + }; + let new_span = self.new_paragraph || self.out.paragraphs.last().and_then(|p| p.last()).is_none_or(|last| last.style != *style); + if new_span && self.spans >= MAX_CLIP_SPANS { + self.out.truncated = true; + return false; + } + if self.new_paragraph { + self.out.paragraphs.push(Vec::new()); + self.new_paragraph = false; + } + self.pending = None; + let Some(spans) = self.out.paragraphs.last_mut() else { return false }; + match spans.last_mut() { + Some(last) if last.style == *style => { + last.text.extend(sep); + last.text.push_str(text); + } + last => { + // A separator belongs to the span before it, so a new style starts at its text. + if let (Some(last), Some(sep)) = (last, sep) { + last.text.push(sep); + } + spans.push(ClipSpan { text: text.to_owned(), style: style.clone() }); + self.spans += 1; + } + } + self.bytes = bytes; + true + } + + pub fn finish(self) -> ClipText { + self.out + } +} + +/// The family names to try for a font's `/BaseFont`, best first: subset tag and style suffix +/// removed, `PSMT`/`MT`/`PS` dropped. The first has camel case and hyphens spaced +/// (`ABCDEF+TimesNewRomanPSMT` → `Times New Roman`, `MS-Gothic` → `MS Gothic`); the second, +/// when it differs, is the name as spelt (`SimSun`). A hyphen ends the family only before a +/// style word (`Arial-BoldMT`). Only letters, digits, spaces, `-`, `_` and `.` are kept, so a +/// name cannot leave its CSS string. Empty when nothing is left. +pub fn font_family(base_font: &str) -> Vec { + let name = match base_font.split_once('+') { + Some((tag, rest)) if tag.len() == 6 && tag.bytes().all(|b| b.is_ascii_uppercase()) => rest, + _ => base_font, + }; + let name = name.split_once(',').map_or(name, |(family, _)| family); + let family = match name.rsplit_once('-') { + Some((family, style)) if STYLE_WORDS.iter().any(|w| style.starts_with(w)) => family, + _ => name, + }; + let family = family.strip_suffix("PSMT").or_else(|| family.strip_suffix("MT")).or_else(|| family.strip_suffix("PS")).unwrap_or(family); + let mut spaced = String::with_capacity(family.len().min(MAX_FAMILY_CHARS * 2)); + let mut prev_lower = false; + for c in family.chars().take(MAX_FAMILY_CHARS) { + if prev_lower && c.is_ascii_uppercase() { + spaced.push(' '); + } + prev_lower = c.is_ascii_lowercase(); + spaced.push(if c == '-' { ' ' } else { c }); + } + let spelt: String = family.chars().take(MAX_FAMILY_CHARS).collect(); + let mut out: Vec = Vec::with_capacity(2); + for f in [safe_family(&spaced), safe_family(&spelt)].into_iter().flatten() { + if !out.contains(&f) { + out.push(f); + } + } + out +} + +/// What may follow a hyphen in a `/BaseFont` when the rest names the style, not the family. +const STYLE_WORDS: &[&str] = &[ + "Bold", "Italic", "Oblique", "Regular", "Roman", "Light", "Medium", "Semi", "Demi", "Black", "Heavy", "Thin", "Book", "Cond", "Extra", "Ultra", + "Narrow", "Plain", "Normal", "It", "Bd", "Lt", "Md", "Blk", "MT", "PS", +]; + +/// `family` with only letters, digits, spaces, `-`, `_` and `.`, so it cannot leave a CSS +/// string; `None` when nothing is left. +fn safe_family(family: &str) -> Option { + let kept: String = family.chars().filter(|c| c.is_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.')).take(MAX_FAMILY_CHARS * 2).collect(); + let kept = kept.trim(); + (!kept.is_empty()).then(|| kept.to_owned()) +} + +/// The copied text as an HTML fragment, and whether [`MAX_CLIP_HTML_BYTES`] cut it: one `

` +/// per paragraph, a `` per run with its families, size and colour inline, bold as +/// `` and italic as `` (so a target that drops style attributes still keeps them). +/// Text is escaped; a line break is `
`, and a space HTML would collapse (after another +/// space, at a paragraph's start or after a break, across runs too) is ` `. Never longer +/// than the cap: a paragraph or run that would pass it ends the fragment. +pub fn clip_html(text: &ClipText) -> (String, bool) { + const OPEN: &str = "

"; + const CLOSE: &str = "

\n"; + let fits = |len: usize, more: usize| len.saturating_add(more).saturating_add(CLOSE.len()) <= MAX_CLIP_HTML_BYTES; + let mut out = String::new(); + for spans in &text.paragraphs { + if !fits(out.len(), OPEN.len()) { + return (out, true); + } + let mut p = String::from(OPEN); + // A paragraph's first space would collapse. + let mut space_before = true; + for span in spans { + let s = span_html(span, &mut space_before); + if !fits(out.len().saturating_add(p.len()), s.len()) { + if p.len() > OPEN.len() { + out.push_str(&p); + out.push_str(CLOSE); + } + return (out, true); + } + p.push_str(&s); + } + p.push_str(CLOSE); + out.push_str(&p); + } + (out, false) +} + +/// One run as HTML. `space_before` says whether the text before it ended in a space or a break, +/// and is left saying so for the next run. +fn span_html(span: &ClipSpan, space_before: &mut bool) -> String { + let st = &span.style; + let mut css = Vec::new(); + let families: Vec = st.families.iter().filter_map(|f| safe_family(f)).map(|f| format!("'{f}'")).collect(); + if !families.is_empty() { + css.push(format!("font-family:{}", families.join(","))); + } + if st.size.is_finite() && st.size != 0.0 { + css.push(format!("font-size:{}pt", num(st.size.clamp(*CLIP_SIZE_PT.start(), *CLIP_SIZE_PT.end())))); + } + if let Some(h) = hex(st.color) { + css.push(format!("color:#{h}")); + } + let mut body = text_html(&span.text, space_before); + if st.italic { + body = format!("{body}"); + } + if st.bold { + body = format!("{body}"); + } + if css.is_empty() { body } else { format!("{body}", esc(&css.join(";"))) } +} + +/// Escape for HTML text in one pass; `\n` becomes `
`, and a space after a space or a break +/// (`space_before` carries that in and out) is ` `, so a run of spaces keeps its width. +/// Dropped: what [`esc`] drops (C0 controls), C1 controls, a carriage return +/// and the bidirectional overrides and isolates, which would reorder the pasted text. +fn text_html(text: &str, space_before: &mut bool) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars().filter(|c| clip_char(*c)) { + match c { + '\n' => out.push_str("
"), + ' ' if *space_before => out.push_str(" "), + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + c => out.push(c), + } + *space_before = matches!(c, ' ' | '\n'); + } + out +} + +/// Whether `c` may go into the clipboard HTML's text. +fn clip_char(c: char) -> bool { + allowed(c) && !matches!(c, '\r' | '\u{80}'..='\u{9F}' | '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') +} + +/// A size with at most one decimal and no trailing `.0`. +fn num(v: f64) -> String { + let r = (v * 10.0).round() / 10.0; + if r.fract() == 0.0 { format!("{r:.0}") } else { format!("{r:.1}") } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn style(size: f64, bold: bool, italic: bool, color: [f64; 3]) -> ClipStyle { + ClipStyle { families: font_family("ABCDEF+Helvetica-Bold"), size, bold, italic, color } + } + + #[test] + fn heading_body_and_coloured_span_keep_their_formatting() { + let mut b = ClipBuilder::new(); + let heading = style(24.0, true, false, [0.0; 3]); + let body = style(11.5, false, false, [0.0; 3]); + let red = style(11.5, false, true, [1.0, 0.0, 0.0]); + b.push("Title", &heading); + b.paragraph(); + b.push("Body", &body); + b.space(); + b.push("text", &body); + b.space(); + b.push("red", &red); + let clip = b.finish(); + assert_eq!(clip.paragraphs.len(), 2); + assert_eq!(clip.paragraphs.get(1).map(Vec::len), Some(2), "{clip:?}"); + let (html, cut) = clip_html(&clip); + assert!(!cut); + assert_eq!( + html, + "

Title

\n\ +

Body text \ + red

\n" + ); + } + + #[test] + fn special_characters_are_escaped_and_cannot_leave_the_style() { + let st = ClipStyle { families: vec!["Evil';}