Draw Thai in form fields whose font can't hold it (#2354)

Text fields and combo boxes draw Thai from the Thai face beside the /DA
font, keeping /DA and leaving /NeedAppearances alone, so other viewers show
the same glyphs. The ink of stacked marks stays inside the border, and an
automatic size shrinks for it. Without a Thai face, filling Thai is refused.

crates/forms/README.md

@@ -45,7 +45,10 @@ let cells: Vec<table::Cell> = grid.cells_for(names)?;      // "Table1_1", "Table
   (cells parted in the border colour), multiline wrapping, password masking, quadding, `/MK`
   background and border, list-box selection.
   The `/DA` font is used when `/DR` has it as a simple font or a composite font with a predefined
-  Unicode CMap; otherwise Helvetica (WinAnsi). A value or button caption the font can't hold
+  Unicode CMap; otherwise Helvetica (WinAnsi). Thai in a text field or combo box that the font
+  can't hold is drawn from the craft-fonts Thai face with Add text's writer
+  (`pdfcraft_fonts::shaped_text`), beside the field's font, keeping `/DA`; a build without a Thai
+  face refuses it. Any other value or button caption the font can't hold
   (Chinese in a Helvetica field, a character beyond U+FFFF in a UCS-2 CMap) is kept, as Acrobat
   keeps it: `/V` is stored as typed, the appearance draws `?` for those characters, the widget is
   marked with the first one (`pdfcraft_fonts::UNDRAWN_MARK`, cleared once all of it can be drawn)

crates/forms/src/appearance.rs

@@ -23,6 +23,7 @@
 
 use pdfcraft_cos::{Dict, Document, Object, Stream};
 use pdfcraft_fonts::pdf::Metrics;
+use pdfcraft_fonts::shaped_text::{self, Piece, ShapedFonts};
 use pdfcraft_fonts::{UnicodeCMap, cjk_width, helvetica_width, literal, win_ansi, wrap_with};
 
 use crate::{Field, FieldKind, Widget, acroform, flags};
@@ -747,6 +748,21 @@ fn draws(doc: &Document, f: &Field, wd: &Dict, text: &str) -> bool {
     enc.first_unencodable(text).is_none()
 }
 
+/// Why text field `f` can't take `text`: it has Thai that a widget's font can't hold and this
+/// build can't draw (it has no Thai face, or the face lacks a character), which would come out
+/// as `?`. `None` for text it can draw, and for a password field, which shows `*`.
+pub(crate) fn thai_refusal(doc: &Document, f: &Field, text: &str) -> Option<String> {
+    if f.has(flags::PASSWORD) || !text.chars().any(shaped_text::is_thai) {
+        return None;
+    }
+    let held = |w: &Widget| {
+        let wd = doc.get(w.obj).as_dict().cloned().unwrap_or_default();
+        let da = parse_da(wd.get(b"DA").and_then(|o| doc.resolve(o).as_string().map(|s| s.to_text())).as_deref().unwrap_or(&f.da));
+        dr_font(doc, &da.font).is_some_and(|(_, enc)| matches!(enc, FontText::Simple(_)) && text.split(['\n', '\r']).all(|l| enc.can_encode(l)))
+    };
+    if f.widgets.iter().all(held) { None } else { shaped_text::thai_refusal(text) }
+}
+
 /// [`field_appearance_drawn`], and whether the text drawn is cut off by the box.
 fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String], format: bool) -> (Stream, Option<char>, Option<Overflow>) {
     let wobj = doc.get(w.obj);
@@ -778,36 +794,43 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
         values.first().cloned().unwrap_or_default()
     };
     let text = if f.has(flags::PASSWORD) { "*".repeat(text.chars().count()) } else { text };
-    let can_show = |enc: &FontText| {
+    let can_show = |enc: &FontText, text: &str| {
         if f.kind == FieldKind::List {
             f.options.iter().all(|(_, display)| enc.can_encode(display))
         } else if f.kind == FieldKind::Text && f.has(flags::MULTILINE) {
             text.split(['\n', '\r']).all(|line| enc.can_encode(line))
         } else {
-            enc.can_encode(&text)
+            enc.can_encode(text)
         }
     };
-    let (font_name, font_obj, enc) = match dr_font(doc, &da.font).filter(|(_, enc)| can_show(enc)) {
+    let dr = dr_font(doc, &da.font);
+    // Thai the form's font can't hold (it isn't a simple font encoding it) is drawn from the Thai
+    // face, as Add text draws it: Type 3 glyphs beside the form's font, which draws the rest. One
+    // line of a text field or combo box at a time; the `/DA` is kept.
+    let thai = matches!(f.kind, FieldKind::Text | FieldKind::Combo)
+        && !f.is_comb()
+        && text.chars().any(shaped_text::is_thai)
+        && !dr.as_ref().is_some_and(|(_, enc)| matches!(enc, FontText::Simple(_)) && can_show(enc, &text))
+        && shaped_text::thai_refusal(&text).is_none();
+    let latin_text: String = if thai { text.chars().filter(|c| !shaped_text::is_thai(*c)).collect() } else { text.clone() };
+    let (font_name, font_obj, enc) = match dr.filter(|(_, enc)| can_show(enc, &latin_text)) {
         Some((o, enc)) => (da.font.clone(), o, enc),
         None => ("Helv".to_string(), helvetica(), FontText::WinAnsi),
     };
-    let width_of = |text: &str, size: f64| enc.width(text, size);
+    let width_of =
+        |text: &str, size: f64| if thai { shaped_text::thai_width(text, size, |s, size| enc.width(s, size)) } else { enc.width(text, size) };
+    // How far a line's ink reaches below and above its baseline (em), where Thai stacks or
+    // descenders reach past the form font's (Helvetica's) -0.207 and 0.718; `None` otherwise.
+    let thai_ink = |line: &str| thai.then(|| shaped_text::thai_reach(line)).flatten().map(|[lo, hi]| [lo.min(-0.207), hi.max(0.718)]);
     let (mut c, bw) = frame(doc, &wd, width, height);
     let pad = 2.0 + bw;
     let inner_w = (width - 2.0 * pad).max(1.0);
     let q = wd.get(b"Q").and_then(|o| doc.resolve(o).as_int()).unwrap_or(f.quadding);
-    let mut body: Vec<u8> = Vec::new();
-    // The first character drawn that the font can't hold (drawn as `?`), for the caller to report.
-    let mut unencodable: Option<char> = None;
-    let mut show = |body: &mut Vec<u8>, x: f64, y: f64, text: &str| {
+    // The lines to draw (where, and what), drawn once all are known: their Thai shares fonts.
+    let mut shown: Vec<(f64, f64, String)> = Vec::new();
+    let mut show = |x: f64, y: f64, text: &str| {
         // A line break in a single-line value is a space, as in viewers (multiline text is split).
-        let text: String = text.chars().map(|c| if matches!(c, '\n' | '\r') { ' ' } else { c }).collect();
-        if unencodable.is_none() {
-            unencodable = enc.first_unencodable(&text);
-        }
-        body.extend(format!("1 0 0 1 {} {} Tm ", n(x), n(y)).bytes());
-        body.extend(literal(&enc.encode(&text)));
-        body.extend_from_slice(b" Tj\n");
+        shown.push((x, y, text.chars().map(|c| if matches!(c, '\n' | '\r') { ' ' } else { c }).collect()));
     };
     let x_for = |text: &str, size: f64| -> f64 {
         let tw = width_of(text, size);
@@ -834,7 +857,7 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
                 if values.contains(export) {
                     c.push_str(&format!("0.6 0.75 0.86 rg\n{} {} {} {} re f\n", n(bw), n(y - line), n(width - 2.0 * bw), n(line)));
                 }
-                show(&mut body, pad, y - line + size * 0.25, display);
+                show(pad, y - line + size * 0.25, display);
                 y -= line;
             }
         }
@@ -844,18 +867,20 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
                 if size == 0.0 {
                     size = auto_text_size(&text, inner_w, height - 2.0 * pad, true, width_of);
                 }
-                let mut y = height - pad - size * 0.85;
                 let lines = wrap_with(&text, size, inner_w, width_of);
+                // The first line hangs from the top by its ascent: higher for stacked Thai marks.
+                let mut y = height - pad - size * lines.first().and_then(|l| thai_ink(l)).map_or(0.85, |[_, hi]| hi.max(0.85));
                 // The lines whose descent stays inside the border; the rest are cut off.
                 let mut room = 0;
                 for line in &lines {
                     if y < -size {
                         break;
                     }
-                    if y - 0.207 * size >= bw - 0.01 {
+                    let descent = thai_ink(line).map_or(0.207, |[lo, _]| -lo);
+                    if y - descent * size >= bw - 0.01 {
                         room += 1;
                     }
-                    show(&mut body, x_for(line, size), y, line);
+                    show(x_for(line, size), y, line);
                     y -= size * 1.15;
                 }
                 if room < lines.len() {
@@ -872,12 +897,22 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
                         auto_text_size(&text, inner_w, height - 2.0 * pad, false, width_of)
                     };
                 }
-                // Centre Helvetica's ascent (0.718) and descent (0.207) vertically.
-                let y = (height - 0.925 * size) / 2.0 + 0.207 * size;
+                // Centre Helvetica's ascent (0.718) and descent (0.207) vertically, or the reach of
+                // Thai stacks and descenders, which an automatic size also keeps inside the padding.
+                let reach = thai_ink(&text);
+                if let Some([lo, hi]) = reach
+                    && da.size == 0.0
+                {
+                    size = size.min((height - 2.0 * pad) / (hi - lo)).max(4.0);
+                }
+                let (y, ink) = match reach {
+                    Some([lo, hi]) => ((height - (hi - lo) * size) / 2.0 - lo * size, (hi - lo) * size),
+                    None => ((height - 0.925 * size) / 2.0 + 0.207 * size, 0.925 * size),
+                };
                 // The clip box is inside the border; a line taller than it is cut at the top and bottom.
                 let room_h = height - 2.0 * bw;
-                if 0.925 * size > room_h + 0.01 {
-                    overflow = Some(Overflow::High { height: 0.925 * size, room: room_h, size });
+                if ink > room_h + 0.01 {
+                    overflow = Some(Overflow::High { height: ink, room: room_h, size });
                 }
                 if comb {
                     let cells = f.max_len.unwrap_or(1).max(1);
@@ -885,18 +920,19 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
                     c.push_str(&comb_dividers(doc, &wd, width, height, cells));
                     for (i, ch) in text.chars().take(cells).enumerate() {
                         let s = ch.to_string();
-                        show(&mut body, cell * i as f64 + (cell - width_of(&s, size)) / 2.0, y, &s);
+                        show(cell * i as f64 + (cell - width_of(&s, size)) / 2.0, y, &s);
                     }
                 } else {
                     let tw = width_of(&text.replace(['\n', '\r'], " "), size);
                     if overflow.is_none() && tw > inner_w + 0.01 {
                         overflow = Some(Overflow::Wide { width: tw, room: inner_w, size });
                     }
-                    show(&mut body, x_for(&text, size), y, &text);
+                    show(x_for(&text, size), y, &text);
                 }
             }
         }
     }
+    let (body, fonts, unencodable) = draw_lines(&shown, size, &font_name, font_obj, &enc, thai);
     // Acrobat regenerates only what is inside /Tx BMC when it redraws a field (e.g. on focus
     // changes), discarding what came before it (issues #1613, #633). The background and border
     // therefore go inside the marked-content block too, so a redraw keeps them with the text.
@@ -913,8 +949,6 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
     .into_bytes();
     content.extend(body);
     content.extend_from_slice(b"ET\nQ\nEMC\n");
-    let mut fonts = Dict::new();
-    fonts.set(font_name.into_bytes(), font_obj);
     let mut res = Dict::new();
     res.set(b"Font".to_vec(), Object::Dict(fonts));
     let mut d = Dict::new();
@@ -927,6 +961,31 @@ fn field_appearance_fit(doc: &Document, f: &Field, w: &Widget, values: &[String]
     (Stream::flate(d, &content), unencodable, overflow)
 }
 
+/// The text operators for the `shown` lines (where each starts, and what it says) at `size`, in
+/// the field's font `font_name` (`font`, written with `enc`) and, with `thai`, their Thai from the
+/// Thai face; the font resources they use, and the first character drawn as `?` because the
+/// font can't hold it, for the caller to report.
+fn draw_lines(shown: &[(f64, f64, String)], size: f64, font_name: &str, font: Object, enc: &FontText, thai: bool) -> (Vec<u8>, Dict, Option<char>) {
+    let mut fonts = Dict::new();
+    fonts.set(font_name.as_bytes().to_vec(), font);
+    let lines: Vec<String> = shown.iter().map(|(_, _, text)| text.clone()).collect();
+    let (pieces, shaped) = if thai {
+        shaped_text::appearance_lines(&lines, &mut fonts)
+    } else {
+        (lines.into_iter().map(|l| vec![Piece::Latin(l)]).collect(), ShapedFonts::default())
+    };
+    let mut body = Vec::new();
+    let mut current = None;
+    for ((x, y, _), pieces) in shown.iter().zip(&pieces) {
+        shaped.show_line(&mut body, pieces, *x, *y, size, font_name, |s| enc.encode(s), |s| enc.width(s, size), &mut current);
+    }
+    let unencodable = pieces.iter().flatten().find_map(|p| match p {
+        Piece::Latin(s) => enc.first_unencodable(s),
+        Piece::Shaped(_) => None,
+    });
+    (body, fonts, unencodable)
+}
+
 /// On/Off appearances for a check box or radio button that has none (a check mark or a dot,
 /// drawn as paths, so no symbol font is needed).
 /// The mark is drawn in the colour of the widget's `/DA`, else of `f_da` (the field's): the

crates/forms/src/appearances.rs

@@ -75,6 +75,9 @@ pub fn with_field_appearances_cached(doc: &Document, cache: &mut FieldAppearance
                 }
             };
             let target = copy.get_or_insert_with(|| doc.clone());
+            let mut stream = stream;
+            // Thai glyphs are streams of their own.
+            pdfcraft_fonts::shaped_text::store_streams(target, &mut stream.dict);
             let ap = target.add(Object::Stream(stream));
             let mut fresh = Dict::new();
             match &state {

crates/forms/src/author/look.rs

@@ -323,8 +323,10 @@ pub(super) fn redraw_widgets(doc: &mut Document, name: &str, only: Option<usize>
         let mut missing = None;
         let ap = match f.kind {
             FieldKind::Text | FieldKind::Combo | FieldKind::List => {
-                let (s, m) = appearance::field_appearance_drawn(doc, &f, w, &f.value, true);
+                let (mut s, m) = appearance::field_appearance_drawn(doc, &f, w, &f.value, true);
                 missing = m;
+                // Thai glyphs are streams of their own.
+                pdfcraft_fonts::shaped_text::store_streams(doc, &mut s.dict);
                 let r = doc.add(Object::Stream(s));
                 let mut d = Dict::new();
                 d.set(b"N".to_vec(), Object::Ref(r));

crates/forms/src/tests.rs

@@ -967,6 +967,61 @@ fn text_a_field_font_cannot_hold_is_kept_marked_and_redrawn_elsewhere() {
     assert!(need_appearances(&doc));
 }
 
+/// Thai typed into a field whose `/DA` font (Helvetica, WinAnsi) can't hold it is drawn from the
+/// Thai face as Add text draws it: Type 3 glyphs (stream objects of their own, with a ToUnicode
+/// map) beside the field's font, which draws the rest, so every viewer shows the same glyphs. The
+/// `/DA` is kept, nothing is marked and other viewers aren't asked to redraw. The ink of stacked
+/// marks and descenders stays inside the border, an automatic size shrinking to keep it there.
+/// A build without a Thai face refuses the text, saying why, and keeps the old value.
+#[test]
+fn thai_a_field_font_cannot_hold_is_drawn_from_the_thai_face() {
+    let objs: Vec<String> = vec![
+        "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(),
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(),
+        "<< /Type /Page /Parent 2 0 R /Annots [6 0 R 7 0 R 8 0 R] >>".into(),
+        "<< /Fields [6 0 R 7 0 R 8 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >>".into(),
+        "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(),
+        "<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /DA (/Helv 0 Tf 0 g) /V (Ada) /Rect [50 700 250 720] /P 3 0 R >>".into(),
+        "<< /Type /Annot /Subtype /Widget /FT /Tx /T (big) /DA (/Helv 16 Tf 0 g) /Rect [50 600 350 640] /P 3 0 R >>".into(),
+        "<< /Type /Annot /Subtype /Widget /FT /Tx /T (note) /Ff 4096 /DA (/Helv 12 Tf 0 g) /Rect [50 400 350 500] /P 3 0 R >>".into(),
+    ];
+    let mut doc = document(&objs);
+    let thai = "ชื่อ: บุญฤทธิ์ น้ำฎู";
+    if pdfcraft_fonts::document_thai_font().is_none() {
+        let err = set_value(&mut doc, "name", &FieldValue::Text(thai.into())).unwrap_err();
+        assert!(matches!(&err, FormError::Invalid(m) if m.contains("Thai") && m.contains("CRAFT_FONTS_DIR")), "{err}");
+        assert_eq!(field(&fields(&doc), "name").value, ["Ada"]);
+        assert!(!need_appearances(&doc) && undrawn_fields(&doc).is_empty());
+        return;
+    }
+    set_value(&mut doc, "name", &FieldValue::Text(thai.into())).unwrap();
+    set_value(&mut doc, "big", &FieldValue::Text(thai.into())).unwrap();
+    set_value(&mut doc, "note", &FieldValue::Text(format!("น้ำฎู {thai}\nปั้น ฎู"))).unwrap();
+    let doc = reopen(&doc);
+    assert!(undrawn_fields(&doc).is_empty() && !need_appearances(&doc));
+    let (lo, hi) = (pdfcraft_fonts::shaped_text::thai_reach("ฎู").unwrap()[0], pdfcraft_fonts::shaped_text::thai_reach("น้ำ").unwrap()[1]);
+    for (name, height) in [("name", 20.0), ("big", 40.0), ("note", 100.0)] {
+        let all = fields(&doc);
+        let w = &field(&all, name).widgets[0];
+        let wd = doc.get(w.obj).as_dict().unwrap().clone();
+        assert!(wd.get(b"DA").and_then(|o| o.as_string()).is_some_and(|s| s.to_text().starts_with("/Helv ")), "the /DA is kept");
+        let n = wd.get(b"AP").unwrap().as_dict().unwrap().reference(b"N").unwrap();
+        let Object::Stream(s) = &*doc.get(n) else { panic!() };
+        let ap = String::from_utf8_lossy(&s.decoded().unwrap()).into_owned();
+        assert!(!ap.contains('?') && ap.contains("/PCTh1 ") && ap.contains("(: ) Tj"), "{ap}");
+        let fonts = s.dict.get(b"Resources").and_then(|r| r.as_dict()).and_then(|r| r.get(b"Font")).and_then(|f| f.as_dict()).unwrap();
+        let font = fonts.get(b"PCTh1").and_then(|f| f.as_dict()).unwrap();
+        assert_eq!(font.name(b"Subtype"), Some(&b"Type3"[..]));
+        let Object::Stream(cmap) = &*doc.get(font.reference(b"ToUnicode").unwrap()) else { panic!("ToUnicode is a stream object") };
+        assert!(String::from_utf8_lossy(&cmap.decoded().unwrap()).contains("<0E190E490E33>"), "น้ำ copies as typed");
+        // The size each line is drawn at, and where: every line's ink is inside the box.
+        let size: f64 = ap.split(" Tf").next().and_then(|s| s.rsplit(' ').next()).and_then(|s| s.parse().ok()).unwrap();
+        let ys: Vec<f64> = ap.split(" Tm").filter_map(|s| s.rsplit(' ').next()?.parse().ok()).collect();
+        let (top, bottom) = (ys.iter().copied().fold(f64::MIN, f64::max), ys.iter().copied().fold(f64::MAX, f64::min));
+        assert!(top + hi * size <= height + 1e-3 && bottom + lo * size >= -1e-3, "{name}: {ys:?} at {size} in {height}");
+    }
+}
+
 /// A document that declares PDF/A can't ask other viewers to redraw its fields (PDF/A forbids
 /// `/NeedAppearances`): the value is still kept and its widget marked, and the flag stays off.
 #[test]

crates/forms/src/values.rs

@@ -206,6 +206,11 @@ pub(crate) fn write_value(doc: &mut Document, f: &Field, value: &FieldValue, scr
             {
                 return invalid(format!("{:?} takes at most {max} characters", f.name));
             }
+            // Thai this build can't draw is refused (before any script runs), rather than stored
+            // and drawn as `?`.
+            if let Some(why) = appearance::thai_refusal(doc, f, t) {
+                return invalid(format!("{:?} can't take this text: {why}", f.name));
+            }
             // Keystroke (on commit) and Validate, as Acrobat runs them before accepting a value.
             let t = &af::keystroke(&f.actions.format, &f.name, t).map_err(FormError::Format)?;
             af::validate(&f.actions.validate, t).map_err(FormError::Invalid)?;
@@ -430,7 +435,7 @@ fn redraw(doc: &mut Document, f: &Field, values: &[String], scripts: &mut dyn Sc
     // fonts can't draw the date language's (#5303).
     let mut english: Option<Option<String>> = None;
     for w in &f.widgets {
-        let (stream, missing) = match &shown {
+        let (mut stream, missing) = match &shown {
             Some(s) => {
                 let drawn = appearance::field_appearance_drawn(doc, f, w, std::slice::from_ref(s), false);
                 // Taken when the English run happened and draws whole, even where it is the
@@ -447,6 +452,8 @@ fn redraw(doc: &mut Document, f: &Field, values: &[String], scripts: &mut dyn Sc
             None => appearance::field_appearance_drawn(doc, f, w, values, true),
         };
         undrawn |= missing.is_some();
+        // Thai glyphs are streams of their own.
+        pdfcraft_fonts::shaped_text::store_streams(doc, &mut stream.dict);
         let existing_ap = in_place
             .then(|| {
                 doc.get(w.obj)