fix: emit glyph marker instead of fabricated encoding text for symbolic-font cmap misses

src/parse/pdf_resources/page_font.h

@@ -84,6 +84,7 @@ namespace pdflib
   private:
 
     std::string get_correct_character(uint32_t c);
+    std::string resolve_unmapped_character(uint32_t c);
     std::string get_character_from_encoding(uint32_t c);
 
     void init_encoding();
@@ -124,6 +125,8 @@ namespace pdflib
     void init_cmap(pdf_timings& timings);
     void init_cmap_resource();
 
+    void init_font_flags();
+
     void init_differences();
 
     void init_charprocs();
@@ -150,6 +153,7 @@ namespace pdflib
     std::string        encoding_name;
     font_encoding_name encoding;
     bool               has_explicit_encoding; // true if encoding was found in PDF, false if defaulted
+    bool               is_symbolic = false;   // /FontDescriptor /Flags bit 3 (PDF 32000-1 table 123)
 
     font_subtype_name  subtype;
 
@@ -575,7 +579,7 @@ namespace pdflib
     else if(cmap_initialized and cmap_numb_to_char.count(c)>0)
       {
         return cmap_numb_to_char.at(c);
-      }         
+      }
     else if(bfonts.has_corresponding_font(font_name))
       {
         // check if the font-name is registered as a 'special' font, eg
@@ -591,7 +595,7 @@ namespace pdflib
         if(has_explicit_encoding &&
            (encoding == MACROMAN || encoding == MACEXPERT || encoding == WINANSI || encoding == STANDARD))
           {
-            return get_character_from_encoding(c);
+            return resolve_unmapped_character(c);
           }
         else if(fm.has(c))
           {
@@ -605,7 +609,7 @@ namespace pdflib
 	      << "; Encoding: " << to_string(_encoding)
 	      << "; font-name: " << font_name;
 	    */
-	    return get_character_from_encoding(c);
+	    return resolve_unmapped_character(c);
 	  }
 	else
 	  {
@@ -627,16 +631,57 @@ namespace pdflib
 			   << "; font-name: " << font_name
 			   << " (corresponding font: " << fontname << ")";
 
-	    return get_character_from_encoding(c);
+	    return resolve_unmapped_character(c);
 	  }
       }
     else
       {
 	//LOG_S(WARNING) << "no known font: " << font_name;
-        return get_character_from_encoding(c);
+        return resolve_unmapped_character(c);
       }
   }
 
+  void pdf_resource<PAGE_FONT>::init_font_flags()
+  {
+    const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 (PDF 32000-1 table 123)
+
+    int flags = 0;
+    if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags))
+      {
+        qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags);
+      }
+
+    is_symbolic = ((flags & FLAG_SYMBOLIC) != 0);
+
+    LOG_S(INFO) << __FUNCTION__ << ": flags=" << flags
+                << ", is_symbolic=" << is_symbolic;
+  }
+
+  std::string pdf_resource<PAGE_FONT>::resolve_unmapped_character(uint32_t c)
+  {
+    // Fallback for codes that none of the authoritative sources resolved
+    // (/Encoding/Differences, the /ToUnicode or predefined CID cmap, or a
+    // known base-font table). When an explicit cmap exists but does not
+    // cover this code, and the code bears no relation to the standard
+    // Latin encodings (symbolic font, or composite font whose codes are
+    // CIDs), the encoding tables would fabricate unrelated text — e.g. an
+    // Arabic ligature glyph subsetted at code 0x23 coming out as '#'
+    // (docling#3802). Emit a glyph marker instead, so downstream consumers
+    // can detect the unresolved glyph.
+    if(cmap_initialized and (is_symbolic or subtype==TYPE_0))
+      {
+        unknown_numbs[c] += 1;
+
+        LOG_S(WARNING) << "Symbol not in the cmap of a symbolic or composite font: "
+                       << int(c) << "; font-name: " << font_name
+                       << "; emitting glyph marker instead of encoding fallback";
+
+        return "GLYPH<" + std::to_string(c) + ">";
+      }
+
+    return get_character_from_encoding(c);
+  }
+
   std::string pdf_resource<PAGE_FONT>::get_character_from_encoding(uint32_t c)
   {
     auto& base_encoding = encodings.get(encoding).get_numb_to_utf8();
@@ -719,7 +764,9 @@ namespace pdflib
 
       init_encoding();
       init_subtype();
-
+
+      init_font_flags();
+
       init_base_font();
 
       init_font_name();
@@ -1413,18 +1460,10 @@ namespace pdflib
     // then resolve glyphs by character code, not by Unicode text.
     bool uses_builtin_encoding = false;
     {
-      const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3
-
-      int flags = 0;
-      if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags))
-        {
-          qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags);
-        }
-
       const bool has_encoding = qpdf_object::has_path(qpdf_font, {"/Encoding"});
 
       uses_builtin_encoding = (subtype != TYPE_0) and
-        ((flags & FLAG_SYMBOLIC) != 0) and (not has_encoding);
+        is_symbolic and (not has_encoding);
     }
 
     font_blob = std::make_shared<const embedded_font_blob>(

tests/test_tounicode_fallback.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+"""Fallback behavior for char codes not covered by an explicit /ToUnicode CMap.
+
+Symbolic fonts (e.g. subsetted Arabic fonts) assign character codes with no
+relation to the standard Latin encodings. When such a font carries a
+/ToUnicode CMap that misses a code (a common generator bug for ligature
+glyphs), falling back to Standard/WinAnsi tables fabricates unrelated ASCII
+text — e.g. an Arabic ligature at code 0x23 coming out as '#'
+(docling-project/docling#3802). The parser must emit a GLYPH marker instead.
+
+These tests build the PDFs in memory: a simple TrueType font, text bytes
+0x21 0x22 0x23, and a /ToUnicode CMap that covers 0x21/0x22 but — depending
+on the variant — omits the ligature entry for 0x23.
+"""
+
+from io import BytesIO
+
+from docling_parse.pdf_parser import DecodeConfig, DoclingPdfParser
+
+
+def _build_pdf(include_ligature: bool, symbolic: bool) -> bytes:
+    bfchars = ["<21> <0634>", "<22> <0623>"]  # ش , أ
+    if include_ligature:
+        bfchars.append("<23> <0641064A>")  # في (one glyph, two codepoints)
+    cmap = (
+        "/CIDInit /ProcSet findresource begin\n"
+        "12 dict begin\n"
+        "begincmap\n"
+        "/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n"
+        "/CMapName /Adobe-Identity-UCS def\n"
+        "/CMapType 2 def\n"
+        "1 begincodespacerange\n"
+        "<00> <FF>\n"
+        "endcodespacerange\n"
+        f"{len(bfchars)} beginbfchar\n" + "\n".join(bfchars) + "\nendbfchar\n"
+        "endcmap\n"
+        "CMapName currentdict /CMap defineresource pop\n"
+        "end\n"
+        "end"
+    )
+
+    flags = 4 if symbolic else 32  # symbolic vs nonsymbolic (bit 3 vs bit 6)
+    content = 'BT /F1 24 Tf 72 700 Td (!"#) Tj ET'
+    objects = [
+        "<< /Type /Catalog /Pages 2 0 R >>",
+        "<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+        "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+        "<< /Type /Font /Subtype /TrueType /BaseFont /Arial "
+        "/FirstChar 33 /LastChar 35 /Widths [500 500 500] "
+        "/Encoding /WinAnsiEncoding /FontDescriptor 7 0 R /ToUnicode 6 0 R >>",
+        f"<< /Length {len(content)} >>\nstream\n{content}\nendstream",
+        f"<< /Length {len(cmap)} >>\nstream\n{cmap}\nendstream",
+        f"<< /Type /FontDescriptor /FontName /Arial /Flags {flags} "
+        "/FontBBox [0 -200 1000 900] /ItalicAngle 0 /Ascent 900 /Descent -200 "
+        "/CapHeight 700 /StemV 80 >>",
+    ]
+
+    out = b"%PDF-1.4\n"
+    offsets = []
+    for index, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{index} 0 obj\n{obj}\nendobj\n".encode("latin-1")
+
+    startxref = len(out)
+    out += f"xref\n0 {len(objects) + 1}\n".encode("latin-1")
+    out += b"0000000000 65535 f \n"
+    for offset in offsets:
+        out += f"{offset:010d} 00000 n \n".encode("latin-1")
+    out += (
+        f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
+        f"startxref\n{startxref}\n%%EOF"
+    ).encode("latin-1")
+    return out
+
+
+def _extract_text(
+    include_ligature: bool, symbolic: bool, keep_glyphs: bool = True
+) -> str:
+    parser = DoclingPdfParser(loglevel="fatal")
+    config = DecodeConfig(keep_glyphs=keep_glyphs)
+    doc = parser.load(
+        path_or_stream=BytesIO(_build_pdf(include_ligature, symbolic)),
+        decode_config=config,
+    )
+    _, page = next(doc.iterate_pages())
+    return "".join(cell.text for cell in page.textline_cells)
+
+
+def test_symbolic_font_tounicode_miss_yields_glyph_marker():
+    # No fabricated '#' from the WinAnsi/Standard tables: the unmapped code
+    # of a symbolic font must surface as a GLYPH marker.
+    text = _extract_text(include_ligature=False, symbolic=True)
+    assert "#" not in text
+    assert "GLYPH<35>" in text
+    assert "ش" in text and "أ" in text  # covered codes still resolve
+
+
+def test_symbolic_font_complete_tounicode_resolves_ligature():
+    text = _extract_text(include_ligature=True, symbolic=True)
+    assert "#" not in text
+    assert "GLYPH" not in text
+    assert "في" in text
+
+
+def test_nonsymbolic_font_keeps_encoding_fallback():
+    # Latin fonts with a partial /ToUnicode still fall back to the declared
+    # encoding: 0x23 in WinAnsi genuinely is '#'.
+    text = _extract_text(include_ligature=False, symbolic=False)
+    assert "#" in text
+    assert "GLYPH" not in text
+
+
+def test_default_config_strips_glyph_markers():
+    # Production default (keep_glyphs=False, see config.h): the marker is
+    # stripped to a space in pdf_states/text.h, so neither GLYPH<...> nor
+    # a fabricated '#' ever reaches the output.
+    text = _extract_text(include_ligature=False, symbolic=True, keep_glyphs=False)
+    assert "GLYPH" not in text
+    assert "#" not in text
+    assert "\u0634" in text and "\u0623" in text  # covered codes still resolve