From 0b84ac2312de07f0848f37ca33cf6d790d29ec8e Mon Sep 17 00:00:00 2001 From: Jeff Witt <152964771+witt3rd@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:59:08 -0400 Subject: [PATCH 1/2] fix: emit glyph marker instead of fabricated encoding text for symbolic-font cmap misses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a character code is missing from both /Encoding/Differences and an explicit /ToUnicode (or predefined CID) CMap, get_correct_character() fell back to the declared base encoding and then StandardEncoding. For symbolic fonts — e.g. subsetted Arabic fonts, whose codes bear no relation to the Latin encodings — this fabricates unrelated ASCII text: an Arabic ligature glyph at code 0x23 whose ToUnicode entry the generator omitted comes out as a literal '#' (docling-project/docling#3802). Route the unmapped-code fallbacks through resolve_unmapped_character(): if an explicit cmap exists but misses the code and the font is symbolic (/FontDescriptor /Flags bit 3) or composite (Type0), emit the existing GLYPH marker so downstream consumers can detect the unresolved glyph. Authoritative sources (Differences, cmaps, known base-font tables) stay ahead of the guard; non-symbolic simple fonts and fonts without any ToUnicode keep the current behavior. Also hoist the /FontDescriptor /Flags parsing into init_font_flags(), reused by build_embedded_font_blob() which previously parsed it inline. Co-Authored-By: Claude Fable 5 Signed-off-by: Jeff Witt <152964771+witt3rd@users.noreply.github.com> --- src/parse/pdf_resources/page_font.h | 69 ++++++++++++++---- tests/test_tounicode_fallback.py | 109 ++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 tests/test_tounicode_fallback.py diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index 3faa58de..0a652bbe 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -84,6 +84,7 @@ namespace pdflib private: std::string get_correct_character(uint32_t c); + std::string resolve_unmapped_character(uint32_t c); std::string get_character_from_encoding(uint32_t c); void init_encoding(); @@ -124,6 +125,8 @@ namespace pdflib void init_cmap(pdf_timings& timings); void init_cmap_resource(); + void init_font_flags(); + void init_differences(); void init_charprocs(); @@ -150,6 +153,7 @@ namespace pdflib std::string encoding_name; font_encoding_name encoding; bool has_explicit_encoding; // true if encoding was found in PDF, false if defaulted + bool is_symbolic = false; // /FontDescriptor /Flags bit 3 (PDF 32000-1 table 123) font_subtype_name subtype; @@ -575,7 +579,7 @@ namespace pdflib else if(cmap_initialized and cmap_numb_to_char.count(c)>0) { return cmap_numb_to_char.at(c); - } + } else if(bfonts.has_corresponding_font(font_name)) { // check if the font-name is registered as a 'special' font, eg @@ -591,7 +595,7 @@ namespace pdflib if(has_explicit_encoding && (encoding == MACROMAN || encoding == MACEXPERT || encoding == WINANSI || encoding == STANDARD)) { - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } else if(fm.has(c)) { @@ -605,7 +609,7 @@ namespace pdflib << "; Encoding: " << to_string(_encoding) << "; font-name: " << font_name; */ - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } else { @@ -627,16 +631,57 @@ namespace pdflib << "; font-name: " << font_name << " (corresponding font: " << fontname << ")"; - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } } else { //LOG_S(WARNING) << "no known font: " << font_name; - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } } + void pdf_resource::init_font_flags() + { + const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 (PDF 32000-1 table 123) + + int flags = 0; + if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags)) + { + qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags); + } + + is_symbolic = ((flags & FLAG_SYMBOLIC) != 0); + + LOG_S(INFO) << __FUNCTION__ << ": flags=" << flags + << ", is_symbolic=" << is_symbolic; + } + + std::string pdf_resource::resolve_unmapped_character(uint32_t c) + { + // Fallback for codes that none of the authoritative sources resolved + // (/Encoding/Differences, the /ToUnicode or predefined CID cmap, or a + // known base-font table). When an explicit cmap exists but does not + // cover this code, and the code bears no relation to the standard + // Latin encodings (symbolic font, or composite font whose codes are + // CIDs), the encoding tables would fabricate unrelated text — e.g. an + // Arabic ligature glyph subsetted at code 0x23 coming out as '#' + // (docling#3802). Emit a glyph marker instead, so downstream consumers + // can detect the unresolved glyph. + if(cmap_initialized and (is_symbolic or subtype==TYPE_0)) + { + unknown_numbs[c] += 1; + + LOG_S(WARNING) << "Symbol not in the cmap of a symbolic or composite font: " + << int(c) << "; font-name: " << font_name + << "; emitting glyph marker instead of encoding fallback"; + + return "GLYPH<" + std::to_string(c) + ">"; + } + + return get_character_from_encoding(c); + } + std::string pdf_resource::get_character_from_encoding(uint32_t c) { auto& base_encoding = encodings.get(encoding).get_numb_to_utf8(); @@ -719,7 +764,9 @@ namespace pdflib init_encoding(); init_subtype(); - + + init_font_flags(); + init_base_font(); init_font_name(); @@ -1413,18 +1460,10 @@ namespace pdflib // then resolve glyphs by character code, not by Unicode text. bool uses_builtin_encoding = false; { - const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 - - int flags = 0; - if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags)) - { - qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags); - } - const bool has_encoding = qpdf_object::has_path(qpdf_font, {"/Encoding"}); uses_builtin_encoding = (subtype != TYPE_0) and - ((flags & FLAG_SYMBOLIC) != 0) and (not has_encoding); + is_symbolic and (not has_encoding); } font_blob = std::make_shared( diff --git a/tests/test_tounicode_fallback.py b/tests/test_tounicode_fallback.py new file mode 100644 index 00000000..cc08d692 --- /dev/null +++ b/tests/test_tounicode_fallback.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +"""Fallback behavior for char codes not covered by an explicit /ToUnicode CMap. + +Symbolic fonts (e.g. subsetted Arabic fonts) assign character codes with no +relation to the standard Latin encodings. When such a font carries a +/ToUnicode CMap that misses a code (a common generator bug for ligature +glyphs), falling back to Standard/WinAnsi tables fabricates unrelated ASCII +text — e.g. an Arabic ligature at code 0x23 coming out as '#' +(docling-project/docling#3802). The parser must emit a GLYPH marker instead. + +These tests build the PDFs in memory: a simple TrueType font, text bytes +0x21 0x22 0x23, and a /ToUnicode CMap that covers 0x21/0x22 but — depending +on the variant — omits the ligature entry for 0x23. +""" + +from io import BytesIO + +from docling_parse.pdf_parser import DecodeConfig, DoclingPdfParser + + +def _build_pdf(include_ligature: bool, symbolic: bool) -> bytes: + bfchars = ["<21> <0634>", "<22> <0623>"] # ش , أ + if include_ligature: + bfchars.append("<23> <0641064A>") # في (one glyph, two codepoints) + cmap = ( + "/CIDInit /ProcSet findresource begin\n" + "12 dict begin\n" + "begincmap\n" + "/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n" + "/CMapName /Adobe-Identity-UCS def\n" + "/CMapType 2 def\n" + "1 begincodespacerange\n" + "<00> \n" + "endcodespacerange\n" + f"{len(bfchars)} beginbfchar\n" + "\n".join(bfchars) + "\nendbfchar\n" + "endcmap\n" + "CMapName currentdict /CMap defineresource pop\n" + "end\n" + "end" + ) + + flags = 4 if symbolic else 32 # symbolic vs nonsymbolic (bit 3 vs bit 6) + content = 'BT /F1 24 Tf 72 700 Td (!"#) Tj ET' + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial " + "/FirstChar 33 /LastChar 35 /Widths [500 500 500] " + "/Encoding /WinAnsiEncoding /FontDescriptor 7 0 R /ToUnicode 6 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + f"<< /Length {len(cmap)} >>\nstream\n{cmap}\nendstream", + f"<< /Type /FontDescriptor /FontName /Arial /Flags {flags} " + "/FontBBox [0 -200 1000 900] /ItalicAngle 0 /Ascent 900 /Descent -200 " + "/CapHeight 700 /StemV 80 >>", + ] + + out = b"%PDF-1.4\n" + offsets = [] + for index, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{index} 0 obj\n{obj}\nendobj\n".encode("latin-1") + + startxref = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode("latin-1") + out += b"0000000000 65535 f \n" + for offset in offsets: + out += f"{offset:010d} 00000 n \n".encode("latin-1") + out += ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{startxref}\n%%EOF" + ).encode("latin-1") + return out + + +def _extract_text(include_ligature: bool, symbolic: bool) -> str: + parser = DoclingPdfParser(loglevel="fatal") + config = DecodeConfig(keep_glyphs=True) + doc = parser.load( + path_or_stream=BytesIO(_build_pdf(include_ligature, symbolic)), + decode_config=config, + ) + _, page = next(doc.iterate_pages()) + return "".join(cell.text for cell in page.textline_cells) + + +def test_symbolic_font_tounicode_miss_yields_glyph_marker(): + # No fabricated '#' from the WinAnsi/Standard tables: the unmapped code + # of a symbolic font must surface as a GLYPH marker. + text = _extract_text(include_ligature=False, symbolic=True) + assert "#" not in text + assert "GLYPH<35>" in text + assert "ش" in text and "أ" in text # covered codes still resolve + + +def test_symbolic_font_complete_tounicode_resolves_ligature(): + text = _extract_text(include_ligature=True, symbolic=True) + assert "#" not in text + assert "GLYPH" not in text + assert "في" in text + + +def test_nonsymbolic_font_keeps_encoding_fallback(): + # Latin fonts with a partial /ToUnicode still fall back to the declared + # encoding: 0x23 in WinAnsi genuinely is '#'. + text = _extract_text(include_ligature=False, symbolic=False) + assert "#" in text + assert "GLYPH" not in text From 1bbca5c342a28bd7978b61c4c2451956997a6ea6 Mon Sep 17 00:00:00 2001 From: Jeff Witt <152964771+witt3rd@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:26:34 -0400 Subject: [PATCH 2/2] test: assert default config strips glyph markers from output With keep_glyphs=false (the production default) the GLYPH<...> marker emitted for a symbolic-font cmap miss is stripped to a space in pdf_states/text.h, so neither the marker nor a fabricated '#' reaches the extracted text. Co-Authored-By: Claude Fable 5 Signed-off-by: Jeff Witt <152964771+witt3rd@users.noreply.github.com> --- tests/test_tounicode_fallback.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_tounicode_fallback.py b/tests/test_tounicode_fallback.py index cc08d692..2af6197b 100644 --- a/tests/test_tounicode_fallback.py +++ b/tests/test_tounicode_fallback.py @@ -74,9 +74,11 @@ def _build_pdf(include_ligature: bool, symbolic: bool) -> bytes: return out -def _extract_text(include_ligature: bool, symbolic: bool) -> str: +def _extract_text( + include_ligature: bool, symbolic: bool, keep_glyphs: bool = True +) -> str: parser = DoclingPdfParser(loglevel="fatal") - config = DecodeConfig(keep_glyphs=True) + config = DecodeConfig(keep_glyphs=keep_glyphs) doc = parser.load( path_or_stream=BytesIO(_build_pdf(include_ligature, symbolic)), decode_config=config, @@ -107,3 +109,13 @@ def test_nonsymbolic_font_keeps_encoding_fallback(): text = _extract_text(include_ligature=False, symbolic=False) assert "#" in text assert "GLYPH" not in text + + +def test_default_config_strips_glyph_markers(): + # Production default (keep_glyphs=False, see config.h): the marker is + # stripped to a space in pdf_states/text.h, so neither GLYPH<...> nor + # a fabricated '#' ever reaches the output. + text = _extract_text(include_ligature=False, symbolic=True, keep_glyphs=False) + assert "GLYPH" not in text + assert "#" not in text + assert "\u0634" in text and "\u0623" in text # covered codes still resolve