diff --git a/src/parse/pdf_resources/page_font.h b/src/parse/pdf_resources/page_font.h index 3faa58de..0a652bbe 100644 --- a/src/parse/pdf_resources/page_font.h +++ b/src/parse/pdf_resources/page_font.h @@ -84,6 +84,7 @@ namespace pdflib private: std::string get_correct_character(uint32_t c); + std::string resolve_unmapped_character(uint32_t c); std::string get_character_from_encoding(uint32_t c); void init_encoding(); @@ -124,6 +125,8 @@ namespace pdflib void init_cmap(pdf_timings& timings); void init_cmap_resource(); + void init_font_flags(); + void init_differences(); void init_charprocs(); @@ -150,6 +153,7 @@ namespace pdflib std::string encoding_name; font_encoding_name encoding; bool has_explicit_encoding; // true if encoding was found in PDF, false if defaulted + bool is_symbolic = false; // /FontDescriptor /Flags bit 3 (PDF 32000-1 table 123) font_subtype_name subtype; @@ -575,7 +579,7 @@ namespace pdflib else if(cmap_initialized and cmap_numb_to_char.count(c)>0) { return cmap_numb_to_char.at(c); - } + } else if(bfonts.has_corresponding_font(font_name)) { // check if the font-name is registered as a 'special' font, eg @@ -591,7 +595,7 @@ namespace pdflib if(has_explicit_encoding && (encoding == MACROMAN || encoding == MACEXPERT || encoding == WINANSI || encoding == STANDARD)) { - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } else if(fm.has(c)) { @@ -605,7 +609,7 @@ namespace pdflib << "; Encoding: " << to_string(_encoding) << "; font-name: " << font_name; */ - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } else { @@ -627,16 +631,57 @@ namespace pdflib << "; font-name: " << font_name << " (corresponding font: " << fontname << ")"; - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } } else { //LOG_S(WARNING) << "no known font: " << font_name; - return get_character_from_encoding(c); + return resolve_unmapped_character(c); } } + void pdf_resource::init_font_flags() + { + const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 (PDF 32000-1 table 123) + + int flags = 0; + if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags)) + { + qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags); + } + + is_symbolic = ((flags & FLAG_SYMBOLIC) != 0); + + LOG_S(INFO) << __FUNCTION__ << ": flags=" << flags + << ", is_symbolic=" << is_symbolic; + } + + std::string pdf_resource::resolve_unmapped_character(uint32_t c) + { + // Fallback for codes that none of the authoritative sources resolved + // (/Encoding/Differences, the /ToUnicode or predefined CID cmap, or a + // known base-font table). When an explicit cmap exists but does not + // cover this code, and the code bears no relation to the standard + // Latin encodings (symbolic font, or composite font whose codes are + // CIDs), the encoding tables would fabricate unrelated text — e.g. an + // Arabic ligature glyph subsetted at code 0x23 coming out as '#' + // (docling#3802). Emit a glyph marker instead, so downstream consumers + // can detect the unresolved glyph. + if(cmap_initialized and (is_symbolic or subtype==TYPE_0)) + { + unknown_numbs[c] += 1; + + LOG_S(WARNING) << "Symbol not in the cmap of a symbolic or composite font: " + << int(c) << "; font-name: " << font_name + << "; emitting glyph marker instead of encoding fallback"; + + return "GLYPH<" + std::to_string(c) + ">"; + } + + return get_character_from_encoding(c); + } + std::string pdf_resource::get_character_from_encoding(uint32_t c) { auto& base_encoding = encodings.get(encoding).get_numb_to_utf8(); @@ -719,7 +764,9 @@ namespace pdflib init_encoding(); init_subtype(); - + + init_font_flags(); + init_base_font(); init_font_name(); @@ -1413,18 +1460,10 @@ namespace pdflib // then resolve glyphs by character code, not by Unicode text. bool uses_builtin_encoding = false; { - const int FLAG_SYMBOLIC = 1 << 2; // /Flags bit 3 - - int flags = 0; - if(not qpdf_object::get_int(qpdf_font, {"/FontDescriptor", "/Flags"}, flags)) - { - qpdf_object::get_int(qpdf_desc_font, {"/FontDescriptor", "/Flags"}, flags); - } - const bool has_encoding = qpdf_object::has_path(qpdf_font, {"/Encoding"}); uses_builtin_encoding = (subtype != TYPE_0) and - ((flags & FLAG_SYMBOLIC) != 0) and (not has_encoding); + is_symbolic and (not has_encoding); } font_blob = std::make_shared( diff --git a/tests/test_tounicode_fallback.py b/tests/test_tounicode_fallback.py new file mode 100644 index 00000000..2af6197b --- /dev/null +++ b/tests/test_tounicode_fallback.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +"""Fallback behavior for char codes not covered by an explicit /ToUnicode CMap. + +Symbolic fonts (e.g. subsetted Arabic fonts) assign character codes with no +relation to the standard Latin encodings. When such a font carries a +/ToUnicode CMap that misses a code (a common generator bug for ligature +glyphs), falling back to Standard/WinAnsi tables fabricates unrelated ASCII +text — e.g. an Arabic ligature at code 0x23 coming out as '#' +(docling-project/docling#3802). The parser must emit a GLYPH marker instead. + +These tests build the PDFs in memory: a simple TrueType font, text bytes +0x21 0x22 0x23, and a /ToUnicode CMap that covers 0x21/0x22 but — depending +on the variant — omits the ligature entry for 0x23. +""" + +from io import BytesIO + +from docling_parse.pdf_parser import DecodeConfig, DoclingPdfParser + + +def _build_pdf(include_ligature: bool, symbolic: bool) -> bytes: + bfchars = ["<21> <0634>", "<22> <0623>"] # ش , أ + if include_ligature: + bfchars.append("<23> <0641064A>") # في (one glyph, two codepoints) + cmap = ( + "/CIDInit /ProcSet findresource begin\n" + "12 dict begin\n" + "begincmap\n" + "/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n" + "/CMapName /Adobe-Identity-UCS def\n" + "/CMapType 2 def\n" + "1 begincodespacerange\n" + "<00> \n" + "endcodespacerange\n" + f"{len(bfchars)} beginbfchar\n" + "\n".join(bfchars) + "\nendbfchar\n" + "endcmap\n" + "CMapName currentdict /CMap defineresource pop\n" + "end\n" + "end" + ) + + flags = 4 if symbolic else 32 # symbolic vs nonsymbolic (bit 3 vs bit 6) + content = 'BT /F1 24 Tf 72 700 Td (!"#) Tj ET' + objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + "/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial " + "/FirstChar 33 /LastChar 35 /Widths [500 500 500] " + "/Encoding /WinAnsiEncoding /FontDescriptor 7 0 R /ToUnicode 6 0 R >>", + f"<< /Length {len(content)} >>\nstream\n{content}\nendstream", + f"<< /Length {len(cmap)} >>\nstream\n{cmap}\nendstream", + f"<< /Type /FontDescriptor /FontName /Arial /Flags {flags} " + "/FontBBox [0 -200 1000 900] /ItalicAngle 0 /Ascent 900 /Descent -200 " + "/CapHeight 700 /StemV 80 >>", + ] + + out = b"%PDF-1.4\n" + offsets = [] + for index, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{index} 0 obj\n{obj}\nendobj\n".encode("latin-1") + + startxref = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode("latin-1") + out += b"0000000000 65535 f \n" + for offset in offsets: + out += f"{offset:010d} 00000 n \n".encode("latin-1") + out += ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{startxref}\n%%EOF" + ).encode("latin-1") + return out + + +def _extract_text( + include_ligature: bool, symbolic: bool, keep_glyphs: bool = True +) -> str: + parser = DoclingPdfParser(loglevel="fatal") + config = DecodeConfig(keep_glyphs=keep_glyphs) + doc = parser.load( + path_or_stream=BytesIO(_build_pdf(include_ligature, symbolic)), + decode_config=config, + ) + _, page = next(doc.iterate_pages()) + return "".join(cell.text for cell in page.textline_cells) + + +def test_symbolic_font_tounicode_miss_yields_glyph_marker(): + # No fabricated '#' from the WinAnsi/Standard tables: the unmapped code + # of a symbolic font must surface as a GLYPH marker. + text = _extract_text(include_ligature=False, symbolic=True) + assert "#" not in text + assert "GLYPH<35>" in text + assert "ش" in text and "أ" in text # covered codes still resolve + + +def test_symbolic_font_complete_tounicode_resolves_ligature(): + text = _extract_text(include_ligature=True, symbolic=True) + assert "#" not in text + assert "GLYPH" not in text + assert "في" in text + + +def test_nonsymbolic_font_keeps_encoding_fallback(): + # Latin fonts with a partial /ToUnicode still fall back to the declared + # encoding: 0x23 in WinAnsi genuinely is '#'. + text = _extract_text(include_ligature=False, symbolic=False) + assert "#" in text + assert "GLYPH" not in text + + +def test_default_config_strips_glyph_markers(): + # Production default (keep_glyphs=False, see config.h): the marker is + # stripped to a space in pdf_states/text.h, so neither GLYPH<...> nor + # a fabricated '#' ever reaches the output. + text = _extract_text(include_ligature=False, symbolic=True, keep_glyphs=False) + assert "GLYPH" not in text + assert "#" not in text + assert "\u0634" in text and "\u0623" in text # covered codes still resolve