fix(security): bound quadratic array-stream decoding in is_pdf_too_complex (SEC-146)

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.26.1
+
+### Fixes
+
+- **Bound array-stream decoding in `is_pdf_too_complex` (SEC-146)**: fixes a quadratic `bytes +=` accumulation over array-based `/Contents` (CVE-2026-33123) that let a crafted PDF spike CPU/memory. Accumulation now uses a `bytearray`, with per-page and document-wide caps on decoded bytes and array entries that fail closed on crafted content, and every file is inspected by default (small files are no longer skipped, since a small compressed file can declare huge content). Indirect `/Contents` arrays are now dereferenced (they were being skipped), a single unreadable stream no longer skips the rest of its page, and operator counting no longer allocates a full match list. Bumps `pypdf` to `>=6.9.1` so its own patched code path is used.
+
 ## 0.26.0
 
 ### Fixes

pyproject.toml

@@ -68,7 +68,7 @@ image = [
     "pdfminer.six>=20251230, <20270000",
     "pi-heif>=1.2.0, <2.0.0",
     "pikepdf>=10.3.0, <11.0.0",
-    "pypdf>=6.6.2, <7.0.0",
+    "pypdf>=6.9.1, <7.0.0",
     "unstructured-inference>=1.6.12, <2.0.0; platform_system != 'Windows'",
     "unstructured-inference>=1.6.12, <2.0.0; platform_system == 'Windows' and python_version >= '3.12' and python_version < '3.13'",
     "unstructured-pytesseract>=0.3.15, <1.0.0",

test_unstructured/partition/pdf_image/test_pdf.py

@@ -6,6 +6,8 @@
 import math
 import os
 import tempfile
+import time
+import zlib
 from dataclasses import dataclass
 from importlib import reload
 from pathlib import Path
@@ -16,6 +18,9 @@
 import pytest
 from pdf2image.exceptions import PDFPageCountError
 from PIL import Image
+from pypdf import PdfWriter
+from pypdf.errors import LimitReachedError
+from pypdf.generic import ArrayObject, DecodedStreamObject, NameObject, NullObject
 from pytest_mock import MockFixture
 from unstructured_inference.inference import layout, pdf_image
 from unstructured_inference.inference.elements import Rectangle
@@ -1698,6 +1703,26 @@ def test_is_pdf_too_complex_skips_small_file_size():
     assert not pdf.is_pdf_too_complex(file=b"tiny", min_file_size_bytes=10)
 
 
+def test_is_pdf_too_complex_inspects_small_files_by_default():
+    """A small compressed file can still declare huge decoded content, so the default
+    (min_file_size_bytes=0) must inspect it rather than skip on file size."""
+
+    # One 100 KB stream referenced 9,000 times: a ~55 KB file, ~900 MB nominal decoded.
+    stream = DecodedStreamObject()
+    stream[NameObject("/Filter")] = NameObject("/FlateDecode")
+    stream._data = zlib.compress(b"\x00" * 100_000)
+    writer = PdfWriter()
+    writer.add_blank_page(width=200, height=200)
+    ref = writer._add_object(stream)
+    writer.pages[0][NameObject("/Contents")] = ArrayObject([ref for _ in range(9_000)])
+    buffer = io.BytesIO()
+    writer.write(buffer)
+    data = buffer.getvalue()
+
+    assert len(data) < 1024 * 1024  # under the old 1 MB skip threshold
+    assert pdf.is_pdf_too_complex(file=data)  # defaults; no min_file_size_bytes override
+
+
 def test_is_pdf_too_complex_detects_vector_heavy_page():
     class MockStream:
         def get_data(self):
@@ -1767,6 +1792,288 @@ def test_is_pdf_too_complex_returns_false_for_normal_pdf():
     assert not pdf.is_pdf_too_complex(filename=example_doc_path("pdf/layout-parser-paper.pdf"))
 
 
+def _pdf_with_content_stream_array(
+    per_stream_payload: bytes,
+    num_streams: int,
+    *,
+    indirect_array: bool = False,
+) -> bytes:
+    """One-page PDF whose ``/Contents`` is an array of ``num_streams`` FlateDecode streams
+    (small file, huge decoded output -- CVE-2026-33123). pypdf private API keeps it small."""
+    writer = PdfWriter()
+    writer.add_blank_page(width=200, height=200)
+    compressed = zlib.compress(per_stream_payload)
+
+    refs = ArrayObject()
+    for _ in range(num_streams):
+        stream = DecodedStreamObject()
+        stream[NameObject("/Filter")] = NameObject("/FlateDecode")
+        stream._data = compressed
+        refs.append(writer._add_object(stream))
+
+    contents = writer._add_object(refs) if indirect_array else refs
+    writer.pages[0][NameObject("/Contents")] = contents
+
+    buffer = io.BytesIO()
+    writer.write(buffer)
+    return buffer.getvalue()
+
+
+@pytest.mark.parametrize("indirect_array", [False, True], ids=["direct", "indirect"])
+def test_is_pdf_too_complex_flags_graphics_heavy_content_array(indirect_array):
+    """A direct or indirect array of graphics-heavy streams is flagged too complex.
+    The indirect case guards the dereference: it used to skip the array branch."""
+
+    payload = b" ".join([b"m"] * 400 + [b"Tj"] * 2)  # graphics-heavy, ratio 200:1
+    data = _pdf_with_content_stream_array(payload, num_streams=300, indirect_array=indirect_array)
+
+    assert pdf.is_pdf_too_complex(
+        file=data,
+        max_graphics_ops=100,
+        min_graphics_to_text_ratio=20.0,
+        min_file_size_bytes=1,
+        min_raw_stream_bytes=1,
+    )
+
+
+def test_is_pdf_too_complex_bounds_array_of_many_streams():
+    """CVE-2026-33123 regression: a ~2 MB file whose array decodes to ~900 MB ran for
+    minutes / OOM'd on the old `bytes +=`; the fix caps it and returns almost at once."""
+
+    data = _pdf_with_content_stream_array(b"\x00" * 100_000, num_streams=9_000)
+    assert len(data) < 10 * 1024 * 1024  # small file, huge nominal decoded size
+
+    start = time.perf_counter()
+    result = pdf.is_pdf_too_complex(file=data, min_file_size_bytes=1, min_raw_stream_bytes=1)
+    elapsed = time.perf_counter() - start
+
+    # Decoded content blows past the 50 MB per-page cap, so the page fails closed.
+    assert result is True
+    assert elapsed < 10.0, f"is_pdf_too_complex took {elapsed:.2f}s -- accumulation is not bounded"
+
+
+def test_is_pdf_too_complex_caps_content_array_entries():
+    """An array of many empty/tiny streams cannot force unbounded work: the entry-count
+    cap short-circuits before any stream is decoded, and the page fails closed."""
+
+    call_count = 0
+
+    class EmptyStream:
+        def get_data(self):
+            nonlocal call_count
+            call_count += 1
+            return b""
+
+    num_streams = 50_000
+    contents = ArrayObject([EmptyStream() for _ in range(num_streams)])
+
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": contents}]
+
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        result = pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_content_stream_array_entries=10_000,
+        )
+
+    assert result is True
+    assert call_count == 0  # cap checked against len(contents) before any decode
+
+
+def test_is_pdf_too_complex_caps_oversized_stream_before_copy():
+    """A stream over the byte cap fails closed before being copied/scanned, in both the
+    array and standalone branches."""
+
+    class BigStream:
+        def get_data(self):
+            return b"a" * 25_000
+
+    # Array branch: oversized entry trips the cap on the first item.
+    array_stream = BigStream()
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": ArrayObject([array_stream, BigStream(), BigStream()])}]
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        assert pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_raw_stream_bytes=10_000,
+        )
+
+    # Standalone (non-array) branch: oversized single stream also fails closed.
+    single_stream = BigStream()
+    reader.pages = [{"/Contents": single_stream}]
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        assert pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_raw_stream_bytes=10_000,
+        )
+
+
+def test_is_pdf_too_complex_bounds_total_bytes_across_pages():
+    """Pages sharing one array stay under the per-page cap, so only the document byte
+    budget can fail them closed once the decoded total exceeds it."""
+
+    # Graphics-light filler so no page trips the ratio -- the byte budget must be what fails.
+    payload = b"\x00" * 40_000  # 40 KB per page, well under the per-page cap
+    stream = DecodedStreamObject()
+    stream[NameObject("/Filter")] = NameObject("/FlateDecode")
+    stream._data = zlib.compress(payload)
+
+    writer = PdfWriter()
+    shared_ref = writer._add_object(stream)  # one stream, referenced by every page
+    for _ in range(10):
+        writer.add_blank_page(width=200, height=200)
+        writer.pages[-1][NameObject("/Contents")] = ArrayObject([shared_ref])
+    buffer = io.BytesIO()
+    writer.write(buffer)
+    data = buffer.getvalue()
+
+    # 250 KB budget below the 400 KB document total -> fail closed.
+    assert pdf.is_pdf_too_complex(
+        file=data,
+        min_file_size_bytes=1,
+        min_raw_stream_bytes=1,
+        max_raw_stream_bytes=1_000_000,
+        max_total_stream_bytes=250_000,
+    )
+
+
+def test_is_pdf_too_complex_document_budget_survives_decode_errors():
+    """Bytes are charged per decoded stream, so a `[valid, raises]` array still counts
+    the valid stream -- a mid-page decode error can't discard the accounting."""
+
+    good = DecodedStreamObject()
+    good[NameObject("/Filter")] = NameObject("/FlateDecode")
+    good._data = zlib.compress(b"\x00" * 90_000)  # decodes to 90 KB
+
+    bad = DecodedStreamObject()
+    bad[NameObject("/Filter")] = NameObject("/UnknownBogusFilter")  # get_data() raises
+    bad._data = b"garbage"
+
+    writer = PdfWriter()
+    good_ref = writer._add_object(good)
+    bad_ref = writer._add_object(bad)
+    shared_ref = writer._add_object(ArrayObject([good_ref, bad_ref]))
+    for _ in range(10):
+        writer.add_blank_page(width=200, height=200)
+        writer.pages[-1][NameObject("/Contents")] = shared_ref
+    buffer = io.BytesIO()
+    writer.write(buffer)
+    data = buffer.getvalue()
+
+    # Two valid 90 KB streams exceed the 150 KB budget despite each page's second raising.
+    assert pdf.is_pdf_too_complex(
+        file=data,
+        min_file_size_bytes=1,
+        min_raw_stream_bytes=1,
+        max_raw_stream_bytes=100_000,
+        max_total_stream_bytes=150_000,
+    )
+
+
+@pytest.mark.parametrize("array_contents", [False, True], ids=["standalone", "array"])
+def test_is_pdf_too_complex_fails_closed_on_decoder_limit(array_contents):
+    """A stream that raises LimitReachedError (pypdf's decode-limit, e.g. a compression
+    bomb) fails the page closed rather than being skipped and handed to PDFMiner."""
+
+    class BombStream:
+        def get_data(self):
+            raise LimitReachedError("Limit reached while decompressing.")
+
+    contents = ArrayObject([BombStream()]) if array_contents else BombStream()
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": contents}]
+
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        assert pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+        )
+
+
+def test_is_pdf_too_complex_unreadable_stream_does_not_skip_rest_of_page():
+    """One stream that fails to decode skips only itself; the remaining streams on the
+    page are still inspected, so a bad sibling can't mask an over-cap stream."""
+
+    class BadStream:
+        def get_data(self):
+            raise NotImplementedError("unsupported filter")
+
+    class ValidStream:
+        def get_data(self):
+            return b"a" * 20_000  # over the 10 KB cap below
+
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": ArrayObject([BadStream(), ValidStream()])}]
+
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        # Old behavior skipped the whole page on the bad stream and returned False.
+        assert pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_raw_stream_bytes=10_000,
+        )
+
+
+def test_is_pdf_too_complex_bounds_total_entries_across_pages():
+    """The document entry budget bounds total streams decoded, so empty streams (which
+    never move the byte budget) can't scale work with page count."""
+
+    get_data_calls = 0
+
+    class EmptyStream:
+        def get_data(self):
+            nonlocal get_data_calls
+            get_data_calls += 1
+            return b""
+
+    # One 1,000-entry array (under the 10,000 per-page cap) shared by every page.
+    shared = ArrayObject([EmptyStream() for _ in range(1_000)])
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": shared} for _ in range(100)]
+
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        result = pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_content_stream_array_entries=10_000,  # per-page cap NOT hit (1,000 < 10,000)
+            max_total_array_entries=5_000,
+        )
+
+    assert result is True  # fail closed once the entry budget is exhausted
+    assert get_data_calls == 5_000  # bounded by the budget, not the 100k possible decodes
+
+
+def test_is_pdf_too_complex_charges_non_stream_entries_to_budget():
+    """The entry budget charges every slot, so a shared array of non-stream objects
+    (nulls) is bounded too -- charging only streams left traversal scaling with pages."""
+
+    # 9,999 non-stream entries (under the 10,000 per-page cap), shared across many pages.
+    shared = ArrayObject([NullObject() for _ in range(9_999)])
+    reader = mock.Mock()
+    reader.pages = [{"/Contents": shared} for _ in range(200)]
+
+    with mock.patch.object(pdf, "PdfReader", return_value=reader):
+        result = pdf.is_pdf_too_complex(
+            file=b"x" * 20,
+            min_file_size_bytes=1,
+            min_raw_stream_bytes=1,
+            max_content_stream_array_entries=10_000,  # per-page cap NOT hit (9,999 < 10,000)
+            max_total_array_entries=1,
+        )
+
+    # Fails closed on page 1; the pre-fix code charged only streams and returned False.
+    assert result is True
+
+
 def test_document_to_element_list_omits_coord_system_when_coord_points_absent():
     # TODO (yao): investigate why we need this test. The LayoutElement definition suggests bbox
     # can't be None and it has to be a Rectangle object that has x1, y1, x2, y2 attributes.

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.26.0"  # pragma: no cover
+__version__ = "0.26.1"  # pragma: no cover