diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0d32d2a8..3f3b900576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.26.3 + +### Fixes + +- **Use fallback character-set detection for file-like objects**: `FileTypeDetectionContext.text_head()` now applies the same `detect_file_encoding()` fallback to file-like objects as it does to file paths when the declared encoding cannot decode the content. Previously it decoded with `errors="ignore"`, silently stripping characters and corrupting the text head for non-UTF-8 streams such as S3/GCS objects and API uploads. + ## 0.26.2 ### Fixes diff --git a/test_unstructured/file_utils/test_filetype.py b/test_unstructured/file_utils/test_filetype.py index e942714373..376bd8245f 100644 --- a/test_unstructured/file_utils/test_filetype.py +++ b/test_unstructured/file_utils/test_filetype.py @@ -1105,15 +1105,32 @@ def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_file_pat assert len(text_head) == 4096 assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") - def but_not_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self): - """Fails silently in this case, returning empty string.""" + def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_a_file_like_object( + self, + ): + """Fallback character detection corrects a wrong encoding arg, like the file-path case.""" with open(example_doc_path("norwich-city.txt"), "rb") as f: file = io.BytesIO(f.read()) ctx = _FileTypeDetectionContext(file=file, encoding="utf_32_be") text_head = ctx.text_head - assert text_head == "" + assert isinstance(text_head, str) + assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") + + def and_it_detects_a_non_utf8_file_like_object_instead_of_stripping_its_characters(self): + """A non-UTF-8 file-like object is decoded via fallback detection, not errors="ignore". + + Regression for #4434: decoding with errors="ignore" silently stripped the + undecodable characters, corrupting the text head for cloud-storage streams. + """ + content = "café à la résumé".encode("iso_8859_1") + ctx = _FileTypeDetectionContext(file=io.BytesIO(content)) + + text_head = ctx.text_head + + assert "café" in text_head + assert "résumé" in text_head def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differentiation(self): with open(example_doc_path("norwich-city.txt"), "rb") as f: @@ -1126,6 +1143,23 @@ def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differenti assert len(text_head) == 4063 assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") + def and_it_runs_character_detection_for_a_truncated_utf8_tail(self): + """A truncated stream must fall through to detection, not silently drop the tail.""" + content = b"caf\xc3" + ctx = _FileTypeDetectionContext(file=io.BytesIO(content)) + + # -- a silently-truncated decode would return exactly "caf" -- + assert ctx.text_head != "caf" + + def and_it_does_not_mistake_a_boundary_split_character_for_a_wrong_encoding(self): + """A multi-byte character split by the 4096-byte read is not a decode error.""" + content = b"a" * 4095 + "é".encode() + ctx = _FileTypeDetectionContext(file=io.BytesIO(content)) + + text_head = ctx.text_head + + assert text_head == "a" * 4095 + def and_it_grabs_the_first_4k_chars_from_text_file_for_textual_type_differentiation(self): """Not a documented behavior to accept IO[str], but support is implemented.""" with open(example_doc_path("norwich-city.txt")) as f: @@ -1147,9 +1181,6 @@ def it_accommodates_a_utf_32_encoded_file_path(self): assert len(text_head) == 188 assert text_head.startswith("This is a test document to use for unit tests.\n\n Doyle") - # TODO: this fails because `.text_head` ignores decoding errors on a file open for binary - # reading. Probably better if it used chardet in that case as it does for a file-path. - @pytest.mark.xfail(reason="WIP", raises=AssertionError, strict=True) def and_it_accommodates_a_utf_32_encoded_file_like_object(self): with open(example_doc_path("fake-text-utf-32.txt"), "rb") as f: file = io.BytesIO(f.read()) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 090d6d4385..977b2e2c88 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.26.2" # pragma: no cover +__version__ = "0.26.3" # pragma: no cover diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index a12a6249e9..d3032b63a9 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -28,6 +28,7 @@ from __future__ import annotations +import codecs import contextlib import functools import importlib.util @@ -671,21 +672,42 @@ def rule_out_zip_content_types(self) -> None: def text_head(self) -> str: """The initial characters of the text file for use with text-format differentiation. + Uses fallback character-set detection when the declared encoding cannot + decode the content, for both file paths and file-like objects. + Raises: - UnicodeDecodeError if file cannot be read as text. + UnprocessableEntityError when the file cannot be decoded with the declared + encoding or any of the common fallback encodings. """ - # TODO: only attempts fallback character-set detection for file-path case, not for - # file-like object case. Seems like we should do both. if file := self._file_arg: file.seek(0) content = file.read(4096) + if not isinstance(content, str): + eof_reached = len(file.read(1)) == 0 file.seek(0) - return ( - content - if isinstance(content, str) - else content.decode(encoding=self.encoding, errors="ignore") - ) + if isinstance(content, str): + return content + try: + return content.decode(encoding=self.encoding) + except (UnicodeDecodeError, UnicodeError): + # A multi-byte character split at the 4096-byte read boundary + # raises UnicodeDecodeError even though the content is validly + # encoded. Decode incrementally with final=eof_reached so an + # incomplete trailing sequence is buffered when more content + # follows, while a genuinely truncated stream still falls + # through to character-set detection. + decoder = codecs.getincrementaldecoder(self.encoding)() + try: + return decoder.decode(content, final=eof_reached) + except (UnicodeDecodeError, UnicodeError): + # Use the same fallback character-set detection as the + # file-path branch. Decoding with errors="ignore" silently + # stripped undecodable characters and corrupted the text + # head for non-UTF-8 streams (S3/GCS objects, API uploads) + # — issue #4434. + _, file_text = detect_file_encoding(file=content) + return file_text[:4096] file_path = self.file_path assert file_path is not None # -- guaranteed by `._validate` --