fix: fallback character-set detection for file-like objects

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.26.3
+
+### Fixes
+
+- **Use fallback character-set detection for file-like objects**: `FileTypeDetectionContext.text_head()` now applies the same `detect_file_encoding()` fallback to file-like objects as it does to file paths when the declared encoding cannot decode the content. Previously it decoded with `errors="ignore"`, silently stripping characters and corrupting the text head for non-UTF-8 streams such as S3/GCS objects and API uploads.
+
 ## 0.26.2
 
 ### Fixes

test_unstructured/file_utils/test_filetype.py

@@ -1105,15 +1105,32 @@ def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_file_pat
         assert len(text_head) == 4096
         assert text_head.startswith("Iwan Roberts\nRoberts celebrating after")
 
-    def but_not_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self):
-        """Fails silently in this case, returning empty string."""
+    def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_a_file_like_object(
+        self,
+    ):
+        """Fallback character detection corrects a wrong encoding arg, like the file-path case."""
         with open(example_doc_path("norwich-city.txt"), "rb") as f:
             file = io.BytesIO(f.read())
         ctx = _FileTypeDetectionContext(file=file, encoding="utf_32_be")
 
         text_head = ctx.text_head
 
-        assert text_head == ""
+        assert isinstance(text_head, str)
+        assert text_head.startswith("Iwan Roberts\nRoberts celebrating after")
+
+    def and_it_detects_a_non_utf8_file_like_object_instead_of_stripping_its_characters(self):
+        """A non-UTF-8 file-like object is decoded via fallback detection, not errors="ignore".
+
+        Regression for #4434: decoding with errors="ignore" silently stripped the
+        undecodable characters, corrupting the text head for cloud-storage streams.
+        """
+        content = "café à la résumé".encode("iso_8859_1")
+        ctx = _FileTypeDetectionContext(file=io.BytesIO(content))
+
+        text_head = ctx.text_head
+
+        assert "café" in text_head
+        assert "résumé" in text_head
 
     def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differentiation(self):
         with open(example_doc_path("norwich-city.txt"), "rb") as f:
@@ -1126,6 +1143,23 @@ def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differenti
             assert len(text_head) == 4063
             assert text_head.startswith("Iwan Roberts\nRoberts celebrating after")
 
+    def and_it_runs_character_detection_for_a_truncated_utf8_tail(self):
+        """A truncated stream must fall through to detection, not silently drop the tail."""
+        content = b"caf\xc3"
+        ctx = _FileTypeDetectionContext(file=io.BytesIO(content))
+
+        # -- a silently-truncated decode would return exactly "caf" --
+        assert ctx.text_head != "caf"
+
+    def and_it_does_not_mistake_a_boundary_split_character_for_a_wrong_encoding(self):
+        """A multi-byte character split by the 4096-byte read is not a decode error."""
+        content = b"a" * 4095 + "é".encode()
+        ctx = _FileTypeDetectionContext(file=io.BytesIO(content))
+
+        text_head = ctx.text_head
+
+        assert text_head == "a" * 4095
+
     def and_it_grabs_the_first_4k_chars_from_text_file_for_textual_type_differentiation(self):
         """Not a documented behavior to accept IO[str], but support is implemented."""
         with open(example_doc_path("norwich-city.txt")) as f:
@@ -1147,9 +1181,6 @@ def it_accommodates_a_utf_32_encoded_file_path(self):
         assert len(text_head) == 188
         assert text_head.startswith("This is a test document to use for unit tests.\n\n    Doyle")
 
-    # TODO: this fails because `.text_head` ignores decoding errors on a file open for binary
-    # reading. Probably better if it used chardet in that case as it does for a file-path.
-    @pytest.mark.xfail(reason="WIP", raises=AssertionError, strict=True)
     def and_it_accommodates_a_utf_32_encoded_file_like_object(self):
         with open(example_doc_path("fake-text-utf-32.txt"), "rb") as f:
             file = io.BytesIO(f.read())

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.26.2"  # pragma: no cover
+__version__ = "0.26.3"  # pragma: no cover

unstructured/file_utils/filetype.py

@@ -28,6 +28,7 @@
 
 from __future__ import annotations
 
+import codecs
 import contextlib
 import functools
 import importlib.util
@@ -671,21 +672,42 @@ def rule_out_zip_content_types(self) -> None:
     def text_head(self) -> str:
         """The initial characters of the text file for use with text-format differentiation.
 
+        Uses fallback character-set detection when the declared encoding cannot
+        decode the content, for both file paths and file-like objects.
+
         Raises:
-            UnicodeDecodeError if file cannot be read as text.
+            UnprocessableEntityError when the file cannot be decoded with the declared
+            encoding or any of the common fallback encodings.
         """
-        # TODO: only attempts fallback character-set detection for file-path case, not for
-        # file-like object case. Seems like we should do both.
 
         if file := self._file_arg:
             file.seek(0)
             content = file.read(4096)
+            if not isinstance(content, str):
+                eof_reached = len(file.read(1)) == 0
             file.seek(0)
-            return (
-                content
-                if isinstance(content, str)
-                else content.decode(encoding=self.encoding, errors="ignore")
-            )
+            if isinstance(content, str):
+                return content
+            try:
+                return content.decode(encoding=self.encoding)
+            except (UnicodeDecodeError, UnicodeError):
+                # A multi-byte character split at the 4096-byte read boundary
+                # raises UnicodeDecodeError even though the content is validly
+                # encoded. Decode incrementally with final=eof_reached so an
+                # incomplete trailing sequence is buffered when more content
+                # follows, while a genuinely truncated stream still falls
+                # through to character-set detection.
+                decoder = codecs.getincrementaldecoder(self.encoding)()
+                try:
+                    return decoder.decode(content, final=eof_reached)
+                except (UnicodeDecodeError, UnicodeError):
+                    # Use the same fallback character-set detection as the
+                    # file-path branch. Decoding with errors="ignore" silently
+                    # stripped undecodable characters and corrupted the text
+                    # head for non-UTF-8 streams (S3/GCS objects, API uploads)
+                    # — issue #4434.
+                    _, file_text = detect_file_encoding(file=content)
+                    return file_text[:4096]
 
         file_path = self.file_path
         assert file_path is not None  # -- guaranteed by `._validate` --