From 5d5e94ae830731afa6093ef81257073ee8883d57 Mon Sep 17 00:00:00 2001 From: Ujjwal Reddy K S Date: Wed, 12 Aug 2026 15:16:43 -0400 Subject: [PATCH 1/3] fix(filetype): implement fallback encoding detection for file-like objects --- test_unstructured/file_utils/test_filetype.py | 10 ++++------ unstructured/file_utils/filetype.py | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test_unstructured/file_utils/test_filetype.py b/test_unstructured/file_utils/test_filetype.py index e942714373..2bdf7f7575 100644 --- a/test_unstructured/file_utils/test_filetype.py +++ b/test_unstructured/file_utils/test_filetype.py @@ -1105,15 +1105,16 @@ def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_file_pat assert len(text_head) == 4096 assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") - def but_not_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self): - """Fails silently in this case, returning empty string.""" + def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self): with open(example_doc_path("norwich-city.txt"), "rb") as f: file = io.BytesIO(f.read()) ctx = _FileTypeDetectionContext(file=file, encoding="utf_32_be") text_head = ctx.text_head - assert text_head == "" + assert isinstance(text_head, str) + assert len(text_head) == 4096 + assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differentiation(self): with open(example_doc_path("norwich-city.txt"), "rb") as f: @@ -1147,9 +1148,6 @@ def it_accommodates_a_utf_32_encoded_file_path(self): assert len(text_head) == 188 assert text_head.startswith("This is a test document to use for unit tests.\n\n Doyle") - # TODO: this fails because `.text_head` ignores decoding errors on a file open for binary - # reading. Probably better if it used chardet in that case as it does for a file-path. - @pytest.mark.xfail(reason="WIP", raises=AssertionError, strict=True) def and_it_accommodates_a_utf_32_encoded_file_like_object(self): with open(example_doc_path("fake-text-utf-32.txt"), "rb") as f: file = io.BytesIO(f.read()) diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index a12a6249e9..e8da49b113 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -674,18 +674,22 @@ def text_head(self) -> str: Raises: UnicodeDecodeError if file cannot be read as text. """ - # TODO: only attempts fallback character-set detection for file-path case, not for - # file-like object case. Seems like we should do both. - if file := self._file_arg: file.seek(0) content = file.read(4096) file.seek(0) - return ( - content - if isinstance(content, str) - else content.decode(encoding=self.encoding, errors="ignore") - ) + + if isinstance(content, str): + return content + + try: + return content.decode(encoding=self.encoding) + except UnicodeDecodeError: + try: + encoding, _ = detect_file_encoding(file=content) + return content.decode(encoding=encoding) + except Exception: + return content.decode(encoding=self.encoding, errors="ignore") file_path = self.file_path assert file_path is not None # -- guaranteed by `._validate` -- From ddbb239c01d4246592c175e5c2ab5a5c51edc91b Mon Sep 17 00:00:00 2001 From: Ujjwal Reddy K S Date: Wed, 12 Aug 2026 15:26:13 -0400 Subject: [PATCH 2/3] fix(filetype): propagate detect_file_encoding errors and restrict lenient fallback to final decode --- test_unstructured/file_utils/test_filetype.py | 2 +- unstructured/file_utils/filetype.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/test_unstructured/file_utils/test_filetype.py b/test_unstructured/file_utils/test_filetype.py index 2bdf7f7575..1f90b33d13 100644 --- a/test_unstructured/file_utils/test_filetype.py +++ b/test_unstructured/file_utils/test_filetype.py @@ -1113,7 +1113,7 @@ def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_a_file_l text_head = ctx.text_head assert isinstance(text_head, str) - assert len(text_head) == 4096 + assert len(text_head) == 4063 assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differentiation(self): diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index e8da49b113..ad08786b22 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -685,11 +685,8 @@ def text_head(self) -> str: try: return content.decode(encoding=self.encoding) except UnicodeDecodeError: - try: - encoding, _ = detect_file_encoding(file=content) - return content.decode(encoding=encoding) - except Exception: - return content.decode(encoding=self.encoding, errors="ignore") + encoding, _ = detect_file_encoding(file=content) + return content.decode(encoding=encoding, errors="ignore") file_path = self.file_path assert file_path is not None # -- guaranteed by `._validate` -- From b9bf177e822426ddb09664c602425e26d7f1e360 Mon Sep 17 00:00:00 2001 From: Ujjwal Reddy K S Date: Wed, 12 Aug 2026 15:36:18 -0400 Subject: [PATCH 3/3] docs(filetype): update text_head docstring to include UnprocessableEntityError --- unstructured/file_utils/filetype.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index ad08786b22..3219259056 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -673,6 +673,7 @@ def text_head(self) -> str: Raises: UnicodeDecodeError if file cannot be read as text. + UnprocessableEntityError if file encoding cannot be determined. """ if file := self._file_arg: file.seek(0)