diff --git a/test_unstructured/file_utils/test_filetype.py b/test_unstructured/file_utils/test_filetype.py index e942714373..1f90b33d13 100644 --- a/test_unstructured/file_utils/test_filetype.py +++ b/test_unstructured/file_utils/test_filetype.py @@ -1105,15 +1105,16 @@ def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_file_pat assert len(text_head) == 4096 assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") - def but_not_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self): - """Fails silently in this case, returning empty string.""" + def and_it_uses_character_detection_to_correct_a_wrong_encoding_arg_for_a_file_like_object_open_in_binary_mode(self): with open(example_doc_path("norwich-city.txt"), "rb") as f: file = io.BytesIO(f.read()) ctx = _FileTypeDetectionContext(file=file, encoding="utf_32_be") text_head = ctx.text_head - assert text_head == "" + assert isinstance(text_head, str) + assert len(text_head) == 4063 + assert text_head.startswith("Iwan Roberts\nRoberts celebrating after") def and_it_grabs_the_first_4k_chars_from_binary_file_for_textual_type_differentiation(self): with open(example_doc_path("norwich-city.txt"), "rb") as f: @@ -1147,9 +1148,6 @@ def it_accommodates_a_utf_32_encoded_file_path(self): assert len(text_head) == 188 assert text_head.startswith("This is a test document to use for unit tests.\n\n Doyle") - # TODO: this fails because `.text_head` ignores decoding errors on a file open for binary - # reading. Probably better if it used chardet in that case as it does for a file-path. - @pytest.mark.xfail(reason="WIP", raises=AssertionError, strict=True) def and_it_accommodates_a_utf_32_encoded_file_like_object(self): with open(example_doc_path("fake-text-utf-32.txt"), "rb") as f: file = io.BytesIO(f.read()) diff --git a/unstructured/file_utils/filetype.py b/unstructured/file_utils/filetype.py index a12a6249e9..3219259056 100644 --- a/unstructured/file_utils/filetype.py +++ b/unstructured/file_utils/filetype.py @@ -673,19 +673,21 @@ def text_head(self) -> str: Raises: UnicodeDecodeError if file cannot be read as text. + UnprocessableEntityError if file encoding cannot be determined. """ - # TODO: only attempts fallback character-set detection for file-path case, not for - # file-like object case. Seems like we should do both. - if file := self._file_arg: file.seek(0) content = file.read(4096) file.seek(0) - return ( - content - if isinstance(content, str) - else content.decode(encoding=self.encoding, errors="ignore") - ) + + if isinstance(content, str): + return content + + try: + return content.decode(encoding=self.encoding) + except UnicodeDecodeError: + encoding, _ = detect_file_encoding(file=content) + return content.decode(encoding=encoding, errors="ignore") file_path = self.file_path assert file_path is not None # -- guaranteed by `._validate` --