diff --git a/admin/docs/raw_image_input.md b/admin/docs/raw_image_input.md index eebea7b..95a7382 100644 --- a/admin/docs/raw_image_input.md +++ b/admin/docs/raw_image_input.md @@ -85,8 +85,8 @@ for it, so no report field carries a zone the evidence never had. - zstd-compressed SquashFS and UBIFS need Python 3.14 or later (`compression.zstd`), which the builds made by `test_builds.yml` use. Run from source on an older Python, a zstd SquashFS is listed as a volume with no files, - and the run log does not say why; a zstd-compressed UBIFS file is listed and - not staged, and the log names the reason. + and the run log gives the reason on that volume's line; a zstd-compressed UBIFS + file is listed and not staged, and the log names the reason. - An encrypted volume (Android file-based encryption, iOS data protection, FileVault, BitLocker) reads, but its names or contents are ciphertext. diff --git a/admin/test/scripts/test_raw_image_seeker.py b/admin/test/scripts/test_raw_image_seeker.py index 31cfe0d..e03ad73 100644 --- a/admin/test/scripts/test_raw_image_seeker.py +++ b/admin/test/scripts/test_raw_image_seeker.py @@ -488,6 +488,30 @@ def test_the_reader_finds_its_ewf_module_when_imported_as_a_package(self): self.assertIs(sys.modules['ewfprobe'], ewfprobe) self.assertIs(qnxprobe.ewfprobe, ewfprobe) + def test_a_socket_or_block_device_is_not_walked_as_a_directory(self): + # S_IFDIR is 0o040000, and a socket (0o140000) and a block device + # (0o060000) both carry that bit. Tested alone it lists them as + # directories and descends into them; on a macOS APFS volume that was + # 47 sockets under private/var listed as directories. + class _Walker: + root = 1 + children = {1: [('d', 2), ('sock', 3), ('blk', 4), ('f', 5)], + 2: [('inner', 6)], 3: [('ghost', 7)], 4: [('ghost', 8)]} + modes = {2: 0o040755, 3: 0o140755, 4: 0o060660, 5: 0o100644, + 6: 0o100644, 7: 0o100644, 8: 0o100644} + + def listdir(self, node): + return list(self.children.get(node, ())) + + def entry(self, node): + return self.modes[node], 3, 0 + + seeker = FileSeekerRaw.__new__(FileSeekerRaw) + seeker.name_list, seeker._entries = [], {} # pylint: disable=protected-access + files, dirs, _route = seeker._walk(_Walker(), 'v') # pylint: disable=protected-access + self.assertEqual(sorted(seeker.name_list), ['v/d/', 'v/d/inner', 'v/f']) + self.assertEqual((files, dirs), (2, 1)) + def test_the_filesystem_list_names_only_kinds_the_reader_walks(self): walkers = {'QNX6': qnxprobe.Qnx6Walker, 'QNX4': qnxprobe.Qnx4Walker, 'ETFS': qnxprobe.EtfsWalker, 'EFS': qnxprobe.EfsWalker, diff --git a/scripts/raw_image.py b/scripts/raw_image.py index 7d070fe..c6fc0fb 100644 --- a/scripts/raw_image.py +++ b/scripts/raw_image.py @@ -286,7 +286,9 @@ def _walk(self, walker, prefix): continue mode, size, mtime = ent member = f'{path}/{child_name}' - if mode & qnxprobe.S_IFDIR: + # The format bits, not the directory bit alone: a socket (0o140000) + # and a block device (0o060000) carry S_IFDIR's bit too. + if mode & qnxprobe.S_IFMT == qnxprobe.S_IFDIR: self.name_list.append(member + '/') self._entries[member + '/'] = None dirs += 1 diff --git a/scripts/vendor/qnxprobe.py b/scripts/vendor/qnxprobe.py index c43aba6..bbaa088 100755 --- a/scripts/vendor/qnxprobe.py +++ b/scripts/vendor/qnxprobe.py @@ -43,7 +43,7 @@ except ImportError: ewfprobe = None -QNXPROBE_VERSION = "1.33" +QNXPROBE_VERSION = "1.34" QNX6_MAGIC = 0x68191122 BOOTBLOCK_SIZE = 0x2000 @@ -5818,6 +5818,13 @@ def lz4_block_decompress(src, limit=None): return bytes(out) +# What a zstd SquashFS says on a Python with no zstd. identify_fs() reports it +# among its lines and the walker carries it as its note, which volumes() passes +# on, so a listing that comes back empty says why wherever the volume is shown. +SQUASHFS_NO_ZSTD_NOTE = ("zstd compressed; this Python has no zstd (3.14 adds it), so " + "the listing may be short and no file can be read") + + def _zstd_module(): try: from compression import zstd # Python 3.14 and later @@ -6078,6 +6085,15 @@ def readlink(self, ref): ino = self.inode(ref) return ino.get("target", b"").decode("utf-8", "surrogateescape") + @property + def note(self): + """Why this volume cannot be read on this Python, or None. listdir() + answers an empty list when its directory table cannot be decompressed, + so volumes() carries this beside the volume to say why.""" + if self.comp == 6 and _zstd_module() is None: + return SQUASHFS_NO_ZSTD_NOTE + return None + def listdir(self, ref): try: ino = self.inode(ref) @@ -6201,8 +6217,7 @@ def identify_squashfs(fh, base, size=None): lines.append(f"note the image records {human(w.bytes_used)} but the region " f"holds {human(size)}, so its end is missing") if w.comp == 6 and _zstd_module() is None: - lines.append("note zstd compressed; this Python has no zstd (3.14 adds " - "it), so the listing may be short and no file can be read") + lines.append("note " + SQUASHFS_NO_ZSTD_NOTE) if not root_ok: if w.comp == 6 and _zstd_module() is None: return "squashfs", lines @@ -9343,6 +9358,12 @@ def volumes(fh, size=None): vol["note"] = f"contents not read: {exc}" except Exception as exc: vol["note"] = f"could not walk this filesystem: {exc}" + # A walker that is built but cannot read its volume here says why, for + # a zstd SquashFS on a Python without zstd, whose listing comes back + # empty rather than raising. + wnote = getattr(vol.get("walker"), "note", None) + if wnote and "note" not in vol: + vol["note"] = wnote out.append(vol) return out @@ -13039,6 +13060,35 @@ def _vol_view(path): "--oob while mounted, against the kernel's own read-back"), ("ubi-nand-history", "ubi", "ubi-nand.history.kernel.sha256", None, "stat", "", (), "a static UBI volume the kernel wrote with ubiupdatevol, from the same image")] + # A zstd SquashFS on a Python with no zstd lists nothing, since its + # directory tables are compressed too, and the walker answers an empty + # listing rather than raising. volumes() has to say why beside the + # volume, or it reads as an empty filesystem. The missing module is + # simulated, so this runs on every Python. + zst_img = os.path.join(fx, "squashfs-zstd.img.gz") + if not os.path.isfile(zst_img): + print(" [SKIP] squashfs-zstd is not beside this script, so the note a " + "Python without zstd gives was not checked") + else: + with gzip.open(zst_img, "rb") as gz: + zst_bytes = gz.read() + zst_globals = globals() + zst_real = zst_globals["_zstd_module"] + try: + zst_globals["_zstd_module"] = lambda: None + zst_hidden = volumes(io.BytesIO(zst_bytes), len(zst_bytes)) + finally: + zst_globals["_zstd_module"] = zst_real + zst_shown = volumes(io.BytesIO(zst_bytes), len(zst_bytes)) + zst_cond = (len(zst_hidden) == 1 and zst_hidden[0]["kind"] == "squashfs" + and zst_hidden[0].get("note") == SQUASHFS_NO_ZSTD_NOTE + and len(zst_shown) == 1 + and (zst_shown[0].get("note") is None) == (zst_real() is not None)) + if not zst_cond: + ok = False + print(f" [{'PASS' if zst_cond else 'FAIL'}] a zstd SquashFS on a Python " + f"without zstd carries a note saying why it lists nothing, and none " + f"where zstd is present") for stem, want_kind, hashes, listing, style, prefix, loose, label in sq + jf + ub + ya + kh: img = os.path.join(fx, stem + ".img.gz") if not (os.path.isfile(img) and os.path.isfile(os.path.join(fx, hashes))): diff --git a/scripts/vendor/vendored.json b/scripts/vendor/vendored.json index b69120e..c0c6096 100644 --- a/scripts/vendor/vendored.json +++ b/scripts/vendor/vendored.json @@ -3,12 +3,12 @@ { "path": "scripts/vendor/qnxprobe.py", "name": "qnxprobe", - "version": "1.33", + "version": "1.34", "upstream": "https://github.com/abrignoni/qnxprobe", "upstream_file": "qnxprobe.py", - "commit": "e9bfb69e2d1556df67d5f1f8301dcf6f9a351749", - "commit_date": "2026-09-25T15:29:51-04:00", - "sha256": "15924af48c61fffb3fbbb1a7a8547f102341263d27970c58f71fba98df7bcdc8", + "commit": "fc74ca829c2773879243b326f2262c29f462df3a", + "commit_date": "2026-09-25T16:01:32-04:00", + "sha256": "ab0f681313bc305f3662d4421199f45da357da3becf90b28696a158fbaae2710", "licence": "MIT", "licence_file": "scripts/vendor/LICENSE-qnxprobe", "note": "Copied verbatim. Fix upstream and re-vendor; edits here are reverted by the next sync."