Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions admin/docs/raw_image_input.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ for it, so no report field carries a zone the evidence never had.
- zstd-compressed SquashFS and UBIFS need Python 3.14 or later
(`compression.zstd`), which the builds made by `test_builds.yml` use. Run from
source on an older Python, a zstd SquashFS is listed as a volume with no files,
and the run log does not say why; a zstd-compressed UBIFS file is listed and
not staged, and the log names the reason.
and the run log gives the reason on that volume's line; a zstd-compressed UBIFS
file is listed and not staged, and the log names the reason.
- An encrypted volume (Android file-based encryption, iOS data protection,
FileVault, BitLocker) reads, but its names or contents are ciphertext.

Expand Down
24 changes: 24 additions & 0 deletions admin/test/scripts/test_raw_image_seeker.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,30 @@ def test_the_reader_finds_its_ewf_module_when_imported_as_a_package(self):
self.assertIs(sys.modules['ewfprobe'], ewfprobe)
self.assertIs(qnxprobe.ewfprobe, ewfprobe)

def test_a_socket_or_block_device_is_not_walked_as_a_directory(self):
# S_IFDIR is 0o040000, and a socket (0o140000) and a block device
# (0o060000) both carry that bit. Tested alone it lists them as
# directories and descends into them; on a macOS APFS volume that was
# 47 sockets under private/var listed as directories.
class _Walker:
root = 1
children = {1: [('d', 2), ('sock', 3), ('blk', 4), ('f', 5)],
2: [('inner', 6)], 3: [('ghost', 7)], 4: [('ghost', 8)]}
modes = {2: 0o040755, 3: 0o140755, 4: 0o060660, 5: 0o100644,
6: 0o100644, 7: 0o100644, 8: 0o100644}

def listdir(self, node):
return list(self.children.get(node, ()))

def entry(self, node):
return self.modes[node], 3, 0

seeker = FileSeekerRaw.__new__(FileSeekerRaw)
seeker.name_list, seeker._entries = [], {} # pylint: disable=protected-access
files, dirs, _route = seeker._walk(_Walker(), 'v') # pylint: disable=protected-access
self.assertEqual(sorted(seeker.name_list), ['v/d/', 'v/d/inner', 'v/f'])
self.assertEqual((files, dirs), (2, 1))

def test_the_filesystem_list_names_only_kinds_the_reader_walks(self):
walkers = {'QNX6': qnxprobe.Qnx6Walker, 'QNX4': qnxprobe.Qnx4Walker,
'ETFS': qnxprobe.EtfsWalker, 'EFS': qnxprobe.EfsWalker,
Expand Down
4 changes: 3 additions & 1 deletion scripts/raw_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,9 @@ def _walk(self, walker, prefix):
continue
mode, size, mtime = ent
member = f'{path}/{child_name}'
if mode & qnxprobe.S_IFDIR:
# The format bits, not the directory bit alone: a socket (0o140000)
# and a block device (0o060000) carry S_IFDIR's bit too.
if mode & qnxprobe.S_IFMT == qnxprobe.S_IFDIR:
self.name_list.append(member + '/')
self._entries[member + '/'] = None
dirs += 1
Expand Down
56 changes: 53 additions & 3 deletions scripts/vendor/qnxprobe.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
except ImportError:
ewfprobe = None

QNXPROBE_VERSION = "1.33"
QNXPROBE_VERSION = "1.34"

QNX6_MAGIC = 0x68191122
BOOTBLOCK_SIZE = 0x2000
Expand Down Expand Up @@ -5818,6 +5818,13 @@ def lz4_block_decompress(src, limit=None):
return bytes(out)


# What a zstd SquashFS says on a Python with no zstd. identify_fs() reports it
# among its lines and the walker carries it as its note, which volumes() passes
# on, so a listing that comes back empty says why wherever the volume is shown.
SQUASHFS_NO_ZSTD_NOTE = ("zstd compressed; this Python has no zstd (3.14 adds it), so "
"the listing may be short and no file can be read")


def _zstd_module():
try:
from compression import zstd # Python 3.14 and later
Expand Down Expand Up @@ -6078,6 +6085,15 @@ def readlink(self, ref):
ino = self.inode(ref)
return ino.get("target", b"").decode("utf-8", "surrogateescape")

@property
def note(self):
"""Why this volume cannot be read on this Python, or None. listdir()
answers an empty list when its directory table cannot be decompressed,
so volumes() carries this beside the volume to say why."""
if self.comp == 6 and _zstd_module() is None:
return SQUASHFS_NO_ZSTD_NOTE
return None

def listdir(self, ref):
try:
ino = self.inode(ref)
Expand Down Expand Up @@ -6201,8 +6217,7 @@ def identify_squashfs(fh, base, size=None):
lines.append(f"note the image records {human(w.bytes_used)} but the region "
f"holds {human(size)}, so its end is missing")
if w.comp == 6 and _zstd_module() is None:
lines.append("note zstd compressed; this Python has no zstd (3.14 adds "
"it), so the listing may be short and no file can be read")
lines.append("note " + SQUASHFS_NO_ZSTD_NOTE)
if not root_ok:
if w.comp == 6 and _zstd_module() is None:
return "squashfs", lines
Expand Down Expand Up @@ -9343,6 +9358,12 @@ def volumes(fh, size=None):
vol["note"] = f"contents not read: {exc}"
except Exception as exc:
vol["note"] = f"could not walk this filesystem: {exc}"
# A walker that is built but cannot read its volume here says why, for
# a zstd SquashFS on a Python without zstd, whose listing comes back
# empty rather than raising.
wnote = getattr(vol.get("walker"), "note", None)
if wnote and "note" not in vol:
vol["note"] = wnote
out.append(vol)
return out

Expand Down Expand Up @@ -13039,6 +13060,35 @@ def _vol_view(path):
"--oob while mounted, against the kernel's own read-back"),
("ubi-nand-history", "ubi", "ubi-nand.history.kernel.sha256", None, "stat", "",
(), "a static UBI volume the kernel wrote with ubiupdatevol, from the same image")]
# A zstd SquashFS on a Python with no zstd lists nothing, since its
# directory tables are compressed too, and the walker answers an empty
# listing rather than raising. volumes() has to say why beside the
# volume, or it reads as an empty filesystem. The missing module is
# simulated, so this runs on every Python.
zst_img = os.path.join(fx, "squashfs-zstd.img.gz")
if not os.path.isfile(zst_img):
print(" [SKIP] squashfs-zstd is not beside this script, so the note a "
"Python without zstd gives was not checked")
else:
with gzip.open(zst_img, "rb") as gz:
zst_bytes = gz.read()
zst_globals = globals()
zst_real = zst_globals["_zstd_module"]
try:
zst_globals["_zstd_module"] = lambda: None
zst_hidden = volumes(io.BytesIO(zst_bytes), len(zst_bytes))
finally:
zst_globals["_zstd_module"] = zst_real
zst_shown = volumes(io.BytesIO(zst_bytes), len(zst_bytes))
zst_cond = (len(zst_hidden) == 1 and zst_hidden[0]["kind"] == "squashfs"
and zst_hidden[0].get("note") == SQUASHFS_NO_ZSTD_NOTE
and len(zst_shown) == 1
and (zst_shown[0].get("note") is None) == (zst_real() is not None))
if not zst_cond:
ok = False
print(f" [{'PASS' if zst_cond else 'FAIL'}] a zstd SquashFS on a Python "
f"without zstd carries a note saying why it lists nothing, and none "
f"where zstd is present")
for stem, want_kind, hashes, listing, style, prefix, loose, label in sq + jf + ub + ya + kh:
img = os.path.join(fx, stem + ".img.gz")
if not (os.path.isfile(img) and os.path.isfile(os.path.join(fx, hashes))):
Expand Down
8 changes: 4 additions & 4 deletions scripts/vendor/vendored.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
{
"path": "scripts/vendor/qnxprobe.py",
"name": "qnxprobe",
"version": "1.33",
"version": "1.34",
"upstream": "https://github.com/abrignoni/qnxprobe",
"upstream_file": "qnxprobe.py",
"commit": "e9bfb69e2d1556df67d5f1f8301dcf6f9a351749",
"commit_date": "2026-09-25T15:29:51-04:00",
"sha256": "15924af48c61fffb3fbbb1a7a8547f102341263d27970c58f71fba98df7bcdc8",
"commit": "fc74ca829c2773879243b326f2262c29f462df3a",
"commit_date": "2026-09-25T16:01:32-04:00",
"sha256": "ab0f681313bc305f3662d4421199f45da357da3becf90b28696a158fbaae2710",
"licence": "MIT",
"licence_file": "scripts/vendor/LICENSE-qnxprobe",
"note": "Copied verbatim. Fix upstream and re-vendor; edits here are reverted by the next sync."
Expand Down
Loading