Make NTFS alternate data streams members of the raw image seeker, and vendor qnxprobe 1.37

admin/test/data/raw_images/README.md

@@ -10,14 +10,18 @@ reader other than the one under test.
 | `fat32-deleted.img.gz` | FAT32, 64 MiB, two live files (`a.bin`, `c.bin`) and deleted ones | 2 | `shasum -a 256` over the image mounted read-only by macOS's own FAT driver, 2026-09-12 |
 | `exfat-deleted.img.gz` | exFAT, 64 MiB, two live files and deleted ones | 2 | `shasum -a 256` over the image mounted read-only by macOS's own exFAT driver, 2026-09-12 |
 | `apfs-fixture.img.gz` | APFS, 32 MiB, a container holding one volume of 411 files, one of them decmpfs-compressed, and a symbolic link | 411 | `shasum -a 256` over the files as macOS's own APFS driver wrote them, and the build fails unless The Sleuth Kit's reading of the finished image agrees (qnxprobe `tools/make_apfs_fixture.sh`) |
+| `ntfs-streams.img.gz` | NTFS, 8 MiB, 28 alternate data streams, among them two `Zone.Identifier` streams and a `$J` whose front is a 1 MiB hole and whose run list continues in a second MFT record, beside a stream sized and never written, which is not listed | 28 streams | The Sleuth Kit's `icat` from each stream's first stored cluster, as counted by `istat`, at build time, and the build fails unless `ntfs-3g` reads back every stream as written (qnxprobe `tools/make_ntfs_streams_fixture.sh`) |
 
 The images are copies of the fixtures committed in
 [abrignoni/qnxprobe](https://github.com/abrignoni/qnxprobe) under `tests/fixtures/`,
-taken at commit `34cc4607799f58ab9157fc4be1d89098f687699d`; that repository's
+taken at commit `34cc4607799f58ab9157fc4be1d89098f687699d` (`ntfs-streams` at
+`75164a971a80b54acaf23306228e2f047ce50a1d`); that repository's
 `tools/` scripts say how each was built. They are stored gzipped and the tests
 decompress them into a temporary folder. The FAT and exFAT images were written on
 a Mac and also hold `._` AppleDouble companions, which the macOS driver hides and
 the reader lists; the hash lists cover the files a person put there.
 
 The `.live.sha256` lists are "<sha256>  <path>" lines, paths relative to the
-volume root, the same shape `sha256sum` writes.
+volume root, the same shape `sha256sum` writes. `ntfs-streams.sha256` has the same
+shape with a stream's path written `<file>:<stream>` (`:<stream>` on the root), after
+a few `#` lines saying which two streams it leaves out because they store nothing.

admin/test/data/raw_images/ntfs-streams.sha256

@@ -0,0 +1,33 @@
+# Every $DATA stream The Sleuth Kit's fls lists on ntfs-streams.img that holds
+# a stored cluster: the sha256 icat reads from its first stored cluster on,
+# counted from istat (ntfsinfo where istat lists no clusters). Written by
+# tools/make_ntfs_streams_fixture.sh.
+# Not listed, because every cluster is sparse: $BadClus:$Bad, hollow.bin:nothing-stored
+95aefacfebf228fd2c9e150a86b0eb1a3924fb25b0995c6e0e7c34feeade0a76  $Secure:$SDS
+ee502838f53f00c9444b311f4cdea74454a1e0c64e8cdec3d63eb5232fb61f82  $UpCase:$Info
+e2589a1859f1a1a7cc52277bb7ab951bac3fd6a937d6106fa36a85b79f30dd3b  :rootstream
+c5d306b372ab076e19db688631453a46dbbc1b40ae7fc54a914d5c534b9d9fd0  big.bin:payload
+f0ae89733282e35f118f3cd16685783db9d0bb4909e251bf880f7bbe4235b430  comp/packed.txt:late
+46aadf4573888fbd6af521be8b6fdc2e6fba5e6ed28998a63209cb7a15860ed4  comp/packed.txt:packed
+3bd5d8d616c3ced007997a4f8ffd719ca07aa3493b291e35a96ac408fee5ba27  downloads/report.pdf:Zone.Identifier
+ec55e4ebef087c3a5c2e7cce1cc1d388d859f2f185f4785b8e4a042c32a9bccf  downloads/setup.exe:Zone.Identifier
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  empty.txt:empty
+d1870c37999c4b9e06a747c1231511d778c4954728089f6872cf3fa0be2cd608  folder:dirstream
+742cbf9708111d6c8e8bdad6521555dbc6e2704e2a36c36bd66ecb29eceeb4e4  holey.bin:holey
+a96c211b576e0b1a1cf70a568e5b41f22bade0eac27c97c0d9171ba280e9467d  journal.bin:$J
+c1753a66a529c1cefc879e217ba925b1f4b6f7d900b3a6d1020e8197b5f761ad  journal.bin:$Max
+899b72ee090380a098705f0fcc56b0643b00abba40b51b52485f4df9ef59ee0d  linked_a.txt:tag
+899b72ee090380a098705f0fcc56b0643b00abba40b51b52485f4df9ef59ee0d  linked_b.txt:tag
+86c7099c50955e8efad745d232d85d2f33a3ddc844e41692f4a39406cdc20bf8  manystreams.txt:s1
+45da5e387c3e6af714fe7fbb32af5167a4eaaed528b501170eec89d9a7e623da  manystreams.txt:s10
+e2ceec5bd7422af598064c20bffdf7a3c8a45143590fe84daa0b8d7d6cf70c13  manystreams.txt:s11
+c03d2b93d41573f97cd58036de85b6064eeb0e35909a734730f6232bea76fe94  manystreams.txt:s12
+4e4d1ee6eb3374846974cdab92cc9fcf16053eedde7387965b2be2046d767e6c  manystreams.txt:s2
+a32b8ed2d70c41a66e3e00ceecb1a05bcdd4238157f1db2c7ffe58dc7733ebc7  manystreams.txt:s3
+bf003ab793953c9670c71153a40be1f560eee6084ade96c2fb43ebc525d87ea5  manystreams.txt:s4
+22f37107ad3afb19f65df41d65f1cece0f47294c7ef3d51570992dc9253d5be5  manystreams.txt:s5
+d66be6793b9b55c6ef4d1b7b3769ca20cac399b58682c4876b760d59fc4b4a15  manystreams.txt:s6
+8e3b48ac9d082b4278df1a7a5e6dde1caa3814001c6296ddc447eaf0595b9ee0  manystreams.txt:s7
+25170f987351a8a51b19e2907aebddc34d1d9b7c4598db2b848647b965698b9a  manystreams.txt:s8
+7c702e9ca0f18cecc54364ced20c165ad8a0fe784a173a6428701f8cd2f632a6  manystreams.txt:s9
+3e8feb372b5cb74b3774c54a20022672f00051ae2a2d32e7895633d197f3056f  notes.txt:ünïcödé

admin/test/scripts/test_raw_image_seeker.py

@@ -32,7 +32,8 @@
 
 import scripts.raw_image as raw_image  # pylint: disable=wrong-import-position
 from scripts.raw_image import (  # pylint: disable=wrong-import-position
-    RAW_IMAGE_FILESYSTEMS, RAW_IMAGE_SUFFIXES, FileSeekerRaw, split_image_sibling)
+    RAW_IMAGE_FILESYSTEMS, RAW_IMAGE_SUFFIXES, FileSeekerRaw, names_a_stream,
+    split_image_sibling)
 from scripts.vendor import ewfprobe, qnxprobe  # pylint: disable=wrong-import-position
 
 FIXTURES = REPO_ROOT / 'admin' / 'test' / 'data' / 'raw_images'
@@ -41,6 +42,8 @@
 def _hash_list(path):
     out = {}
     for line in path.read_text(encoding='utf-8').splitlines():
+        if line.startswith('#'):
+            continue
         digest, _, rel = line.partition('  ')
         if rel:
             out[rel.strip()] = digest.strip()
@@ -153,18 +156,21 @@ class RawImageSeekerTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.work = tempfile.mkdtemp(prefix='raw_image_test_')
-        for stem in ('ntfs-fixture', 'fat32-deleted', 'exfat-deleted', 'apfs-fixture'):
+        for stem in ('ntfs-fixture', 'fat32-deleted', 'exfat-deleted', 'apfs-fixture',
+                     'ntfs-streams'):
             with gzip.open(FIXTURES / f'{stem}.img.gz', 'rb') as src, \
                     open(os.path.join(cls.work, f'{stem}.img'), 'wb') as dst:
                 shutil.copyfileobj(src, dst)
         cls.ntfs = os.path.join(cls.work, 'ntfs-fixture.img')
         cls.fat32 = os.path.join(cls.work, 'fat32-deleted.img')
         cls.exfat = os.path.join(cls.work, 'exfat-deleted.img')
         cls.apfs = os.path.join(cls.work, 'apfs-fixture.img')
+        cls.streams = os.path.join(cls.work, 'ntfs-streams.img')
         cls.ntfs_hashes = _hash_list(FIXTURES / 'ntfs-fixture.sha256')
         cls.fat32_hashes = _hash_list(FIXTURES / 'fat32-deleted.live.sha256')
         cls.exfat_hashes = _hash_list(FIXTURES / 'exfat-deleted.live.sha256')
         cls.apfs_hashes = _hash_list(FIXTURES / 'apfs-fixture.sha256')
+        cls.stream_hashes = _hash_list(FIXTURES / 'ntfs-streams.sha256')
 
     @classmethod
     def tearDownClass(cls):
@@ -270,7 +276,8 @@ def test_a_pattern_is_matched_once_and_cached(self):
     # with the faster route taken away.
 
     def _members(self, image, **patches):
-        """(member list, {member: (node, size, mtime, reading)}) from a fresh seeker."""
+        """(member list, {member: (node, size, mtime, reading)}, stream list) from
+        a fresh seeker."""
         self.data = tempfile.mkdtemp(prefix='raw_image_data_')
         self.addCleanup(shutil.rmtree, self.data, True)
         with contextlib.ExitStack() as stack:
@@ -282,7 +289,7 @@ def _members(self, image, **patches):
         entries = {member: None if entry is None else
                    (repr(entry.node), entry.size, entry.mtime, entry.reading)
                    for member, entry in seeker._entries.items()}  # pylint: disable=protected-access
-        return list(seeker.name_list), entries
+        return list(seeker.name_list), entries, list(seeker.stream_list)
 
     def test_ntfs_is_listed_in_one_pass_and_matches_the_tree_walk(self):
         fast = self._members(self.ntfs)
@@ -341,11 +348,134 @@ def doubled(walker):
                 if row[0] == 'many/file_0001.txt':
                     yield ('many/file_0001.txt', row[1] + 100000) + row[2:]
 
-        names, entries = self._members(self.ntfs, ntfs={'listing': doubled})
+        names, entries, _streams = self._members(self.ntfs, ntfs={'listing': doubled})
         self.assertEqual(names.count('lba0/many/file_0001.txt'), 1)
         self.assertNotEqual(entries['lba0/many/file_0001.txt'][0], repr(100000))
         self.assertIn('claimed by more than one record', self.log.text())
 
+    # ---- NTFS alternate data streams --------------------------------------------
+    #
+    # ntfs-streams.img holds streams in each shape the reader has a rule for, and its
+    # hash list is The Sleuth Kit's reading of each stream from its first stored
+    # cluster, which is what the reader returns (see the README beside it). A
+    # stream is a member only a pattern naming a stream can reach, so every
+    # pattern written before streams were members is handed exactly what it was.
+
+    def _staged_streams(self, seeker, pattern):
+        """{path:stream within the volume: staged path} for a stream pattern."""
+        found = seeker.search(pattern)
+        out = {}
+        for staged in found:
+            source = seeker.file_infos[staged].source_path
+            self.assertTrue(source.startswith('lba0/'), source)
+            out[source[len('lba0/'):]] = staged
+        return out
+
+    def test_every_ntfs_stream_matches_the_independent_hash_list(self):
+        seeker = self._seeker(self.streams)
+        self.assertEqual(len(self.stream_hashes), 28)
+        staged = self._staged_streams(seeker, '*:*')
+        self.assertEqual(sorted(staged), sorted(self.stream_hashes))
+        differ = [rel for rel, digest in self.stream_hashes.items()
+                  if _sha256(staged[rel]) != digest]
+        self.assertEqual(differ, [])
+        self.assertEqual(len(seeker.stream_list), 28)
+
+    def test_a_pattern_that_names_no_stream_is_never_handed_one(self):
+        seeker = self._seeker(self.streams)
+        everything = seeker.search('*')
+        self.assertTrue(everything)
+        sources = [seeker.file_infos[path].source_path for path in everything]
+        self.assertEqual([s for s in sources if ':' in s], [])
+        downloads = seeker.search('*/downloads/*')
+        self.assertEqual(sorted(seeker.file_infos[p].source_path for p in downloads),
+                         ['lba0/downloads/', 'lba0/downloads/report.pdf',
+                          'lba0/downloads/setup.exe'])
+        self.assertTrue(set(seeker.name_list).isdisjoint(seeker.stream_list))
+
+    def test_zone_identifier_streams_are_staged_under_a_name_without_the_colon(self):
+        seeker = self._seeker(self.streams)
+        staged = self._staged_streams(seeker, '*:Zone.Identifier')
+        self.assertEqual(sorted(staged), ['downloads/report.pdf:Zone.Identifier',
+                                          'downloads/setup.exe:Zone.Identifier'])
+        for rel, path in staged.items():
+            self.assertEqual(_sha256(path), self.stream_hashes[rel])
+            self.assertNotIn(':', os.path.basename(path))
+        with open(staged['downloads/setup.exe:Zone.Identifier'], 'rb') as handle:
+            self.assertTrue(handle.read().startswith(b'[ZoneTransfer]\r\nZoneId=3\r\n'))
+
+    def test_a_stream_carries_its_file_s_dates(self):
+        seeker = self._seeker(self.streams)
+        stream = seeker.search('*/downloads/setup.exe:Zone.Identifier')[0]
+        host = seeker.search('*/downloads/setup.exe')[0]
+        self.assertGreater(seeker.file_infos[stream].modification_date, 1_600_000_000)
+        self.assertEqual(seeker.file_infos[stream].creation_date,
+                         seeker.file_infos[host].creation_date)
+        self.assertEqual(seeker.file_infos[stream].modification_date,
+                         seeker.file_infos[host].modification_date)
+
+    def test_the_journal_is_staged_from_its_first_stored_cluster(self):
+        # journal.bin:$J is the shape of $Extend/$UsnJrnl:$J: 1 MiB of hole, then
+        # records in 365 runs whose run list continues in a second MFT record.
+        # Its recorded size is 2,859,008 bytes; what is stored is 1,810,432.
+        seeker = self._seeker(self.streams)
+        staged = self._staged_streams(seeker, '*/journal.bin:$J')
+        self.assertEqual(list(staged), ['journal.bin:$J'])
+        path = staged['journal.bin:$J']
+        self.assertEqual(os.path.getsize(path), 1_810_432)
+        self.assertEqual(_sha256(path), self.stream_hashes['journal.bin:$J'])
+
+    def test_a_stream_that_stores_nothing_is_not_a_member(self):
+        seeker = self._seeker(self.streams)
+        self.assertNotIn('lba0/$BadClus:$Bad', seeker.stream_list)
+        self.assertNotIn('lba0/hollow.bin:nothing-stored', seeker.stream_list)
+        self.assertEqual(seeker.search('*:$Bad'), [])
+        self.assertIn('lba0/empty.txt:empty', seeker.stream_list)
+        self.assertEqual(os.path.getsize(seeker.search('*/empty.txt:empty')[0]), 0)
+
+    def test_streams_on_directories_and_the_root_are_members(self):
+        seeker = self._seeker(self.streams)
+        self.assertIn('lba0/folder:dirstream', seeker.stream_list)
+        self.assertIn('lba0/:rootstream', seeker.stream_list)
+        root = seeker.search('*/:rootstream')
+        self.assertEqual(len(root), 1)
+        self.assertEqual(_sha256(root[0]), self.stream_hashes[':rootstream'])
+
+    def test_streams_are_the_same_by_both_routes(self):
+        fast = self._members(self.streams)
+        slow = self._members(self.streams, ntfs={'listing': None})
+        self.assertEqual(len(fast[2]), 28)
+        self.assertEqual(fast, slow)
+
+    def test_the_run_log_counts_the_streams(self):
+        self._seeker(self.streams)
+        self.assertIn('and 28 alternate data streams, matched only by a pattern', self.log.text())
+        self.assertIn('members and 28 streams', self.log.text())
+
+    def test_a_reader_without_streams_leaves_the_members_as_they_were(self):
+        # a vendored qnxprobe older than 1.37 has no streams(); the seeker then
+        # lists exactly the members it did, and no stream
+        with_streams = self._members(self.streams)
+        without = self._members(self.streams, ntfs={'streams': None})
+        self.assertEqual(without[2], [])
+        self.assertEqual(without[0], with_streams[0])
+        self.assertNotIn('alternate data streams', self.log.text().split('Reading ')[-1])
+
+    def test_the_stream_on_the_file_fixture(self):
+        # the fixture every other NTFS test reads carries one stream of its own
+        seeker = self._seeker(self.ntfs)
+        found = seeker.search('*/ads.txt:hidden')
+        self.assertEqual(len(found), 1)
+        with open(found[0], 'rb') as handle:
+            self.assertEqual(handle.read(), b'the hidden stream\n')
+
+    def test_what_names_a_stream(self):
+        for pattern in ('*:Zone.Identifier', '*/$Extend/$UsnJrnl:$J', '*/:rootstream', '*:*',
+                        '*\\x.exe:Zone.Identifier'):
+            self.assertTrue(names_a_stream(pattern), pattern)
+        for pattern in ('*', '*/Recent/*', '*/c:/Users/*/NTUSER.DAT', '*/Windows/Prefetch/*.pf'):
+            self.assertFalse(names_a_stream(pattern), pattern)
+
     # ---- members and metadata ---------------------------------------------------
 
     def test_directories_are_members_with_a_trailing_slash(self):
@@ -508,7 +638,7 @@ def entry(self, node):
 
         seeker = FileSeekerRaw.__new__(FileSeekerRaw)
         seeker.name_list, seeker._entries = [], {}  # pylint: disable=protected-access
-        files, dirs, _route = seeker._walk(_Walker(), 'v')  # pylint: disable=protected-access
+        files, dirs, _streams, _route = seeker._walk(_Walker(), 'v')  # pylint: disable=protected-access
         self.assertEqual(sorted(seeker.name_list), ['v/d/', 'v/d/inner', 'v/f'])
         self.assertEqual((files, dirs), (2, 1))