diff --git a/README.md b/README.md index db8508375..6f70c4067 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ python ileapp.py -t raw -i /path/to/acquisition.E01 -o /path/to/output/ `raw` reads a disk image (`.img`, `.dd`, `.bin`, or any numbered `.001` segment of a split set), or an EnCase/EWF `.E01` acquisition and the segments beside it, in place: no mounting and no administrator rights. Its NTFS, FAT32, exFAT, ext2/3/4, -HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS volumes are searched directly, and +F2FS, HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS volumes are searched directly, and only the files an artifact asks for are read out of the image. The GUI picks `raw` on its own for those extensions. See `admin/docs/raw_image_input.md`. diff --git a/admin/docs/raw_image_input.md b/admin/docs/raw_image_input.md index 3f97c42bc..748c592b8 100644 --- a/admin/docs/raw_image_input.md +++ b/admin/docs/raw_image_input.md @@ -6,7 +6,7 @@ For maintainers, and shared by iLEAPP, ALEAPP, RLEAPP, VLEAPP and DLEAPP. segment of a split set) or an EnCase/EWF acquisition (`.E01` and its segments) and reads it in place: no mounting, no administrator rights, and no copy of the image or of its files anywhere but the files an artifact asks for. Its NTFS, -FAT32, exFAT, ext2/3/4, HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS volumes +FAT32, exFAT, ext2/3/4, F2FS, HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS volumes are searched directly. ## Where the pieces are @@ -76,8 +76,10 @@ for it, so no report field carries a zone the evidence never had. collapsed and per-app row counts multiply; on a bare iOS Data volume the `iosfilesystemevents` family and `diagnosticlogdevents` find nothing. A full-disk or full-`/data` image carries the expected root and neither happens. -- F2FS, the filesystem most current Android userdata partitions use, is not - one the reader walks; such a volume is listed as not recognised. +- On F2FS, a file the filesystem compressed is listed with its size and not + decompressed, and one under per-file encryption (the norm on a current + Android `userdata`) is listed and its content refused rather than staged as + ciphertext. - An encrypted volume (Android file-based encryption, iOS data protection, FileVault, BitLocker) reads, but its names or contents are ciphertext. diff --git a/admin/test/scripts/test_raw_image_seeker.py b/admin/test/scripts/test_raw_image_seeker.py index 86085971b..6983dd5c6 100644 --- a/admin/test/scripts/test_raw_image_seeker.py +++ b/admin/test/scripts/test_raw_image_seeker.py @@ -388,7 +388,8 @@ def test_the_reader_finds_its_ewf_module_when_imported_as_a_package(self): def test_the_filesystem_list_names_only_kinds_the_reader_walks(self): walkers = {'QNX6': qnxprobe.Qnx6Walker, 'QNX4': qnxprobe.Qnx4Walker, 'ETFS': qnxprobe.EtfsWalker, 'EFS': qnxprobe.EfsWalker, - 'ext2/3/4': qnxprobe.ExtWalker, 'FAT32': qnxprobe.Fat32Walker, + 'ext2/3/4': qnxprobe.ExtWalker, 'F2FS': qnxprobe.F2fsWalker, + 'FAT32': qnxprobe.Fat32Walker, 'exFAT': qnxprobe.ExfatWalker, 'NTFS': qnxprobe.NtfsWalker, 'HFS+': qnxprobe.HfsPlusWalker, 'APFS': qnxprobe.ApfsWalker, 'QNX IFS': qnxprobe.IfsWalker} diff --git a/ileapp.py b/ileapp.py index a1a9a3864..eee8a796a 100755 --- a/ileapp.py +++ b/ileapp.py @@ -189,7 +189,7 @@ def main(): "'itunes' for a folder containing a raw iTunes backup with hashed paths and names, " "'raw' for a disk image (.img, .dd, .bin, or any numbered .001 segment of a split " "set) or an EnCase/EWF .E01 acquisition, read in place without mounting: its " - "NTFS, FAT32, exFAT, ext2/3/4, HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS " + "NTFS, FAT32, exFAT, ext2/3/4, F2FS, HFS+, APFS, QNX6, QNX4, ETFS, EFS and QNX IFS " "volumes are searched directly, " "'file' for a single file input.")) parser.add_argument('-o', '--output_path', required=False, action="store", diff --git a/scripts/raw_image.py b/scripts/raw_image.py index e4713f011..a4cb6d526 100644 --- a/scripts/raw_image.py +++ b/scripts/raw_image.py @@ -56,7 +56,7 @@ # What the vendored reader walks, for the file dialog and the -t help. A test # asserts each entry here has a walker in the vendored copy, so the two cannot # drift apart quietly. -RAW_IMAGE_FILESYSTEMS = ('QNX6, QNX4, ETFS, EFS, ext2/3/4, FAT32, exFAT, NTFS, ' +RAW_IMAGE_FILESYSTEMS = ('QNX6, QNX4, ETFS, EFS, ext2/3/4, F2FS, FAT32, exFAT, NTFS, ' 'HFS+, APFS, QNX IFS') RAW_IMAGE_LABEL = f'Raw disk image or acquisition ({RAW_IMAGE_FILESYSTEMS})' diff --git a/scripts/vendor/qnxprobe.py b/scripts/vendor/qnxprobe.py index 5d89a9fdd..0123fb827 100644 --- a/scripts/vendor/qnxprobe.py +++ b/scripts/vendor/qnxprobe.py @@ -42,7 +42,7 @@ except ImportError: ewfprobe = None -QNXPROBE_VERSION = "1.27" +QNXPROBE_VERSION = "1.28" QNX6_MAGIC = 0x68191122 BOOTBLOCK_SIZE = 0x2000 @@ -981,6 +981,400 @@ def _zeros(n, cap): root = 2 +# --------------------------------------------------------------------------- +# F2FS (Flash-Friendly File System). +# +# F2FS is the filesystem Android uses for /data on most phones, so an Android +# image or a userdata partition can carry one where an older device would have +# ext4. It is read here directly, no mounting. +# +# The layout is sourced from the Linux kernel's own F2FS at v7.0 (commit +# 028ef9c96e96197026887c0f092424679298aae8): the on-disk structures in +# include/linux/f2fs_fs.h (f2fs_super_block, f2fs_checkpoint, f2fs_inode, +# node_footer, f2fs_dir_entry, f2fs_dentry_block, f2fs_nat_entry), node +# resolution in fs/f2fs/node.{c,h} (current_nat_addr, get_node_path), +# checkpoint selection in fs/f2fs/checkpoint.c (validate_checkpoint) and +# fs/f2fs/super.c (sanity_check_raw_super), the NAT journal in +# fs/f2fs/segment.c (read_normal_summaries), the inode read in +# fs/f2fs/inode.c (do_read_inode) and the directory walk in fs/f2fs/dir.c +# (f2fs_fill_dentries). No F2FS implementation's code was copied; qnxprobe is +# MIT and both the kernel driver and f2fs-tools are GPL. f2fs-tools' dump.f2fs +# is used only as an independent oracle to validate this reader. +# +# To find a file this reader has to resolve a node id (nid) to the block that +# holds its node. That mapping lives in the NAT (Node Address Table), and the +# current copy of each NAT block is chosen by a bitmap in the active +# checkpoint, with recent updates overriding it from a journal in the +# checkpoint's data summary. Once the inode block is read, its data is +# addressed the way ext is: pointers in the inode, then direct, indirect and +# double-indirect node blocks. Small files and directories keep their content +# inline in the inode instead. +# --------------------------------------------------------------------------- +F2FS_MAGIC = 0xF2F52010 +F2FS_SUPER_OFF = 1024 # the superblock sits 1024 bytes into block 0 +F2FS_BLKSIZE_BITS_MIN = 12 # only block == page size is valid: 4K or 16K +F2FS_BLKSIZE_BITS_MAX = 16 +# i_inline flags (f2fs_fs.h) +F2FS_INLINE_XATTR, F2FS_INLINE_DATA = 0x01, 0x02 +F2FS_INLINE_DENTRY, F2FS_EXTRA_ATTR = 0x04, 0x20 +F2FS_COMPR_FL = 0x00000004 # i_flags: file is compressed +F2FS_FADVISE_ENCRYPT = 0x04 # i_advise: file is encrypted +# superblock feature flags (f2fs_fs.h F2FS_FEATURE_*) +F2FS_FEAT_ENCRYPT = 0x0001 +F2FS_FEAT_EXTRA_ATTR = 0x0008 +F2FS_FEAT_FLEX_INLINE_XATTR = 0x0040 +F2FS_FEAT_INODE_CRTIME = 0x0100 +F2FS_FEAT_COMPRESSION = 0x2000 +F2FS_FEAT_CASEFOLD = 0x1000 +F2FS_FEAT_PACKED_SSA = 0x10000 +# checkpoint flags (f2fs_fs.h CP_*) +F2FS_CP_UMOUNT = 0x0001 +F2FS_CP_COMPACT_SUM = 0x0004 +F2FS_CP_FASTBOOT = 0x0020 +F2FS_CP_ERROR = 0x0008 +F2FS_CP_LARGE_NAT_BITMAP = 0x0400 +# reserved data-block-address markers (f2fs_fs.h) +F2FS_NULL_ADDR, F2FS_NEW_ADDR, F2FS_COMPRESS_ADDR = 0, 0xFFFFFFFF, 0xFFFFFFFE +F2FS_DEFAULT_INLINE_XATTR_ADDRS = 50 +F2FS_NAT_ENTRY_SIZE = 9 # f2fs_nat_entry: u8 version, le32 ino, le32 block_addr +F2FS_ENC_NAME_MARKERS = () # names are not decoded; encryption is reported per file + + +def f2fs_time(v): + """An F2FS timestamp (le64 Unix seconds, UTC) as a Unix second count, or 0.""" + return v if 0 < v < (1 << 40) else 0 + + +class F2fsUnreadable(Exception): + """A file this reader will not hand back whole (encrypted or compressed); + the message says which.""" + + +class F2fsWalker: + """List and read files from an F2FS volume. Nodes are addressed by nid, so + listdir()/entry()/read_file() take a nid and the shared collect() and + extract_to_zip() work unchanged; the root is the superblock's root_ino.""" + + def __init__(self, fh, base): + self.fh, self.base = fh, base + sb = self._sb(base) + if sb is None: + raise F2fsUnreadable("no valid F2FS superblock") + self.log_bs = struct.unpack_from("= 200 + and struct.unpack_from(" best[0]: + best = (ver, pack, head, total) + if best is None: + raise F2fsUnreadable("no valid F2FS checkpoint") + self.cp_ver, self.cur_pack, self.ckpt, self.cp_total = best + self.start_cp = self.cp_blkaddr + (self.seg_blocks if self.cur_pack == 2 else 0) + self.cp_flags = struct.unpack_from(" 0: + self.nat_bitmap = self.ckpt[192:] + else: + self.nat_bitmap = self.ckpt[192 + sit_bm:] + self._load_nat_journal() + + def _load_nat_journal(self): + """Recent NAT updates not yet flushed to the on-disk table live in a + journal inside the checkpoint's hot-data summary (fs/f2fs/segment.c + read_normal_summaries / read_compacted_summaries). They override the + on-disk NAT, so a nid found here wins.""" + self.nat_j = {} + sum_bs = 4096 if (self.feature & F2FS_FEAT_PACKED_SSA) else self.bs + sum_entry_size = 7 * (sum_bs // 8) # entries_in_sum summaries of 7 bytes + if self.cp_flags & F2FS_CP_COMPACT_SUM: + blk = self.start_cp + struct.unpack_from(" len(buf): + return + n_nats = struct.unpack_from(" len(buf): + break + nid = struct.unpack_from("> 3 + return (bm[j] >> (i & 7)) & 1 if j < len(bm) else 0 + + def resolve(self, nid): + """The block address holding node nid, or 0 if it is unallocated.""" + if nid in self.nat_j: + return self.nat_j[nid] + start = (nid // self.nepb) * self.nepb + block_off = start // self.nepb + addr = self.nat_blkaddr + (block_off << 1) - (block_off & (self.seg_blocks - 1)) + if self._test_bit(self.nat_bitmap, block_off): + addr += self.seg_blocks + blk = self.block(addr) + eo = (nid - start) * F2FS_NAT_ENTRY_SIZE + if eo + F2FS_NAT_ENTRY_SIZE > len(blk): + return 0 + return struct.unpack_from("= self.bs else None + + def inode(self, nid): + """The parsed inode for nid, or None. A node whose footer nid equals its + footer ino is an inode (f2fs.h RAW_IS_INODE).""" + b = self._node(nid) + if b is None: + return None + if struct.unpack_from(" len(b): + return 0 + return struct.unpack_from(" 255: + bit += 1 + continue + slots = (name_len + 7) // 8 + ns = off + foff + bit * 8 + name = buf[ns:ns + name_len].decode("utf-8", "replace") + bit += slots + if name not in (".", ".."): + out.append((name, child)) + return out + + def listdir(self, nid): + ino = self.inode(nid) + if ino is None or not (ino["mode"] & S_IFDIR): + return [] + if ino["inline"] & F2FS_INLINE_DENTRY: + off = ino["iaddr_off"] + 4 # DEF_INLINE_RESERVED_SIZE + span = 4 * (ino["addrs_in_inode"] - 1) # MAX_INLINE_DATA + return sorted(self._parse_dentry(ino["raw"], off, span)) + out = [] + for L in range((ino["size"] + self.bs - 1) // self.bs): + addr = self._block_of(ino, L) + if addr in (F2FS_NULL_ADDR, F2FS_NEW_ADDR, F2FS_COMPRESS_ADDR): + continue + buf = self.block(addr) + if len(buf) >= self.bs: + out.extend(self._parse_dentry(buf, 0, self.bs)) + return sorted(out) + + def read_file(self, nid, size): + """Yield exactly `size` bytes, holes emitted as zeros so offsets stay + correct. Inline data lives in the inode; encrypted or compressed content + is refused rather than guessed at.""" + ino = self.inode(nid) + if ino is None: + return + if self._encrypted(ino): + raise F2fsUnreadable("the file is encrypted and the volume holds no key") + if self._compressed(ino): + raise F2fsUnreadable("the file is compressed (F2FS compression is not read here)") + if ino["inline"] & F2FS_INLINE_DATA: + off = ino["iaddr_off"] + 4 # skip DEF_INLINE_RESERVED_SIZE + yield ino["raw"][off:off + size] + return + left, L = size, 0 + while left > 0: + addr = self._block_of(ino, L) + if addr in (F2FS_NULL_ADDR, F2FS_NEW_ADDR, F2FS_COMPRESS_ADDR): + buf = bytes(self.bs) + else: + buf = self.block(addr) + if len(buf) < self.bs: + buf = buf + bytes(self.bs - len(buf)) + take = min(self.bs, left) + yield buf[:take] + left -= take + L += 1 + + # --------------------------------------------------------------------------- # FAT32 and exFAT. # @@ -5287,6 +5681,8 @@ def walker_for(kind, fh, base, size=None): return ExfatWalker(fh, base) if kind == "ntfs": return NtfsWalker(fh, base) + if kind == "f2fs": + return F2fsWalker(fh, base) if kind in ("hfs+", "hfsx"): return HfsPlusWalker(fh, base) if kind == "apfs": @@ -5529,6 +5925,66 @@ def identify_fat(fh, base): return None +def identify_f2fs(fh, base): + """Return ("f2fs", lines) for an F2FS volume at base, else None. + + F2FS names itself with a 4-byte magic 1024 bytes into the volume, and the + reserved inode numbers that follow (node 1, meta 2, root 3) are fixed, so + both are required rather than the magic alone (fs/f2fs/super.c + sanity_check_raw_super). Block size and blocks-per-segment must also be the + values the format allows. + """ + b = read_at(fh, base + F2FS_SUPER_OFF, 3072) + if len(b) < 200 or struct.unpack_from("