fix superblock struct; fix extended attribute parse

.github/workflows/build.yaml

@@ -29,6 +29,11 @@ jobs:
     steps:
       - name: Checkout the Git repository
         uses: actions/checkout@v4
+      - name: Install dependencies
+        shell: bash
+        run: |
+          set -e
+          sudo apt-get install -y attr
       - name: Generate test.ext4
         shell: bash
         run: |
@@ -38,8 +43,10 @@ jobs:
         with:
           name: test.ext4
           path: |
-            test.ext4
-            test.ext4.tmp
+            test32.ext4
+            test32.ext4.tmp
+            test64.ext4
+            test64.ext4.tmp
           if-no-files-found: error
   test:
     name: Test on ${{ matrix.os }} python ${{ matrix.python }}

_test_image.sh

@@ -6,8 +6,16 @@ trap "rm -r \"$tmp_dir\"" EXIT
 echo "hello world" > "$tmp_dir"/test.txt
 for i in {1..100};do
   echo "hello world" >> "$tmp_dir"/test.txt
+  echo "hello world$i" > "$tmp_dir"/test$i.txt
+  for j in {1..20};do
+    setfattr -n user.name$j -v value${i}_$j "$tmp_dir"/test$i.txt
+  done
 done
-dd if=/dev/zero of=test.ext4.tmp count=1024 bs=1024
-mkfs.ext4 test.ext4.tmp -d "$tmp_dir"
-echo -n F > test.ext4
-cat test.ext4.tmp >> test.ext4
+dd if=/dev/zero of=test32.ext4.tmp count=20 bs=1048576
+dd if=/dev/zero of=test64.ext4.tmp count=20 bs=1048576
+mkfs.ext4 -g 1024 -O 64bit test64.ext4.tmp -d "$tmp_dir"
+mkfs.ext4 -g 1024 -O ^64bit test32.ext4.tmp -d "$tmp_dir"
+echo -n F > test32.ext4
+cat test32.ext4.tmp >> test32.ext4
+echo -n F > test64.ext4
+cat test64.ext4.tmp >> test64.ext4

ext4/inode.py

@@ -33,6 +33,7 @@
 from .htree import DXRoot
 
 from .xattr import ExtendedAttributeIBodyHeader
+from .xattr import ExtendedAttributeHeader
 
 
 class OpenDirectoryError(Exception):
@@ -192,6 +193,10 @@ def __init__(self, volume, offset, i_no):
     def superblock(self):
         return self.volume.superblock
 
+    @property
+    def block_size(self):
+        return self.volume.block_size
+
     @property
     def i_size(self):
         return self.i_size_high << 32 | self.i_size_lo
@@ -300,7 +305,7 @@ def open(self, mode="rb", encoding=None, newline=None):
     @property
     def xattrs(self):
         inline_offset = self.offset + self.EXT2_GOOD_OLD_INODE_SIZE + self.i_extra_isize
-        inline_size = self.superblock.s_inode_size - inline_offset
+        inline_size = self.offset + self.superblock.s_inode_size - inline_offset
         if inline_size > sizeof(ExtendedAttributeIBodyHeader):
             try:
                 header = ExtendedAttributeIBodyHeader(self, inline_offset, inline_size)
@@ -314,9 +319,7 @@ def xattrs(self):
         if self.i_file_acl != 0:
             block_offset = self.i_file_acl * self.block_size
             try:
-                header = ExtendedAttributeIBodyHeader(
-                    self, block_offset, self.volume.block_size
-                )
+                header = ExtendedAttributeHeader(self, block_offset, self.block_size)
                 header.verify()
                 for name, value in header:
                     yield name, value
@@ -370,14 +373,6 @@ def validate(self):
         super().validate()
         # TODO validate each directory entry block with DirectoryEntryTail
 
-    @property
-    def superblock(self):
-        return self.volume.superblock
-
-    @property
-    def block_size(self):
-        return self.volume.block_size
-
     @property
     def has_filetype(self):
         return self.superblock.s_feature_incompat & EXT4_FEATURE_INCOMPAT.FILETYPE != 0

ext4/superblock.py

@@ -76,6 +76,7 @@ class Superblock(Ext4Struct):
         ("s_desc_size", c_uint16),
         ("s_default_mount_opts", EXT4_DEFM),
         ("s_first_meta_bg", c_uint32),
+        ("s_mkfs_time", c_uint32),
         ("s_jnl_blocks", c_uint32 * 17),
         ("s_blocks_count_hi", c_uint32),
         ("s_r_blocks_count_hi", c_uint32),
@@ -110,7 +111,7 @@ class Superblock(Ext4Struct):
         ("s_usr_quota_inum", c_uint32),
         ("s_grp_quota_inum", c_uint32),
         ("s_overhead_blocks", c_uint32),
-        ("s_backup_bgs", c_uint32),
+        ("s_backup_bgs", c_uint32 * 2),
         ("s_encrypt_algos", FS_ENCRYPTION_MODE * 4),
         ("s_encrypt_pw_salt", c_uint8 * 16),
         ("s_lpf_ino", c_uint32),
@@ -183,3 +184,10 @@ def seed(self):
             return self.s_checksum_seed
 
         return crc32c(bytes(self.s_uuid))
+
+    @property
+    def desc_size(self):
+        if self.s_feature_incompat & EXT4_FEATURE_INCOMPAT.IS64BIT != 0:
+            return self.s_desc_size
+
+        return 32

ext4/volume.py

@@ -61,6 +61,7 @@ def __init__(
         ignore_flags=False,
         ignore_magic=False,
         ignore_checksum=False,
+        ignore_attr_name_index=False,
     ):
         if not isinstance(stream, io.RawIOBase) and not isinstance(
             stream, io.BufferedIOBase
@@ -73,6 +74,7 @@ def __init__(
         self.ignore_flags = ignore_flags
         self.ignore_magic = ignore_magic
         self.ignore_checksum = ignore_checksum
+        self.ignore_attr_name_index = ignore_attr_name_index
         self.superblock = Superblock(self)
         self.superblock.verify()
         self.group_descriptors = []
@@ -83,7 +85,7 @@ def __init__(
         ):
             descriptor = BlockDescriptor(
                 self,
-                table_offset + (index * self.superblock.s_desc_size),
+                table_offset + (index * self.superblock.desc_size),
                 index,
             )
             descriptor.verify()

ext4/xattr.py

@@ -1,3 +1,4 @@
+from typing import override
 import warnings
 
 from ctypes import c_uint32
@@ -8,6 +9,7 @@
 from .struct import Ext4Struct
 from .struct import crc32c
 from .enum import EXT4_FL
+from .enum import EXT4_FEATURE_INCOMPAT
 
 
 class ExtendedAttributeError(Exception):
@@ -40,21 +42,24 @@ def magic(self):
     def expected_magic(self):
         return 0xEA020000
 
+    def value_offset(self, entry):
+        return self.offset + sizeof(self) + entry.e_value_offs
+
     def __iter__(self):
         offset = self.offset + (4 * ((sizeof(self) + 3) // 4))
         i = 0
         while i < self.data_size:
-            entry = ExtendedAttributeEntry(offset + i)
+            entry = ExtendedAttributeEntry(self.inode, offset + i, self.data_size - i)
             if (
                 entry.e_name_len
                 | entry.e_name_index
                 | entry.e_value_offs
-                | entry.e_value_inum
+                | entry.value_inum
             ) == 0:
                 break
 
-            if entry.e_value_inum != 0:
-                inode = self.volue.inodes[entry.e_value_inum]
+            if entry.value_inum != 0:
+                inode = self.volume.inodes[entry.value_inum]
                 if (inode.i_flags & EXT4_FL.EA_INODE) != 0:
                     message = f"Inode {inode.i_no:d} is not marked as large extended attribute value"
                     if not self.volume.ignore_flags:
@@ -64,12 +69,18 @@ def __iter__(self):
                 # TODO determine if e_value_size or i_size are required to limit results?
                 value = inode.open().read()
 
+            elif entry.e_value_size != 0:
+                value_offset = self.value_offset(entry)
+                if value_offset + entry.e_value_size > self.offset + self.data_size:
+                    value = b""
+                else:
+                    self.volume.seek(value_offset)
+                    value = self.volume.read(entry.e_value_size)
             else:
-                self.volume.seek(offset + i + entry.e_value_offs)
-                self.volume.read(entry.e_value_size)
+                value = b""
 
             yield entry.name_str, value
-            i += (entry.size + 3) // 4
+            i += 4 * ((entry.size + 3) // 4)
 
 
 class ExtendedAttributeHeader(ExtendedAttributeIBodyHeader):
@@ -80,7 +91,7 @@ class ExtendedAttributeHeader(ExtendedAttributeIBodyHeader):
         ("h_blocks", c_uint32),
         ("h_hash", c_uint32),
         ("h_checksum", c_uint32),
-        ("h_reserved", c_uint32 * 2),
+        ("h_reserved", c_uint32 * 3),
     ]
 
     def verify(self):
@@ -91,6 +102,10 @@ def verify(self):
                 f"{self.inode.i_no:d}: {self.h_blocks:d} (expected 1)"
             )
 
+    @override
+    def value_offset(self, entry):
+        return self.offset + entry.e_value_offs
+
     @property
     def expected_checksum(self):
         if not self.h_checksum:
@@ -117,6 +132,7 @@ class ExtendedAttributeEntry(ExtendedAttributeBase):
         "system.posix_acl_access",
         "system.posix_acl_default",
         "trusted.",
+        "",
         "security.",
         "system.",
         "system.richacl",
@@ -143,13 +159,24 @@ def size(self):
 
     @property
     def name_str(self):
-        if 0 > self.e_name_index or self.e_name_index > len(
-            ExtendedAttributeEntry.NAME_INDICES
-        ):
-            raise ExtendedAttributeError(
-                f"Unknown attribute prefix {self.e_name_index:d}"
-            )
+        name_index = self.e_name_index
+        if 0 > name_index or name_index >= len(ExtendedAttributeEntry.NAME_INDICES):
+            msg = f"Unknown attribute prefix {self.e_name_index:d}"
+            if self.volume.ignore_attr_name_index:
+                warnings.warn(msg, RuntimeWarning)
+                name_index = 0
+            else:
+                raise ExtendedAttributeError(msg)
+
+        return ExtendedAttributeEntry.NAME_INDICES[name_index] + self.e_name.decode(
+            "iso-8859-2"
+        )
 
-        return ExtendedAttributeEntry.NAME_INDICES[
-            self.e_name_index
-        ] + self.e_name.decode("iso-8859-2")
+    @property
+    def value_inum(self):
+        if (
+            self.volume.superblock.s_feature_incompat & EXT4_FEATURE_INCOMPAT.EA_INODE
+        ) != 0:
+            return self.e_value_inum
+        else:
+            return 0

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ext4"
-version = "1.1.1"
+version = "1.2"
 authors = [
   { name="Eeems", email="eeems@eeems.email" },
 ]

test.py

@@ -44,35 +44,43 @@ def _assert(source: str):
 test_path_tuple("/test/test", (b"test", b"test"))
 test_path_tuple(b"/test/test", (b"test", b"test"))
 
-offset = os.path.getsize("test.ext4") - os.path.getsize("test.ext4.tmp")
-_assert("offset > 0")
-with open("test.ext4", "rb") as f:
-    try:
-        print("check MagicError: ", end="")
-        _ = ext4.Volume(f, offset=0)
-        FAILED = True
-        print("fail")
-        print("  MagicError not raised")
-    except ext4.struct.MagicError:
-        print("pass")
-
-    except Exception as e:
-        FAILED = True
-        print("fail")
-        print("  ", end="")
-        print(e)
-
-    # Extract specific file
-    volume = ext4.Volume(f, offset=offset)
-    inode = cast(ext4.File, volume.inode_at("/test.txt"))
-    _assert("isinstance(inode, ext4.File)")
-    b = inode.open()
-    _assert("isinstance(b, ext4.BlockIO)")
-    _assert("b.readable()")
-    _assert("b.seekable()")
-    _assert("b.seek(1) == 1")
-    _assert("b.seek(0) == 0")
-    _assert("b.seek(10) == 10")
+for img_file in ("test32.ext4", "test64.ext4"):
+    offset = os.path.getsize(img_file) - os.path.getsize(f"{img_file}.tmp")
+    _assert("offset > 0")
+    with open(img_file, "rb") as f:
+        try:
+            print("check MagicError: ", end="")
+            _ = ext4.Volume(f, offset=0)
+            FAILED = True
+            print("fail")
+            print("  MagicError not raised")
+        except ext4.struct.MagicError:
+            print("pass")
+
+        except Exception as e:
+            FAILED = True
+            print("fail")
+            print("  ", end="")
+            print(e)
+
+        # Extract specific file
+        volume = ext4.Volume(f, offset=offset)
+        inode = cast(ext4.File, volume.inode_at("/test.txt"))
+        _assert("isinstance(inode, ext4.File)")
+        b = inode.open()
+        _assert("isinstance(b, ext4.BlockIO)")
+        _assert("b.readable()")
+        _assert("b.seekable()")
+        _assert("b.seek(1) == 1")
+        _assert("b.seek(0) == 0")
+        _assert("b.seek(10) == 10")
+        for i in range(1, 101):
+            inode = volume.inode_at(f"/test{i}.txt")
+            attrs = {k: v for k, v in inode.xattrs}
+            for j in range(1, 21):
+                _assert(f'attrs["user.name{j}"] == b"value{i}_{j}"')
+            data = inode.open().read()
+            _assert(f'data == b"hello world{i}\\n"')
 
 if FAILED:
     sys.exit(1)

test.sh

@@ -15,8 +15,8 @@ python -m pip install wheel
 python -m pip install \
   --extra-index-url=https://wheels.eeems.codes/ \
   -r requirements.txt
-if ! [ -f test.ext4 ] || ! [ -f test.ext4.tmp ];then
+if ! [ -f test32.ext4 ] || ! [ -f test32.ext4.tmp ] || ! [ -f test64.ext4 ] || ! [ -f test64.ext4.tmp ] ;then
   ./_test_image.sh
-  trap "rm -f test.ext4{,.tmp}" EXIT
+  trap "rm -f test{32,64}.ext4{,.tmp}" EXIT
 fi
 python test.py