Added fsext back-end #464

appveyor.yml

@@ -20,7 +20,7 @@ install:
 - cmd: IF [%PYTHON_VERSION%]==[3.8] (
     mkdir dependencies &&
     set PYTHONPATH=..\l2tdevtools &&
-    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track "%L2TBINARIES_TRACK%" PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsntfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pytsk3 six )
+    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track "%L2TBINARIES_TRACK%" PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfsntfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pytsk3 six )
 
 build: off
 

config/dpkg/control

@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200424), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200424), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

config/linux/gift_copr_install.sh

@@ -11,6 +11,7 @@ set -e
 PYTHON3_DEPENDENCIES="libbde-python3
                       libewf-python3
                       libfsapfs-python3
+                      libfsext-python3
                       libfsntfs-python3
                       libfvde-python3
                       libfwnt-python3
@@ -47,6 +48,8 @@ DEBUG_DEPENDENCIES="libbde-debuginfo
                     libewf-python3-debuginfo
                     libfsapfs-debuginfo
                     libfsapfs-python3-debuginfo
+                    libfsext-debuginfo
+                    libfsext-python3-debuginfo
                     libfsntfs-debuginfo
                     libfsntfs-python3-debuginfo
                     libfvde-debuginfo

config/linux/ubuntu_install_dfvfs.sh

@@ -18,6 +18,7 @@ export DEBIAN_FRONTEND=noninteractive
 PYTHON_DEPENDENCIES="libbde-python3
                      libewf-python3
                      libfsapfs-python3
+                     libfsext-python3
                      libfsntfs-python3
                      libfvde-python3
                      libfwnt-python3
@@ -56,6 +57,8 @@ DEBUG_DEPENDENCIES="libbde-dbg
                     libewf-python3-dbg
                     libfsapfs-dbg
                     libfsapfs-python3-dbg
+                    libfsext-dbg
+                    libfsext-python3-dbg
                     libfsntfs-dbg
                     libfsntfs-python3-dbg
                     libfvde-dbg

config/travis/install.sh

@@ -5,11 +5,11 @@
 # This file is generated by l2tdevtools update-dependencies.py any dependency
 # related changes should be made in dependencies.ini.
 
-DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-yaml";
+DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-yaml";
 
 DPKG_PYTHON3_TEST_DEPENDENCIES="python3-coverage python3-distutils python3-mock python3-pbr python3-setuptools python3-six";
 
-RPM_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml";
+RPM_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml";
 
 RPM_PYTHON3_TEST_DEPENDENCIES="python3-mock python3-pbr python3-setuptools python3-six";
 

config/travis/runtests.sh

@@ -50,7 +50,14 @@ then
 
 elif test "${TARGET}" = "dockerfile";
 then
-	cd config/docker && docker build --build-arg PPA_TRACK="dev" -f Dockerfile .
+	SOURCE_PATH=${PWD};
+	CONTAINER_NAME="test";
+
+	cd config/docker
+
+	docker build --build-arg PPA_TRACK="dev" -f Dockerfile -t ${CONTAINER_NAME} .
+
+	# TODO: add tests
 
 elif test "${TRAVIS_OS_NAME}" = "osx";
 then

dependencies.ini

@@ -53,6 +53,14 @@ pypi_name: libfsapfs-python
 rpm_name: libfsapfs-python3
 version_property: get_version()
 
+[pyfsext]
+dpkg_name: libfsext-python3
+l2tbinaries_name: libfsext
+minimum_version: 20200819
+pypi_name: libfsext-python
+rpm_name: libfsext-python3
+version_property: get_version()
+
 [pyfsntfs]
 dpkg_name: libfsntfs-python3
 l2tbinaries_name: libfsntfs

dfvfs/analyzer/__init__.py

@@ -7,6 +7,7 @@
 from dfvfs.analyzer import bzip2_analyzer_helper
 from dfvfs.analyzer import cpio_analyzer_helper
 from dfvfs.analyzer import ewf_analyzer_helper
+from dfvfs.analyzer import ext_analyzer_helper
 from dfvfs.analyzer import fvde_analyzer_helper
 from dfvfs.analyzer import gzip_analyzer_helper
 from dfvfs.analyzer import lvm_analyzer_helper

dfvfs/analyzer/ext_analyzer_helper.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""The EXT format analyzer helper implementation."""
+
+from __future__ import unicode_literals
+
+from dfvfs.analyzer import analyzer
+from dfvfs.analyzer import analyzer_helper
+from dfvfs.analyzer import specification
+from dfvfs.lib import definitions
+
+
+class EXTAnalyzerHelper(analyzer_helper.AnalyzerHelper):
+  """EXT analyzer helper."""
+
+  FORMAT_CATEGORIES = frozenset([
+      definitions.FORMAT_CATEGORY_FILE_SYSTEM])
+
+  TYPE_INDICATOR = definitions.TYPE_INDICATOR_EXT
+
+  def GetFormatSpecification(self):
+    """Retrieves the format specification.
+
+    Returns:
+      FormatSpecification: format specification or None if the format cannot
+          be defined by a specification object.
+    """
+    format_specification = specification.FormatSpecification(
+        self.type_indicator)
+
+    # EXT file system signature.
+    format_specification.AddNewSignature(b'\x53\xef', offset=1080)
+
+    return format_specification
+
+  def IsEnabled(self):
+    """Determines if the analyzer helper is enabled.
+
+    Returns:
+      bool: True if the analyzer helper is enabled.
+    """
+    return definitions.PREFERRED_EXT_BACK_END == self.TYPE_INDICATOR
+
+
+analyzer.Analyzer.RegisterHelper(EXTAnalyzerHelper())

dfvfs/analyzer/tsk_analyzer_helper.py

@@ -46,8 +46,9 @@ def GetFormatSpecification(self):
     # HFSX file system signature.
     format_specification.AddNewSignature(b'HX', offset=1024)
 
-    # Ext file system signature.
-    format_specification.AddNewSignature(b'\x53\xef', offset=1080)
+    if definitions.PREFERRED_EXT_BACK_END == self.TYPE_INDICATOR:
+      # Ext file system signature.
+      format_specification.AddNewSignature(b'\x53\xef', offset=1080)
 
     # ISO9660 file system signature.
     format_specification.AddNewSignature(b'CD001', offset=32769)

dfvfs/file_io/ext_file_io.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+"""The Extended File System (EXT) file-like object implementation."""
+
+from __future__ import unicode_literals
+
+import os
+
+from dfvfs.file_io import file_io
+from dfvfs.lib import errors
+from dfvfs.resolver import resolver
+
+
+class EXTFile(file_io.FileIO):
+  """File-like object using pyfsext.file_entry"""
+
+  def __init__(self, resolver_context):
+    """Initializes a file-like object.
+
+    Args:
+      resolver_context (Context): resolver context.
+    """
+    super(EXTFile, self).__init__(resolver_context)
+    self._file_system = None
+    self._fsext_file_entry = None
+
+  def _Close(self):
+    """Closes the file-like object."""
+    self._fsext_file_entry = None
+
+    self._file_system.Close()
+    self._file_system = None
+
+  def _Open(self, path_spec=None, mode='rb'):
+    """Opens the file-like object defined by path specification.
+
+    Args:
+      path_spec (PathSpec): path specification.
+      mode (Optional[str]): file access mode.
+
+    Raises:
+      AccessError: if the access to open the file was denied.
+      IOError: if the file-like object could not be opened.
+      NotSupported: if a data stream, like the resource or named fork, is
+          requested to be opened.
+      OSError: if the file-like object could not be opened.
+      PathSpecError: if the path specification is incorrect.
+      ValueError: if the path specification is invalid.
+    """
+    if not path_spec:
+      raise ValueError('Missing path specification.')
+
+    data_stream = getattr(path_spec, 'data_stream', None)
+    if data_stream:
+      raise errors.NotSupported(
+          'Open data stream: {0:s} not supported.'.format(data_stream))
+
+    self._file_system = resolver.Resolver.OpenFileSystem(
+        path_spec, resolver_context=self._resolver_context)
+
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    if not file_entry:
+      raise IOError('Unable to open file entry.')
+
+    fsext_file_entry = file_entry.GetEXTFileEntry()
+    if not fsext_file_entry:
+      raise IOError('Unable to open EXT file entry.')
+
+    self._fsext_file_entry = fsext_file_entry
+
+  # Note: that the following functions do not follow the style guide
+  # because they are part of the file-like object interface.
+  # pylint: disable=invalid-name
+
+  def read(self, size=None):
+    """Reads a byte string from the file-like object at the current offset.
+
+    The function will read a byte string of the specified size or
+    all of the remaining data if no size was specified.
+
+    Args:
+      size (Optional[int]): number of bytes to read, where None is all
+          remaining data.
+
+    Returns:
+      bytes: data read.
+
+    Raises:
+      IOError: if the read failed.
+      OSError: if the read failed.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    return self._fsext_file_entry.read(size=size)
+
+  def seek(self, offset, whence=os.SEEK_SET):
+    """Seeks to an offset within the file-like object.
+
+    Args:
+      offset (int): offset to seek to.
+      whence (Optional(int)): value that indicates whether offset is an absolute
+          or relative position within the file.
+
+    Raises:
+      IOError: if the seek failed.
+      OSError: if the seek failed.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    self._fsext_file_entry.seek(offset, whence)
+
+  def get_offset(self):
+    """Retrieves the current offset into the file-like object.
+
+    Return:
+      int: current offset into the file-like object.
+
+    Raises:
+      IOError: if the file-like object has not been opened.
+      OSError: if the file-like object has not been opened.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    return self._fsext_file_entry.get_offset()
+
+  def get_size(self):
+    """Retrieves the size of the file-like object.
+
+    Returns:
+      int: size of the file-like object data.
+
+    Raises:
+      IOError: if the file-like object has not been opened.
+      OSError: if the file-like object has not been opened.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    return self._fsext_file_entry.get_size()

dfvfs/lib/definitions.py

@@ -39,6 +39,7 @@
 TYPE_INDICATOR_ENCODED_STREAM = 'ENCODED_STREAM'
 TYPE_INDICATOR_ENCRYPTED_STREAM = 'ENCRYPTED_STREAM'
 TYPE_INDICATOR_EWF = 'EWF'
+TYPE_INDICATOR_EXT = 'EXT'
 TYPE_INDICATOR_FAKE = 'FAKE'
 TYPE_INDICATOR_FVDE = 'FVDE'
 TYPE_INDICATOR_GZIP = 'GZIP'
@@ -74,6 +75,7 @@
 
 FILE_SYSTEM_TYPE_INDICATORS = frozenset([
     TYPE_INDICATOR_APFS,
+    TYPE_INDICATOR_EXT,
     TYPE_INDICATOR_FAKE,
     TYPE_INDICATOR_NTFS,
     TYPE_INDICATOR_TSK])
@@ -92,6 +94,7 @@
     TYPE_INDICATOR_VSHADOW])
 
 # The preferred back-ends.
+PREFERRED_EXT_BACK_END = TYPE_INDICATOR_TSK
 PREFERRED_NTFS_BACK_END = TYPE_INDICATOR_NTFS
 
 # The NTFS attribute types.

dfvfs/path/__init__.py

@@ -10,6 +10,7 @@
 from dfvfs.path import encoded_stream_path_spec
 from dfvfs.path import encrypted_stream_path_spec
 from dfvfs.path import ewf_path_spec
+from dfvfs.path import ext_path_spec
 from dfvfs.path import fake_path_spec
 from dfvfs.path import fvde_path_spec
 from dfvfs.path import gzip_path_spec

dfvfs/path/ext_path_spec.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""The EXT path specification implementation."""
+
+from __future__ import unicode_literals
+
+from dfvfs.lib import definitions
+from dfvfs.path import factory
+from dfvfs.path import path_spec
+
+
+class EXTPathSpec(path_spec.PathSpec):
+  """EXT path specification implementation.
+
+  Attributes:
+    inode (int): inode.
+    location (str): location.
+  """
+
+  TYPE_INDICATOR = definitions.TYPE_INDICATOR_EXT
+
+  def __init__(
+      self, inode=None, location=None, parent=None, **kwargs):
+    """Initializes a path specification.
+
+    Note that an EXT path specification must have a parent.
+
+    Args:
+      inode (Optional[int]): inode.
+      location (Optional[str]): location.
+      parent (Optional[PathSpec]): parent path specification.
+
+    Raises:
+      ValueError: when parent or both inode and location are not set.
+    """
+    if (not inode and not location) or not parent:
+      raise ValueError('Missing inode and location, or parent value.')
+
+    super(EXTPathSpec, self).__init__(parent=parent, **kwargs)
+    self.inode = inode
+    self.location = location
+
+  @property
+  def comparable(self):
+    """str: comparable representation of the path specification."""
+    string_parts = []
+
+    if self.inode is not None:
+      string_parts.append('inode: {0:d}'.format(self.inode))
+    if self.location is not None:
+      string_parts.append('location: {0:s}'.format(self.location))
+
+    return self._GetComparable(sub_comparable_string=', '.join(string_parts))
+
+
+factory.Factory.RegisterPathSpec(EXTPathSpec)