Skip to content

Commit

Permalink
Added fsext back-end #464
Browse files Browse the repository at this point in the history
  • Loading branch information
joachimmetz committed Sep 1, 2020
1 parent 674c982 commit 2231389
Show file tree
Hide file tree
Showing 25 changed files with 1,415 additions and 7 deletions.
2 changes: 1 addition & 1 deletion appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ install:
- cmd: IF [%PYTHON_VERSION%]==[3.8] (
mkdir dependencies &&
set PYTHONPATH=..\l2tdevtools &&
"%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track "%L2TBINARIES_TRACK%" PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsntfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pytsk3 six )
"%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track "%L2TBINARIES_TRACK%" PyYAML cffi cryptography dfdatetime dtfabric idna libbde libewf libfsapfs libfsext libfsntfs libfvde libfwnt libluksde libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pytsk3 six )

build: off

Expand Down
2 changes: 1 addition & 1 deletion config/dpkg/control
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs

Package: python3-dfvfs
Architecture: all
Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsntfs-python3 (>= 20200424), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181205), libfsext-python3 (>= 20200819), libfsntfs-python3 (>= 20200424), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libluksde-python3 (>= 20200101), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20200809), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
Description: Python 3 module of dfVFS
dfVFS, or Digital Forensics Virtual File System, provides read-only access to
file-system objects from various storage media types and file formats. The goal
Expand Down
3 changes: 3 additions & 0 deletions config/linux/gift_copr_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ set -e
PYTHON3_DEPENDENCIES="libbde-python3
libewf-python3
libfsapfs-python3
libfsext-python3
libfsntfs-python3
libfvde-python3
libfwnt-python3
Expand Down Expand Up @@ -47,6 +48,8 @@ DEBUG_DEPENDENCIES="libbde-debuginfo
libewf-python3-debuginfo
libfsapfs-debuginfo
libfsapfs-python3-debuginfo
libfsext-debuginfo
libfsext-python3-debuginfo
libfsntfs-debuginfo
libfsntfs-python3-debuginfo
libfvde-debuginfo
Expand Down
3 changes: 3 additions & 0 deletions config/linux/ubuntu_install_dfvfs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ export DEBIAN_FRONTEND=noninteractive
PYTHON_DEPENDENCIES="libbde-python3
libewf-python3
libfsapfs-python3
libfsext-python3
libfsntfs-python3
libfvde-python3
libfwnt-python3
Expand Down Expand Up @@ -56,6 +57,8 @@ DEBUG_DEPENDENCIES="libbde-dbg
libewf-python3-dbg
libfsapfs-dbg
libfsapfs-python3-dbg
libfsext-dbg
libfsext-python3-dbg
libfsntfs-dbg
libfsntfs-python3-dbg
libfvde-dbg
Expand Down
4 changes: 2 additions & 2 deletions config/travis/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
# This file is generated by l2tdevtools update-dependencies.py any dependency
# related changes should be made in dependencies.ini.

DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-yaml";
DPKG_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi-backend python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-yaml";

DPKG_PYTHON3_TEST_DEPENDENCIES="python3-coverage python3-distutils python3-mock python3-pbr python3-setuptools python3-six";

RPM_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml";
RPM_PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsext-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libluksde-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-cffi python3-cryptography python3-dfdatetime python3-dtfabric python3-idna python3-pytsk3 python3-pyyaml";

RPM_PYTHON3_TEST_DEPENDENCIES="python3-mock python3-pbr python3-setuptools python3-six";

Expand Down
9 changes: 8 additions & 1 deletion config/travis/runtests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,14 @@ then

elif test "${TARGET}" = "dockerfile";
then
cd config/docker && docker build --build-arg PPA_TRACK="dev" -f Dockerfile .
SOURCE_PATH=${PWD};
CONTAINER_NAME="test";

cd config/docker

docker build --build-arg PPA_TRACK="dev" -f Dockerfile -t ${CONTAINER_NAME} .

# TODO: add tests

elif test "${TRAVIS_OS_NAME}" = "osx";
then
Expand Down
8 changes: 8 additions & 0 deletions dependencies.ini
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ pypi_name: libfsapfs-python
rpm_name: libfsapfs-python3
version_property: get_version()

[pyfsext]
dpkg_name: libfsext-python3
l2tbinaries_name: libfsext
minimum_version: 20200819
pypi_name: libfsext-python
rpm_name: libfsext-python3
version_property: get_version()

[pyfsntfs]
dpkg_name: libfsntfs-python3
l2tbinaries_name: libfsntfs
Expand Down
1 change: 1 addition & 0 deletions dfvfs/analyzer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from dfvfs.analyzer import bzip2_analyzer_helper
from dfvfs.analyzer import cpio_analyzer_helper
from dfvfs.analyzer import ewf_analyzer_helper
from dfvfs.analyzer import ext_analyzer_helper
from dfvfs.analyzer import fvde_analyzer_helper
from dfvfs.analyzer import gzip_analyzer_helper
from dfvfs.analyzer import lvm_analyzer_helper
Expand Down
44 changes: 44 additions & 0 deletions dfvfs/analyzer/ext_analyzer_helper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
"""The EXT format analyzer helper implementation."""

from __future__ import unicode_literals

from dfvfs.analyzer import analyzer
from dfvfs.analyzer import analyzer_helper
from dfvfs.analyzer import specification
from dfvfs.lib import definitions


class EXTAnalyzerHelper(analyzer_helper.AnalyzerHelper):
"""EXT analyzer helper."""

FORMAT_CATEGORIES = frozenset([
definitions.FORMAT_CATEGORY_FILE_SYSTEM])

TYPE_INDICATOR = definitions.TYPE_INDICATOR_EXT

def GetFormatSpecification(self):
"""Retrieves the format specification.
Returns:
FormatSpecification: format specification or None if the format cannot
be defined by a specification object.
"""
format_specification = specification.FormatSpecification(
self.type_indicator)

# EXT file system signature.
format_specification.AddNewSignature(b'\x53\xef', offset=1080)

return format_specification

def IsEnabled(self):
"""Determines if the analyzer helper is enabled.
Returns:
bool: True if the analyzer helper is enabled.
"""
return definitions.PREFERRED_EXT_BACK_END == self.TYPE_INDICATOR


analyzer.Analyzer.RegisterHelper(EXTAnalyzerHelper())
5 changes: 3 additions & 2 deletions dfvfs/analyzer/tsk_analyzer_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ def GetFormatSpecification(self):
# HFSX file system signature.
format_specification.AddNewSignature(b'HX', offset=1024)

# Ext file system signature.
format_specification.AddNewSignature(b'\x53\xef', offset=1080)
if definitions.PREFERRED_EXT_BACK_END == self.TYPE_INDICATOR:
# Ext file system signature.
format_specification.AddNewSignature(b'\x53\xef', offset=1080)

# ISO9660 file system signature.
format_specification.AddNewSignature(b'CD001', offset=32769)
Expand Down
141 changes: 141 additions & 0 deletions dfvfs/file_io/ext_file_io.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
# -*- coding: utf-8 -*-
"""The Extended File System (EXT) file-like object implementation."""

from __future__ import unicode_literals

import os

from dfvfs.file_io import file_io
from dfvfs.lib import errors
from dfvfs.resolver import resolver


class EXTFile(file_io.FileIO):
"""File-like object using pyfsext.file_entry"""

def __init__(self, resolver_context):
"""Initializes a file-like object.
Args:
resolver_context (Context): resolver context.
"""
super(EXTFile, self).__init__(resolver_context)
self._file_system = None
self._fsext_file_entry = None

def _Close(self):
"""Closes the file-like object."""
self._fsext_file_entry = None

self._file_system.Close()
self._file_system = None

def _Open(self, path_spec=None, mode='rb'):
"""Opens the file-like object defined by path specification.
Args:
path_spec (PathSpec): path specification.
mode (Optional[str]): file access mode.
Raises:
AccessError: if the access to open the file was denied.
IOError: if the file-like object could not be opened.
NotSupported: if a data stream, like the resource or named fork, is
requested to be opened.
OSError: if the file-like object could not be opened.
PathSpecError: if the path specification is incorrect.
ValueError: if the path specification is invalid.
"""
if not path_spec:
raise ValueError('Missing path specification.')

data_stream = getattr(path_spec, 'data_stream', None)
if data_stream:
raise errors.NotSupported(
'Open data stream: {0:s} not supported.'.format(data_stream))

self._file_system = resolver.Resolver.OpenFileSystem(
path_spec, resolver_context=self._resolver_context)

file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
if not file_entry:
raise IOError('Unable to open file entry.')

fsext_file_entry = file_entry.GetEXTFileEntry()
if not fsext_file_entry:
raise IOError('Unable to open EXT file entry.')

self._fsext_file_entry = fsext_file_entry

# Note: that the following functions do not follow the style guide
# because they are part of the file-like object interface.
# pylint: disable=invalid-name

def read(self, size=None):
"""Reads a byte string from the file-like object at the current offset.
The function will read a byte string of the specified size or
all of the remaining data if no size was specified.
Args:
size (Optional[int]): number of bytes to read, where None is all
remaining data.
Returns:
bytes: data read.
Raises:
IOError: if the read failed.
OSError: if the read failed.
"""
if not self._is_open:
raise IOError('Not opened.')

return self._fsext_file_entry.read(size=size)

def seek(self, offset, whence=os.SEEK_SET):
"""Seeks to an offset within the file-like object.
Args:
offset (int): offset to seek to.
whence (Optional(int)): value that indicates whether offset is an absolute
or relative position within the file.
Raises:
IOError: if the seek failed.
OSError: if the seek failed.
"""
if not self._is_open:
raise IOError('Not opened.')

self._fsext_file_entry.seek(offset, whence)

def get_offset(self):
"""Retrieves the current offset into the file-like object.
Return:
int: current offset into the file-like object.
Raises:
IOError: if the file-like object has not been opened.
OSError: if the file-like object has not been opened.
"""
if not self._is_open:
raise IOError('Not opened.')

return self._fsext_file_entry.get_offset()

def get_size(self):
"""Retrieves the size of the file-like object.
Returns:
int: size of the file-like object data.
Raises:
IOError: if the file-like object has not been opened.
OSError: if the file-like object has not been opened.
"""
if not self._is_open:
raise IOError('Not opened.')

return self._fsext_file_entry.get_size()
3 changes: 3 additions & 0 deletions dfvfs/lib/definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
TYPE_INDICATOR_ENCODED_STREAM = 'ENCODED_STREAM'
TYPE_INDICATOR_ENCRYPTED_STREAM = 'ENCRYPTED_STREAM'
TYPE_INDICATOR_EWF = 'EWF'
TYPE_INDICATOR_EXT = 'EXT'
TYPE_INDICATOR_FAKE = 'FAKE'
TYPE_INDICATOR_FVDE = 'FVDE'
TYPE_INDICATOR_GZIP = 'GZIP'
Expand Down Expand Up @@ -74,6 +75,7 @@

FILE_SYSTEM_TYPE_INDICATORS = frozenset([
TYPE_INDICATOR_APFS,
TYPE_INDICATOR_EXT,
TYPE_INDICATOR_FAKE,
TYPE_INDICATOR_NTFS,
TYPE_INDICATOR_TSK])
Expand All @@ -92,6 +94,7 @@
TYPE_INDICATOR_VSHADOW])

# The preferred back-ends.
PREFERRED_EXT_BACK_END = TYPE_INDICATOR_TSK
PREFERRED_NTFS_BACK_END = TYPE_INDICATOR_NTFS

# The NTFS attribute types.
Expand Down
1 change: 1 addition & 0 deletions dfvfs/path/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from dfvfs.path import encoded_stream_path_spec
from dfvfs.path import encrypted_stream_path_spec
from dfvfs.path import ewf_path_spec
from dfvfs.path import ext_path_spec
from dfvfs.path import fake_path_spec
from dfvfs.path import fvde_path_spec
from dfvfs.path import gzip_path_spec
Expand Down
55 changes: 55 additions & 0 deletions dfvfs/path/ext_path_spec.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
"""The EXT path specification implementation."""

from __future__ import unicode_literals

from dfvfs.lib import definitions
from dfvfs.path import factory
from dfvfs.path import path_spec


class EXTPathSpec(path_spec.PathSpec):
"""EXT path specification implementation.
Attributes:
inode (int): inode.
location (str): location.
"""

TYPE_INDICATOR = definitions.TYPE_INDICATOR_EXT

def __init__(
self, inode=None, location=None, parent=None, **kwargs):
"""Initializes a path specification.
Note that an EXT path specification must have a parent.
Args:
inode (Optional[int]): inode.
location (Optional[str]): location.
parent (Optional[PathSpec]): parent path specification.
Raises:
ValueError: when parent or both inode and location are not set.
"""
if (not inode and not location) or not parent:
raise ValueError('Missing inode and location, or parent value.')

super(EXTPathSpec, self).__init__(parent=parent, **kwargs)
self.inode = inode
self.location = location

@property
def comparable(self):
"""str: comparable representation of the path specification."""
string_parts = []

if self.inode is not None:
string_parts.append('inode: {0:d}'.format(self.inode))
if self.location is not None:
string_parts.append('location: {0:s}'.format(self.location))

return self._GetComparable(sub_comparable_string=', '.join(string_parts))


factory.Factory.RegisterPathSpec(EXTPathSpec)
Loading

0 comments on commit 2231389

Please sign in to comment.