Improved and refactored setupapi parser #2717 (#2718)

data/presets.yaml

@@ -113,6 +113,7 @@ parsers:
 - olecf
 - pe
 - prefetch
+- setupapi
 - sccm
 - skydrive_log
 - skydrive_log_old

plaso/formatters/setupapi.py

@@ -14,11 +14,11 @@ class SetupapiLogFormatter(interface.ConditionalEventFormatter):
 
   FORMAT_STRING_PIECES = [
       '{entry_type}',
-      '{entry_status}']
+      '{exit_status}']
 
   # Reversing fields for short description to prevent truncating the status
   FORMAT_STRING_SHORT_PIECES = [
-      '{entry_status}',
+      '{exit_status}',
       '{entry_type}']
 
   FORMAT_STRING_SEPARATOR = ' - '

plaso/parsers/iis.py

@@ -179,6 +179,7 @@ def _ParseFieldsMetadata(self, structure):
       structure (pyparsing.ParseResults): structure parsed from the log file.
     """
     fields = self._GetValueFromStructure(structure, 'fields', default_value='')
+    fields = fields.strip()
     fields = fields.split(' ')
 
     log_line_structure = pyparsing.Empty()

plaso/parsers/mac_wifi.py

@@ -228,15 +228,15 @@ def _ParseLogLine(self, parser_mediator, key, structure):
     self._last_month = time_elements_tuple[1]
 
     function = self._GetValueFromStructure(structure, 'function')
-    if function:
-      # Due to the use of CharsNotIn the function value can contain leading
-      # or trailing whitespace.
-      function = function.strip()
+
+    text = self._GetValueFromStructure(structure, 'text')
+    if text:
+      text = text.strip()
 
     event_data = MacWifiLogEventData()
     event_data.agent = self._GetValueFromStructure(structure, 'agent')
     event_data.function = function
-    event_data.text = self._GetValueFromStructure(structure, 'text')
+    event_data.text = text
 
     if key == 'known_function_logline':
       event_data.action = self._GetAction(

plaso/parsers/setupapi.py

@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
-"""Parser for Windows Setupapi log files."""
+"""Parser for Windows Setupapi log files.
+
+The format is documented at:
+https://docs.microsoft.com/en-us/windows-hardware/drivers/install/setupapi-text-logs
+"""
 
 from __future__ import unicode_literals
 
@@ -20,8 +24,10 @@ class SetupapiLogEventData(events.EventData):
   """Setupapi log event data.
 
   Attributes:
-    entry_type (str): log entry type such as "Device Install".
-    entry_status (str): the status of the entry.
+    entry_type (str): log entry type, for examaple "Device Install -
+        PCI\\VEN_104C&DEV_8019&SUBSYS_8010104C&REV_00\\3&61aaa01&0&38" or
+        "Sysprep Respecialize - {804b345a-ffd7-854c-a1b5-ca9598907846}".
+    exit_status (str): the exit status of the logged operation.
   """
 
   DATA_TYPE = 'setupapi:log:line'
@@ -30,11 +36,10 @@ def __init__(self):
     """Initializes event data."""
     super(SetupapiLogEventData, self).__init__(data_type=self.DATA_TYPE)
     self.entry_type = None
-    # TODO: Parse additional fields from the body of setupapi messages
-    self.entry_status = None
+    self.exit_status = None
 
 
-class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser):
+class SetupapiLogParser(text_parser.PyparsingSingleLineTextParser):
   """Parses events from Windows Setupapi log files."""
 
   NAME = 'setupapi'
@@ -43,9 +48,6 @@ class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser):
 
   _ENCODING = 'utf-8'
 
-  # Increase the buffer size, as log messages can be very long.
-  BUFFER_SIZE = 262144
-
   _SLASH = pyparsing.Literal('/').suppress()
 
   _FOUR_DIGITS = text_parser.PyparsingConstants.FOUR_DIGITS
@@ -63,74 +65,109 @@ class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser):
       _THREE_DIGITS
   )
 
-  _SETUPAPI_LINE = (
-      pyparsing.SkipTo('>>>  [', include=True).suppress() +
-      pyparsing.SkipTo(']').setResultsName('entry_type') +
-      pyparsing.SkipTo('>>>  Section start', include=True).suppress() +
+  # Disable pylint due to long URLs for documenting structures.
+  # pylint: disable=line-too-long
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-header
+  _LOG_HEADER_START = (
+      pyparsing.Literal('[Device Install Log]') +
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-header
+  _LOG_HEADER_END = (
+      pyparsing.Literal('[BeginLog]') +
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-header
+  _SECTION_HEADER = (
+      pyparsing.Literal('>>>  [').suppress() +
+      pyparsing.CharsNotIn(']').setResultsName('entry_type') +
+      pyparsing.Literal(']') +
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-header
+  _SECTION_HEADER_START = (
+      pyparsing.Literal('>>>  Section start').suppress() +
       _SETUPAPI_DATE_TIME.setResultsName('start_time') +
-      pyparsing.SkipTo('<<<  Section end ').setResultsName('message') +
-      pyparsing.GoToColumn(17) +
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-footer
+  _SECTION_END = (
+      pyparsing.Literal('<<<  Section end ').suppress() +
       _SETUPAPI_DATE_TIME.setResultsName('end_time') +
-      pyparsing.SkipTo('<<<  [Exit status: ', include=True).suppress() +
-      pyparsing.SkipTo(']').setResultsName('entry_status') +
-      pyparsing.SkipTo(pyparsing.lineEnd()) +
-      pyparsing.ZeroOrMore(pyparsing.lineEnd()))
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-footer
+  _SECTION_END_EXIT_STATUS = (
+      pyparsing.Literal('<<<  [Exit status: ').suppress() +
+      pyparsing.CharsNotIn(']').setResultsName('exit_status') +
+      pyparsing.Literal(']') +
+      pyparsing.lineEnd())
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-log-entries-that-are-not-part-of-a-text-log-section
+  _SECTION_BODY_LINE = (
+      pyparsing.stringStart +
+      pyparsing.MatchFirst([
+          pyparsing.Literal('!!!  '),
+          pyparsing.Literal('!    '),
+          pyparsing.Literal('     ')]) +
+      pyparsing.restOfLine).leaveWhitespace()
+
+  # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-log-entries-that-are-not-part-of-a-text-log-section
+  _NON_SECTION_LINE = (
+      pyparsing.stringStart +
+      pyparsing.MatchFirst([
+          pyparsing.Literal('   . '),
+          pyparsing.Literal('!!!  '),
+          pyparsing.Literal('!    '),
+          pyparsing.Literal('     ')]) +
+      pyparsing.restOfLine).leaveWhitespace()
+
+  # These lines do not appear to be documented in the Microsoft documentation.
+  _BOOT_SESSION_LINE = (
+      pyparsing.Literal('[Boot Session:') +
+      _SETUPAPI_DATE_TIME +
+      pyparsing.Literal(']'))
+
+  # pylint: enable=line-too-long
 
   LINE_STRUCTURES = [
-      ('logline', _SETUPAPI_LINE),
-  ]
+      ('ignorable_line', _BOOT_SESSION_LINE),
+      ('ignorable_line', _LOG_HEADER_END),
+      ('ignorable_line', _LOG_HEADER_START),
+      ('ignorable_line', _NON_SECTION_LINE),
+      ('ignorable_line', _SECTION_BODY_LINE),
+      ('section_end', _SECTION_END),
+      ('section_end_exit_status', _SECTION_END_EXIT_STATUS),
+      ('section_header', _SECTION_HEADER),
+      ('section_start', _SECTION_HEADER_START)]
+
+  def __init__(self):
+    """Initializes a setupapi parser."""
+    super(SetupapiLogParser, self).__init__()
+    self._last_end_time = None
+    self._last_entry_type = None
 
-  def _ParseRecordLogline(self, parser_mediator, structure):
-    """Parses a logline record structure and produces events.
+  def _GetTimeElements(self, time_structure):
+    """Builds time elements from a setupapi time_stamp field.
 
     Args:
-      parser_mediator (ParserMediator): mediates interactions between parsers
-          and other components, such as storage and dfvfs.
-      structure (pyparsing.ParseResults): structure of tokens derived from
-          log entry.
+      time_structure (pyparsing.ParseResults): structure of tokens derived from
+          a setupapi time_stamp field.
+
+    Returns:
+      dfdatetime.TimeElements: date and time extracted from the value or None
+          if the structure does not represent a valid date and time value.
     """
-    time_zone = parser_mediator.timezone
-    time_elements_structure = self._GetValueFromStructure(
-        structure, 'start_time')
     try:
       date_time = dfdatetime_time_elements.TimeElementsInMilliseconds(
-          time_elements_tuple=time_elements_structure)
-      # Setupapi logs stores date and time values in local time.
+          time_elements_tuple=time_structure)
+      # Setupapi logs store date and time values in local time.
       date_time.is_local_time = True
-    except ValueError:
-      parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_structure))
-      return
-
-    event_data = SetupapiLogEventData()
-    event_data.entry_type = self._GetValueFromStructure(structure, 'entry_type')
-    event_data.entry_status = 'START'
-
-    event = time_events.DateTimeValuesEvent(
-        date_time, definitions.TIME_DESCRIPTION_START, time_zone=time_zone)
-
-    # Create event for the start of the setupapi section
-    parser_mediator.ProduceEventWithEventData(event, event_data)
+      return date_time
 
-    event_data.entry_status = self._GetValueFromStructure(
-        structure, 'entry_status')
-
-    time_elements_structure = self._GetValueFromStructure(
-        structure, 'end_time')
-    try:
-      date_time = dfdatetime_time_elements.TimeElementsInMilliseconds(
-          time_elements_tuple=time_elements_structure)
-      date_time.is_local_time = True
     except ValueError:
-      parser_mediator.ProduceExtractionWarning(
-          'invalid date time value: {0!s}'.format(time_elements_structure))
-      return
-
-    event = time_events.DateTimeValuesEvent(
-        date_time, definitions.TIME_DESCRIPTION_END, time_zone=time_zone)
-
-    # Create event for the end of the setupapi section
-    parser_mediator.ProduceEventWithEventData(event, event_data)
+      return None
 
   def ParseRecord(self, parser_mediator, key, structure):
     """Parses a log record structure and produces events.
@@ -145,47 +182,83 @@ def ParseRecord(self, parser_mediator, key, structure):
     Raises:
       ParseError: when the structure type is unknown.
     """
-    if key != 'logline':
-      raise errors.ParseError(
-          'Unable to parse record, unknown structure: {0:s}'.format(key))
+    if key == 'ignorable_line':
+      return
+
+    if key == 'section_header':
+      self._last_entry_type = self._GetValueFromStructure(
+          structure, 'entry_type')
+      return
+
+    if key == 'section_start':
+      time_structure = self._GetValueFromStructure(structure, 'start_time')
+      start_time = self._GetTimeElements(time_structure)
+      if not start_time:
+        parser_mediator.ProduceExtractionWarning(
+            'invalid date time value: {0!s}'.format(time_structure))
+        return
+
+      event_data = SetupapiLogEventData()
+      event_data.entry_type = self._last_entry_type
 
-    self._ParseRecordLogline(parser_mediator, structure)
+      event = time_events.DateTimeValuesEvent(
+          start_time, definitions.TIME_DESCRIPTION_START,
+          time_zone=parser_mediator.timezone)
+
+      # Create event for the start of the setupapi section
+      parser_mediator.ProduceEventWithEventData(event, event_data)
+      return
 
-  def VerifyStructure(self, parser_mediator, lines):
+    if key == 'section_end':
+      time_structure = self._GetValueFromStructure(structure, 'end_time')
+      end_time = self._GetTimeElements(time_structure)
+      if not end_time:
+        parser_mediator.ProduceExtractionWarning(
+            'invalid date time value: {0!s}'.format(time_structure))
+      # Store last end time so that an event with the data from the
+      # following exit status section can be created.
+      self._last_end_time = end_time
+      return
+
+    if key == 'section_end_exit_status':
+      exit_status = self._GetValueFromStructure(
+          structure, 'exit_status')
+      if self._last_end_time:
+        event_data = SetupapiLogEventData()
+        event_data.entry_type = self._last_entry_type
+        event_data.exit_status = exit_status
+        event = time_events.DateTimeValuesEvent(
+            self._last_end_time, definitions.TIME_DESCRIPTION_END,
+            time_zone=parser_mediator.timezone)
+        parser_mediator.ProduceEventWithEventData(event, event_data)
+        # Reset entry type and status and end time in case a line is missing.
+        self._last_entry_type = None
+        self._last_end_time = None
+        return
+
+    raise errors.ParseError(
+        'Unable to parse record, unknown structure: {0:s}'.format(key))
+
+  def VerifyStructure(self, parser_mediator, line):
     """Verify that this file is a Windows Setupapi log file.
 
     Args:
       parser_mediator (ParserMediator): mediates interactions between parsers
           and other components, such as storage and dfvfs.
-      lines (str): one or more lines from the text file.
+      line (str): single line from the text file.
 
     Returns:
       bool: True if this is the correct parser, False otherwise.
     """
     try:
-      structure = self._SETUPAPI_LINE.parseString(lines)
+      self._LOG_HEADER_START.parseString(line)
+      # Reset stored values for parsing a new file.
+      self._last_end_time = None
+      self._last_entry_type = None
     except pyparsing.ParseException as exception:
       logger.debug('Not a Windows Setupapi log file: {0!s}'.format(exception))
       return False
 
-    time_elements_structure = self._GetValueFromStructure(
-        structure, 'start_time')
-
-    try:
-      date_time = dfdatetime_time_elements.TimeElementsInMilliseconds(
-          time_elements_tuple=time_elements_structure)
-    except ValueError as exception:
-      logger.debug((
-          'Not a Windows Setupapi log file, invalid date/time: {0!s} '
-          'with error: {1!s}').format(time_elements_structure, exception))
-      return False
-
-    if not date_time:
-      logger.debug((
-          'Not a Windows Setupapi log file, '
-          'invalid date/time: {0!s}').format(time_elements_structure))
-      return False
-
     return True
 
 

plaso/parsers/text_parser.py

@@ -358,7 +358,7 @@ def _ReadLine(self, text_file_object, max_len=None, depth=0):
 
       return self._ReadLine(text_file_object, max_len=max_len, depth=depth + 1)
 
-    return line.strip()
+    return line
 
   def ParseFileObject(self, parser_mediator, file_object):
     """Parses a text file-like object using a pyparsing definition.

plaso/parsers/winfirewall.py

@@ -224,7 +224,8 @@ def VerifyStructure(self, parser_mediator, line):
     """
     # TODO: Examine other versions of the file format and if this parser should
     # support them.
-    return line == '#Version: 1.5'
+    stripped_line = line.rstrip()
+    return stripped_line == '#Version: 1.5'
 
 
 manager.ParsersManager.RegisterParser(WinFirewallParser)