From e22155f9ecdd60fc7c1e82a30a207fe4e5421fa5 Mon Sep 17 00:00:00 2001 From: Daniel White Date: Mon, 2 Sep 2019 16:58:47 +0200 Subject: [PATCH] Improved and refactored setupapi parser #2717 (#2718) --- data/presets.yaml | 1 + plaso/formatters/setupapi.py | 4 +- plaso/parsers/iis.py | 1 + plaso/parsers/mac_wifi.py | 10 +- plaso/parsers/setupapi.py | 251 ++++++++++++++++++++++------------- plaso/parsers/text_parser.py | 2 +- plaso/parsers/winfirewall.py | 3 +- tests/formatters/setupapi.py | 2 +- tests/parsers/setupapi.py | 28 ++-- 9 files changed, 189 insertions(+), 113 deletions(-) diff --git a/data/presets.yaml b/data/presets.yaml index e854209f1c..40f50a7225 100644 --- a/data/presets.yaml +++ b/data/presets.yaml @@ -113,6 +113,7 @@ parsers: - olecf - pe - prefetch +- setupapi - sccm - skydrive_log - skydrive_log_old diff --git a/plaso/formatters/setupapi.py b/plaso/formatters/setupapi.py index 4b66553db2..7be5d1f2a2 100644 --- a/plaso/formatters/setupapi.py +++ b/plaso/formatters/setupapi.py @@ -14,11 +14,11 @@ class SetupapiLogFormatter(interface.ConditionalEventFormatter): FORMAT_STRING_PIECES = [ '{entry_type}', - '{entry_status}'] + '{exit_status}'] # Reversing fields for short description to prevent truncating the status FORMAT_STRING_SHORT_PIECES = [ - '{entry_status}', + '{exit_status}', '{entry_type}'] FORMAT_STRING_SEPARATOR = ' - ' diff --git a/plaso/parsers/iis.py b/plaso/parsers/iis.py index 38d8fb01d0..2f29a126b1 100644 --- a/plaso/parsers/iis.py +++ b/plaso/parsers/iis.py @@ -179,6 +179,7 @@ def _ParseFieldsMetadata(self, structure): structure (pyparsing.ParseResults): structure parsed from the log file. """ fields = self._GetValueFromStructure(structure, 'fields', default_value='') + fields = fields.strip() fields = fields.split(' ') log_line_structure = pyparsing.Empty() diff --git a/plaso/parsers/mac_wifi.py b/plaso/parsers/mac_wifi.py index a77dc168d6..99541bbd9e 100644 --- a/plaso/parsers/mac_wifi.py +++ b/plaso/parsers/mac_wifi.py @@ -228,15 +228,15 @@ def _ParseLogLine(self, parser_mediator, key, structure): self._last_month = time_elements_tuple[1] function = self._GetValueFromStructure(structure, 'function') - if function: - # Due to the use of CharsNotIn the function value can contain leading - # or trailing whitespace. - function = function.strip() + + text = self._GetValueFromStructure(structure, 'text') + if text: + text = text.strip() event_data = MacWifiLogEventData() event_data.agent = self._GetValueFromStructure(structure, 'agent') event_data.function = function - event_data.text = self._GetValueFromStructure(structure, 'text') + event_data.text = text if key == 'known_function_logline': event_data.action = self._GetAction( diff --git a/plaso/parsers/setupapi.py b/plaso/parsers/setupapi.py index 6dbc7dee01..1120b32d92 100644 --- a/plaso/parsers/setupapi.py +++ b/plaso/parsers/setupapi.py @@ -1,5 +1,9 @@ # -*- coding: utf-8 -*- -"""Parser for Windows Setupapi log files.""" +"""Parser for Windows Setupapi log files. + +The format is documented at: +https://docs.microsoft.com/en-us/windows-hardware/drivers/install/setupapi-text-logs +""" from __future__ import unicode_literals @@ -20,8 +24,10 @@ class SetupapiLogEventData(events.EventData): """Setupapi log event data. Attributes: - entry_type (str): log entry type such as "Device Install". - entry_status (str): the status of the entry. + entry_type (str): log entry type, for examaple "Device Install - + PCI\\VEN_104C&DEV_8019&SUBSYS_8010104C&REV_00\\3&61aaa01&0&38" or + "Sysprep Respecialize - {804b345a-ffd7-854c-a1b5-ca9598907846}". + exit_status (str): the exit status of the logged operation. """ DATA_TYPE = 'setupapi:log:line' @@ -30,11 +36,10 @@ def __init__(self): """Initializes event data.""" super(SetupapiLogEventData, self).__init__(data_type=self.DATA_TYPE) self.entry_type = None - # TODO: Parse additional fields from the body of setupapi messages - self.entry_status = None + self.exit_status = None -class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser): +class SetupapiLogParser(text_parser.PyparsingSingleLineTextParser): """Parses events from Windows Setupapi log files.""" NAME = 'setupapi' @@ -43,9 +48,6 @@ class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser): _ENCODING = 'utf-8' - # Increase the buffer size, as log messages can be very long. - BUFFER_SIZE = 262144 - _SLASH = pyparsing.Literal('/').suppress() _FOUR_DIGITS = text_parser.PyparsingConstants.FOUR_DIGITS @@ -63,74 +65,109 @@ class SetupapiLogParser(text_parser.PyparsingMultiLineTextParser): _THREE_DIGITS ) - _SETUPAPI_LINE = ( - pyparsing.SkipTo('>>> [', include=True).suppress() + - pyparsing.SkipTo(']').setResultsName('entry_type') + - pyparsing.SkipTo('>>> Section start', include=True).suppress() + + # Disable pylint due to long URLs for documenting structures. + # pylint: disable=line-too-long + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-header + _LOG_HEADER_START = ( + pyparsing.Literal('[Device Install Log]') + + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-header + _LOG_HEADER_END = ( + pyparsing.Literal('[BeginLog]') + + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-header + _SECTION_HEADER = ( + pyparsing.Literal('>>> [').suppress() + + pyparsing.CharsNotIn(']').setResultsName('entry_type') + + pyparsing.Literal(']') + + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-header + _SECTION_HEADER_START = ( + pyparsing.Literal('>>> Section start').suppress() + _SETUPAPI_DATE_TIME.setResultsName('start_time') + - pyparsing.SkipTo('<<< Section end ').setResultsName('message') + - pyparsing.GoToColumn(17) + + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-footer + _SECTION_END = ( + pyparsing.Literal('<<< Section end ').suppress() + _SETUPAPI_DATE_TIME.setResultsName('end_time') + - pyparsing.SkipTo('<<< [Exit status: ', include=True).suppress() + - pyparsing.SkipTo(']').setResultsName('entry_status') + - pyparsing.SkipTo(pyparsing.lineEnd()) + - pyparsing.ZeroOrMore(pyparsing.lineEnd())) + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-a-text-log-section-footer + _SECTION_END_EXIT_STATUS = ( + pyparsing.Literal('<<< [Exit status: ').suppress() + + pyparsing.CharsNotIn(']').setResultsName('exit_status') + + pyparsing.Literal(']') + + pyparsing.lineEnd()) + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-log-entries-that-are-not-part-of-a-text-log-section + _SECTION_BODY_LINE = ( + pyparsing.stringStart + + pyparsing.MatchFirst([ + pyparsing.Literal('!!! '), + pyparsing.Literal('! '), + pyparsing.Literal(' ')]) + + pyparsing.restOfLine).leaveWhitespace() + + # See https://docs.microsoft.com/en-us/windows-hardware/drivers/install/format-of-log-entries-that-are-not-part-of-a-text-log-section + _NON_SECTION_LINE = ( + pyparsing.stringStart + + pyparsing.MatchFirst([ + pyparsing.Literal(' . '), + pyparsing.Literal('!!! '), + pyparsing.Literal('! '), + pyparsing.Literal(' ')]) + + pyparsing.restOfLine).leaveWhitespace() + + # These lines do not appear to be documented in the Microsoft documentation. + _BOOT_SESSION_LINE = ( + pyparsing.Literal('[Boot Session:') + + _SETUPAPI_DATE_TIME + + pyparsing.Literal(']')) + + # pylint: enable=line-too-long LINE_STRUCTURES = [ - ('logline', _SETUPAPI_LINE), - ] + ('ignorable_line', _BOOT_SESSION_LINE), + ('ignorable_line', _LOG_HEADER_END), + ('ignorable_line', _LOG_HEADER_START), + ('ignorable_line', _NON_SECTION_LINE), + ('ignorable_line', _SECTION_BODY_LINE), + ('section_end', _SECTION_END), + ('section_end_exit_status', _SECTION_END_EXIT_STATUS), + ('section_header', _SECTION_HEADER), + ('section_start', _SECTION_HEADER_START)] + + def __init__(self): + """Initializes a setupapi parser.""" + super(SetupapiLogParser, self).__init__() + self._last_end_time = None + self._last_entry_type = None - def _ParseRecordLogline(self, parser_mediator, structure): - """Parses a logline record structure and produces events. + def _GetTimeElements(self, time_structure): + """Builds time elements from a setupapi time_stamp field. Args: - parser_mediator (ParserMediator): mediates interactions between parsers - and other components, such as storage and dfvfs. - structure (pyparsing.ParseResults): structure of tokens derived from - log entry. + time_structure (pyparsing.ParseResults): structure of tokens derived from + a setupapi time_stamp field. + + Returns: + dfdatetime.TimeElements: date and time extracted from the value or None + if the structure does not represent a valid date and time value. """ - time_zone = parser_mediator.timezone - time_elements_structure = self._GetValueFromStructure( - structure, 'start_time') try: date_time = dfdatetime_time_elements.TimeElementsInMilliseconds( - time_elements_tuple=time_elements_structure) - # Setupapi logs stores date and time values in local time. + time_elements_tuple=time_structure) + # Setupapi logs store date and time values in local time. date_time.is_local_time = True - except ValueError: - parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_structure)) - return - - event_data = SetupapiLogEventData() - event_data.entry_type = self._GetValueFromStructure(structure, 'entry_type') - event_data.entry_status = 'START' - - event = time_events.DateTimeValuesEvent( - date_time, definitions.TIME_DESCRIPTION_START, time_zone=time_zone) - - # Create event for the start of the setupapi section - parser_mediator.ProduceEventWithEventData(event, event_data) + return date_time - event_data.entry_status = self._GetValueFromStructure( - structure, 'entry_status') - - time_elements_structure = self._GetValueFromStructure( - structure, 'end_time') - try: - date_time = dfdatetime_time_elements.TimeElementsInMilliseconds( - time_elements_tuple=time_elements_structure) - date_time.is_local_time = True except ValueError: - parser_mediator.ProduceExtractionWarning( - 'invalid date time value: {0!s}'.format(time_elements_structure)) - return - - event = time_events.DateTimeValuesEvent( - date_time, definitions.TIME_DESCRIPTION_END, time_zone=time_zone) - - # Create event for the end of the setupapi section - parser_mediator.ProduceEventWithEventData(event, event_data) + return None def ParseRecord(self, parser_mediator, key, structure): """Parses a log record structure and produces events. @@ -145,47 +182,83 @@ def ParseRecord(self, parser_mediator, key, structure): Raises: ParseError: when the structure type is unknown. """ - if key != 'logline': - raise errors.ParseError( - 'Unable to parse record, unknown structure: {0:s}'.format(key)) + if key == 'ignorable_line': + return + + if key == 'section_header': + self._last_entry_type = self._GetValueFromStructure( + structure, 'entry_type') + return + + if key == 'section_start': + time_structure = self._GetValueFromStructure(structure, 'start_time') + start_time = self._GetTimeElements(time_structure) + if not start_time: + parser_mediator.ProduceExtractionWarning( + 'invalid date time value: {0!s}'.format(time_structure)) + return + + event_data = SetupapiLogEventData() + event_data.entry_type = self._last_entry_type - self._ParseRecordLogline(parser_mediator, structure) + event = time_events.DateTimeValuesEvent( + start_time, definitions.TIME_DESCRIPTION_START, + time_zone=parser_mediator.timezone) + + # Create event for the start of the setupapi section + parser_mediator.ProduceEventWithEventData(event, event_data) + return - def VerifyStructure(self, parser_mediator, lines): + if key == 'section_end': + time_structure = self._GetValueFromStructure(structure, 'end_time') + end_time = self._GetTimeElements(time_structure) + if not end_time: + parser_mediator.ProduceExtractionWarning( + 'invalid date time value: {0!s}'.format(time_structure)) + # Store last end time so that an event with the data from the + # following exit status section can be created. + self._last_end_time = end_time + return + + if key == 'section_end_exit_status': + exit_status = self._GetValueFromStructure( + structure, 'exit_status') + if self._last_end_time: + event_data = SetupapiLogEventData() + event_data.entry_type = self._last_entry_type + event_data.exit_status = exit_status + event = time_events.DateTimeValuesEvent( + self._last_end_time, definitions.TIME_DESCRIPTION_END, + time_zone=parser_mediator.timezone) + parser_mediator.ProduceEventWithEventData(event, event_data) + # Reset entry type and status and end time in case a line is missing. + self._last_entry_type = None + self._last_end_time = None + return + + raise errors.ParseError( + 'Unable to parse record, unknown structure: {0:s}'.format(key)) + + def VerifyStructure(self, parser_mediator, line): """Verify that this file is a Windows Setupapi log file. Args: parser_mediator (ParserMediator): mediates interactions between parsers and other components, such as storage and dfvfs. - lines (str): one or more lines from the text file. + line (str): single line from the text file. Returns: bool: True if this is the correct parser, False otherwise. """ try: - structure = self._SETUPAPI_LINE.parseString(lines) + self._LOG_HEADER_START.parseString(line) + # Reset stored values for parsing a new file. + self._last_end_time = None + self._last_entry_type = None except pyparsing.ParseException as exception: logger.debug('Not a Windows Setupapi log file: {0!s}'.format(exception)) return False - time_elements_structure = self._GetValueFromStructure( - structure, 'start_time') - - try: - date_time = dfdatetime_time_elements.TimeElementsInMilliseconds( - time_elements_tuple=time_elements_structure) - except ValueError as exception: - logger.debug(( - 'Not a Windows Setupapi log file, invalid date/time: {0!s} ' - 'with error: {1!s}').format(time_elements_structure, exception)) - return False - - if not date_time: - logger.debug(( - 'Not a Windows Setupapi log file, ' - 'invalid date/time: {0!s}').format(time_elements_structure)) - return False - return True diff --git a/plaso/parsers/text_parser.py b/plaso/parsers/text_parser.py index e0157235d7..f6e7da4d51 100644 --- a/plaso/parsers/text_parser.py +++ b/plaso/parsers/text_parser.py @@ -358,7 +358,7 @@ def _ReadLine(self, text_file_object, max_len=None, depth=0): return self._ReadLine(text_file_object, max_len=max_len, depth=depth + 1) - return line.strip() + return line def ParseFileObject(self, parser_mediator, file_object): """Parses a text file-like object using a pyparsing definition. diff --git a/plaso/parsers/winfirewall.py b/plaso/parsers/winfirewall.py index b5bfc36dcf..4d0215b079 100644 --- a/plaso/parsers/winfirewall.py +++ b/plaso/parsers/winfirewall.py @@ -224,7 +224,8 @@ def VerifyStructure(self, parser_mediator, line): """ # TODO: Examine other versions of the file format and if this parser should # support them. - return line == '#Version: 1.5' + stripped_line = line.rstrip() + return stripped_line == '#Version: 1.5' manager.ParsersManager.RegisterParser(WinFirewallParser) diff --git a/tests/formatters/setupapi.py b/tests/formatters/setupapi.py index d2c0d507f5..c64b826daf 100644 --- a/tests/formatters/setupapi.py +++ b/tests/formatters/setupapi.py @@ -25,7 +25,7 @@ def testGetFormatStringAttributeNames(self): expected_attribute_names = [ 'entry_type', - 'entry_status'] + 'exit_status'] self._TestGetFormatStringAttributeNames( event_formatter, expected_attribute_names) diff --git a/tests/parsers/setupapi.py b/tests/parsers/setupapi.py index 8411e062c1..1bebfbca66 100644 --- a/tests/parsers/setupapi.py +++ b/tests/parsers/setupapi.py @@ -44,10 +44,10 @@ def testParseDevLog(self): expected_message = ( 'Device Install (Hardware initiated) - SWD\\IP_TUNNEL_VBUS' - '\\Teredo_Tunnel_Device - START') + '\\Teredo_Tunnel_Device') expected_short_message = ( - 'START - Device Install (Hardware initiated) - SWD\\IP_TUNNEL_VBUS' - '\\Teredo_Tunne...') + 'Device Install (Hardware initiated) - SWD\\IP_TUNNEL_VBUS' + '\\Teredo_Tunnel_Device') self._TestGetMessageStrings( event_data, expected_message, expected_short_message) @@ -71,10 +71,10 @@ def testParseDevLog(self): expected_message = ( 'Device Install (Hardware initiated) - SWD\\WPDBUSENUM' '\\_??_USBSTOR#Disk&Ven_Generic&Prod_Flash_Disk&Rev_8.07#99E2116A&0' - '#{53f56307-b6bf-11d0-94f2-00a0c91efb8b} - START') + '#{53f56307-b6bf-11d0-94f2-00a0c91efb8b}') expected_short_message = ( - 'START - Device Install (Hardware initiated) - SWD\\WPDBUSENUM' - '\\_??_USBSTOR#Disk...') + 'Device Install (Hardware initiated) - SWD\\WPDBUSENUM' + '\\_??_USBSTOR#Disk&Ven_Gen...') self._TestGetMessageStrings( event_data, expected_message, expected_short_message) @@ -101,8 +101,8 @@ def testParseSetupLog(self): self.CheckTimestamp(event.timestamp, '2015-11-22 17:53:29.305000') - expected_message = ('Setup Plug and Play Device Install - START') - expected_short_message = ('START - Setup Plug and Play Device Install') + expected_message = 'Setup Plug and Play Device Install' + expected_short_message = 'Setup Plug and Play Device Install' self._TestGetMessageStrings( event_data, expected_message, expected_short_message) @@ -112,10 +112,10 @@ def testParseSetupLog(self): expected_message = ( 'Setup online Device Install (Hardware initiated) - SW' '\\{97ebaacc-95bd-11d0-a3ea-00a0c9223196}' - '\\{53172480-4791-11D0-A5D6-28DB04C10000} - START') + '\\{53172480-4791-11D0-A5D6-28DB04C10000}') expected_short_message = ( - 'START - Setup online Device Install (Hardware initiated) - SW' - '\\{97ebaacc-95bd-...') + 'Setup online Device Install (Hardware initiated) - SW' + '\\{97ebaacc-95bd-11d0-a3e...') self._TestGetMessageStrings( event_data, expected_message, expected_short_message) @@ -126,10 +126,10 @@ def testParseSetupLog(self): expected_message = ( 'Setup Import Driver Package - C:\\Windows\\system32' - '\\spool\\tools\\Microsoft XPS Document Writer\\prnms001.Inf - START') + '\\spool\\tools\\Microsoft XPS Document Writer\\prnms001.Inf') expected_short_message = ( - 'START - Setup Import Driver Package - C:\\Windows\\system32\\spool' - '\\tools\\Microso...') + 'Setup Import Driver Package - C:\\Windows\\system32\\spool' + '\\tools\\Microsoft XPS D...') self._TestGetMessageStrings( event_data, expected_message, expected_short_message)