diff --git a/event_tracker/cred_extractor/__init__.py b/event_tracker/cred_extractor/__init__.py index 6484788..aeee6da 100644 --- a/event_tracker/cred_extractor/__init__.py +++ b/event_tracker/cred_extractor/__init__.py @@ -8,10 +8,13 @@ class CredentialExtractor(ABC): """ @abstractmethod - def extract(self, input_text: str, default_system: str) -> [Credential]: + def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]): """ - Returns a list of Credential objects found in the text. - Objects will be added to the DB if required _by the caller_. + Returns a tuple of: + a list of new Credential objects found in the text to add to the database, + a list of existing Credential objects superceeded as a result of the first list which should be deleted. + + Objects will be added and removed to/from the DB if required _by the caller_. """ pass @@ -25,8 +28,8 @@ class CredentialExtractorGenerator(CredentialExtractor, ABC): """ Special type of CredentialExtractor which uses a generator method """ - def extract(self, input_text: str, default_system: str) -> [Credential]: - return list(self.cred_generator(input_text, default_system)) + def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]): + return list(self.cred_generator(input_text, default_system)), list() @abstractmethod def cred_generator(self, input_text: str, default_system: str) -> Credential: diff --git a/event_tracker/cred_extractor/extractor.py b/event_tracker/cred_extractor/extractor.py index 895d9a0..c3cda22 100644 --- a/event_tracker/cred_extractor/extractor.py +++ b/event_tracker/cred_extractor/extractor.py @@ -34,11 +34,11 @@ @transaction.atomic def extract_and_save(input_text: str, default_system: str) -> tuple[int, int]: - credentials = extract(input_text, default_system) + credentials_to_add, credentials_to_delete = extract(input_text, default_system) saved_secrets = 0 creds_to_add_in_bulk = [] - for cred in credentials: + for cred in credentials_to_add: if cred.secret: # post-save action should be called, as we have a secret # use keys_to_save as a pseudo-uniqueness constraint for this write operation @@ -55,15 +55,23 @@ def extract_and_save(input_text: str, default_system: str) -> tuple[int, int]: Credential.objects.bulk_create(creds_to_add_in_bulk, ignore_conflicts=True, unique_fields=["hash", "hash_type", "account", "system"]) + for obj in credentials_to_delete: + obj.delete() + return Credential.objects.count() - pre_insert_count, saved_secrets -def extract(input_text: str, default_system: str) -> [Credential]: - credentials = [] +def extract(input_text: str, default_system: str) -> ([Credential], [Credential]): + credentials_to_add = [] + credentials_to_remove = [] functions = [subclass().extract for subclass in extractor_classes] futures = [] for function in functions: futures.append(executor.submit(function, input_text, default_system)) for future in concurrent.futures.as_completed(futures): - credentials += future.result() - return credentials + result = future.result() + if result is not None: + to_add, to_remove = result + credentials_to_add += to_add + credentials_to_remove += to_remove + return credentials_to_add, credentials_to_remove diff --git a/event_tracker/cred_extractor/rubeus_extractor.py b/event_tracker/cred_extractor/rubeus_extractor.py index 77a9893..581841d 100644 --- a/event_tracker/cred_extractor/rubeus_extractor.py +++ b/event_tracker/cred_extractor/rubeus_extractor.py @@ -3,7 +3,7 @@ from django.db.models import CharField, Value from django.db.models.functions import Length, StrIndex -from event_tracker.cred_extractor import CredentialExtractorGenerator +from event_tracker.cred_extractor import CredentialExtractorGenerator, CredentialExtractor from event_tracker.cred_extractor.kerberoast_extractor import convert_tgs_to_hashcat_format from event_tracker.models import Credential, HashCatMode @@ -22,8 +22,10 @@ def cred_generator(self, input_text: str, default_system: str): purpose="Windows Login", source="Rubeus U2U") -class RubeusKerberoastExtractor(CredentialExtractorGenerator): - def cred_generator(self, input_text: str, default_system: str): +class RubeusKerberoastExtractor(CredentialExtractor): + def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]): + credentials_to_add = [] + for match in rubeus_kerberoast_regex.finditer(input_text): hash_str = match.groupdict()["hash"].replace(" ", "").replace("\n", "").replace("\r", "") @@ -36,21 +38,22 @@ def cred_generator(self, input_text: str, default_system: str): elif hash_str.startswith("$krb5tgs$17$"): hash_type = 19600 - yield Credential(hash=hash_str, account=match.groupdict()["account"], + credentials_to_add.append(Credential(hash=hash_str, account=match.groupdict()["account"], hash_type=hash_type, system=match.groupdict()["system"] or default_system, purpose=f"Windows Login (used by SPN: {match.groupdict()['purpose'].strip()})", - source="Rubeus Kerberoasting") + source="Rubeus Kerberoasting")) # Remove any similar but truncated hashes which haven't cracked, these are a result of stream processing kicking # in before the multiline kerberos ticket has been fully parsed from CS logs CharField.register_lookup(Length) - Credential.objects.filter(account=match.groupdict()["account"], + credentials_to_remove = list(Credential.objects.filter(account=match.groupdict()["account"], hash_type=hash_type, system=match.groupdict()["system"] or default_system, purpose=f"Windows Login (used by SPN: {match.groupdict()['purpose'].strip()})", source="Rubeus Kerberoasting") \ .filter(hash__length__lt=len(hash_str), secret__isnull=True) \ - .annotate(stri=StrIndex(Value(hash_str), "hash")).filter(stri=1) \ - .delete() + .annotate(stri=StrIndex(Value(hash_str), "hash")).filter(stri=1)) + + return credentials_to_add, credentials_to_remove class RubeusASREPRoastExtractor(CredentialExtractorGenerator): diff --git a/event_tracker/cred_extractor/snaffler_extractor.py b/event_tracker/cred_extractor/snaffler_extractor.py index 11c6c85..0337a61 100644 --- a/event_tracker/cred_extractor/snaffler_extractor.py +++ b/event_tracker/cred_extractor/snaffler_extractor.py @@ -10,8 +10,9 @@ db_connection_string = re.compile(r'(?=.*Password=)(;?\s*User ID=(?P[^;<>\"]+)|;?\s*Password=(?P[^;<>\"]+)|;?\s*(Data Source|Server)=(?P[^;<>\"]+)|;?[^;<>\"]+)+', re.IGNORECASE) # Similar to above, but embedded in XML, so switch quotes to angle brackets websense_client_password = re.compile(r'WDEUtil[^\n]+-password +(?P\S+)', re.IGNORECASE) + class SnafflerExtractor(CredentialExtractor): - def extract(self, input_text: str, default_system: str) -> [Credential]: + def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]): result = [] for match in snaffler_finding.finditer(input_text): @@ -66,7 +67,7 @@ def extract(self, input_text: str, default_system: str) -> [Credential]: source_time=match['ainfo'].split('|')[-1], purpose='Websense Client Password')) - return result + return result, [] def unescape_content(self, match): content = re.sub(r'\\r', r'\r', match["cinfo"]) diff --git a/event_tracker/cred_extractor/test_extractors.py b/event_tracker/cred_extractor/test_extractors.py index 11850eb..058051f 100644 --- a/event_tracker/cred_extractor/test_extractors.py +++ b/event_tracker/cred_extractor/test_extractors.py @@ -7,7 +7,7 @@ class ExtractorTestCase(django.test.SimpleTestCase): def test_netntlmv1(self): # Hash from hashcat sample hashes - result = extract( + result, _ = extract( "u4-netntlm::kNS:338d08f8e26de93300000000000000000000000000000000:9526fb8c23a90751cdd619b6cea564742e1e4bf33006ba41:cb8086049ec4736c", "DUMMY") @@ -18,7 +18,7 @@ def test_netntlmv1(self): def test_netntlmv2(self): # Hash from hashcat sample hashes - result = extract( + result, _ = extract( "admin::N46iSNekpT:08ca45b7d7ea58ee:88dcbe4446168966a153a0064958dac6:5c7830315c7830310000000000000b45c67103d07d7b95acd12ffa11230e0000000052920b85f78d013c31cdb3b92f5d765c783030", "DUMMY") @@ -30,7 +30,7 @@ def test_netntlmv2(self): def test_dcc2(self): # Hash from hashcat sample hashes, embedded in secrets dump output - result = extract( + result, _ = extract( """ [*] Dumping cached domain logon information (domain/username:hash) DOMAIN.COMPANY.COM/tom:$DCC2$10240#tom#e4e938d12fe5974dc42a90120bd9c90f: (2024-10-17 12:31:41) @@ -44,7 +44,7 @@ def test_dcc2(self): def test_secretsdump_local_users_datestamped(self): # Hash from hashcat sample hashes, embedded in secrets dump output - result = extract( + result, _ = extract( """ [2024-10-16 16:12:05] [*] Administrator:500:aad3b435b51404eeaad3b435b51404ee:0123456789abcdef0123456789abcdef::: [2024-10-16 16:12:07] [*] Guest:501:aad3b435b51404eeaad3b435b51404ee:31d6cfe0d16ae931b73c59d7e0c089c0::: @@ -62,7 +62,7 @@ def test_secretsdump_local_users_datestamped(self): def test_secretsdump_machine_account(self): # Hash from hashcat sample hashes, embedded in secrets dump output - result = extract(r""" + result, _ = extract(r""" [2024-10-16 16:12:23] [*] DOMAIN\HOST$:aad3b435b51404eeaad3b435b51404ee:0123456789abcdef0123456789abcdef::: """, "DUMMY") @@ -73,7 +73,7 @@ def test_secretsdump_machine_account(self): def test_sharpsccm_naa(self): # Hash from hashcat sample hashes, embedded in secrets dump output - result = extract(r""" + result, _ = extract(r""" [+] Decrypting network access account credentials NetworkAccessUsername: DOMAIN\USER diff --git a/event_tracker/cred_extractor/test_extractors_db.py b/event_tracker/cred_extractor/test_extractors_db.py index 8eceb76..0c31009 100644 --- a/event_tracker/cred_extractor/test_extractors_db.py +++ b/event_tracker/cred_extractor/test_extractors_db.py @@ -8,7 +8,7 @@ class ExtractorTestCaseWithDB(django.test.TestCase): def test_rubeus_kerberoast_crlf(self): # Hash from HashCat example hashes - result = extract(""" + result, _ = extract(""" [*] SamAccountName : USER\r [*] DistinguishedName : CN=USER,OU=Service Accounts,OU=Non-Personal,OU=Accounts,OU=Brand,DC=domain,DC=local\r [*] ServicePrincipalName : test/spn\r @@ -28,7 +28,7 @@ def test_rubeus_kerberoast_crlf(self): def test_rubeus_kerberoast_lf(self): # Hash from HashCat example hashes - result = extract(""" + result, _ = extract(""" [*] SamAccountName : USER [*] DistinguishedName : CN=USER,OU=Service Accounts,OU=Non-Personal,OU=Accounts,OU=Brand,DC=domain,DC=local [*] ServicePrincipalName : test/spn