Skip to content

Commit

Permalink
Fixed "database locked" issue with rubeus credential extractor caused…
Browse files Browse the repository at this point in the history
… by trying to delete objects from a thread outside the current transaction. Now extractors can list objects they want to be deleted as well as added.
  • Loading branch information
neonbunny committed Dec 12, 2024
1 parent d529076 commit 6c70396
Show file tree
Hide file tree
Showing 6 changed files with 44 additions and 29 deletions.
13 changes: 8 additions & 5 deletions event_tracker/cred_extractor/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@ class CredentialExtractor(ABC):
"""

@abstractmethod
def extract(self, input_text: str, default_system: str) -> [Credential]:
def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]):
"""
Returns a list of Credential objects found in the text.
Objects will be added to the DB if required _by the caller_.
Returns a tuple of:
a list of new Credential objects found in the text to add to the database,
a list of existing Credential objects superceeded as a result of the first list which should be deleted.
Objects will be added and removed to/from the DB if required _by the caller_.
"""
pass

Expand All @@ -25,8 +28,8 @@ class CredentialExtractorGenerator(CredentialExtractor, ABC):
"""
Special type of CredentialExtractor which uses a generator method
"""
def extract(self, input_text: str, default_system: str) -> [Credential]:
return list(self.cred_generator(input_text, default_system))
def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]):
return list(self.cred_generator(input_text, default_system)), list()

@abstractmethod
def cred_generator(self, input_text: str, default_system: str) -> Credential:
Expand Down
20 changes: 14 additions & 6 deletions event_tracker/cred_extractor/extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@

@transaction.atomic
def extract_and_save(input_text: str, default_system: str) -> tuple[int, int]:
credentials = extract(input_text, default_system)
credentials_to_add, credentials_to_delete = extract(input_text, default_system)
saved_secrets = 0

creds_to_add_in_bulk = []
for cred in credentials:
for cred in credentials_to_add:
if cred.secret:
# post-save action should be called, as we have a secret
# use keys_to_save as a pseudo-uniqueness constraint for this write operation
Expand All @@ -55,15 +55,23 @@ def extract_and_save(input_text: str, default_system: str) -> tuple[int, int]:
Credential.objects.bulk_create(creds_to_add_in_bulk, ignore_conflicts=True,
unique_fields=["hash", "hash_type", "account", "system"])

for obj in credentials_to_delete:
obj.delete()

return Credential.objects.count() - pre_insert_count, saved_secrets


def extract(input_text: str, default_system: str) -> [Credential]:
credentials = []
def extract(input_text: str, default_system: str) -> ([Credential], [Credential]):
credentials_to_add = []
credentials_to_remove = []
functions = [subclass().extract for subclass in extractor_classes]
futures = []
for function in functions:
futures.append(executor.submit(function, input_text, default_system))
for future in concurrent.futures.as_completed(futures):
credentials += future.result()
return credentials
result = future.result()
if result is not None:
to_add, to_remove = result
credentials_to_add += to_add
credentials_to_remove += to_remove
return credentials_to_add, credentials_to_remove
19 changes: 11 additions & 8 deletions event_tracker/cred_extractor/rubeus_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from django.db.models import CharField, Value
from django.db.models.functions import Length, StrIndex

from event_tracker.cred_extractor import CredentialExtractorGenerator
from event_tracker.cred_extractor import CredentialExtractorGenerator, CredentialExtractor
from event_tracker.cred_extractor.kerberoast_extractor import convert_tgs_to_hashcat_format
from event_tracker.models import Credential, HashCatMode

Expand All @@ -22,8 +22,10 @@ def cred_generator(self, input_text: str, default_system: str):
purpose="Windows Login", source="Rubeus U2U")


class RubeusKerberoastExtractor(CredentialExtractorGenerator):
def cred_generator(self, input_text: str, default_system: str):
class RubeusKerberoastExtractor(CredentialExtractor):
def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]):
credentials_to_add = []

for match in rubeus_kerberoast_regex.finditer(input_text):
hash_str = match.groupdict()["hash"].replace(" ", "").replace("\n", "").replace("\r", "")

Expand All @@ -36,21 +38,22 @@ def cred_generator(self, input_text: str, default_system: str):
elif hash_str.startswith("$krb5tgs$17$"):
hash_type = 19600

yield Credential(hash=hash_str, account=match.groupdict()["account"],
credentials_to_add.append(Credential(hash=hash_str, account=match.groupdict()["account"],
hash_type=hash_type, system=match.groupdict()["system"] or default_system,
purpose=f"Windows Login (used by SPN: {match.groupdict()['purpose'].strip()})",
source="Rubeus Kerberoasting")
source="Rubeus Kerberoasting"))

# Remove any similar but truncated hashes which haven't cracked, these are a result of stream processing kicking
# in before the multiline kerberos ticket has been fully parsed from CS logs
CharField.register_lookup(Length)
Credential.objects.filter(account=match.groupdict()["account"],
credentials_to_remove = list(Credential.objects.filter(account=match.groupdict()["account"],
hash_type=hash_type, system=match.groupdict()["system"] or default_system,
purpose=f"Windows Login (used by SPN: {match.groupdict()['purpose'].strip()})",
source="Rubeus Kerberoasting") \
.filter(hash__length__lt=len(hash_str), secret__isnull=True) \
.annotate(stri=StrIndex(Value(hash_str), "hash")).filter(stri=1) \
.delete()
.annotate(stri=StrIndex(Value(hash_str), "hash")).filter(stri=1))

return credentials_to_add, credentials_to_remove


class RubeusASREPRoastExtractor(CredentialExtractorGenerator):
Expand Down
5 changes: 3 additions & 2 deletions event_tracker/cred_extractor/snaffler_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
db_connection_string = re.compile(r'(?=.*Password=)(;?\s*User ID=(?P<account>[^;<>\"]+)|;?\s*Password=(?P<secret>[^;<>\"]+)|;?\s*(Data Source|Server)=(?P<system>[^;<>\"]+)|;?[^;<>\"]+)+', re.IGNORECASE) # Similar to above, but embedded in XML, so switch quotes to angle brackets
websense_client_password = re.compile(r'WDEUtil[^\n]+-password +(?P<secret>\S+)', re.IGNORECASE)


class SnafflerExtractor(CredentialExtractor):
def extract(self, input_text: str, default_system: str) -> [Credential]:
def extract(self, input_text: str, default_system: str) -> ([Credential], [Credential]):
result = []

for match in snaffler_finding.finditer(input_text):
Expand Down Expand Up @@ -66,7 +67,7 @@ def extract(self, input_text: str, default_system: str) -> [Credential]:
source_time=match['ainfo'].split('|')[-1],
purpose='Websense Client Password'))

return result
return result, []

def unescape_content(self, match):
content = re.sub(r'\\r', r'\r', match["cinfo"])
Expand Down
12 changes: 6 additions & 6 deletions event_tracker/cred_extractor/test_extractors.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
class ExtractorTestCase(django.test.SimpleTestCase):
def test_netntlmv1(self):
# Hash from hashcat sample hashes
result = extract(
result, _ = extract(
"u4-netntlm::kNS:338d08f8e26de93300000000000000000000000000000000:9526fb8c23a90751cdd619b6cea564742e1e4bf33006ba41:cb8086049ec4736c",
"DUMMY")

Expand All @@ -18,7 +18,7 @@ def test_netntlmv1(self):

def test_netntlmv2(self):
# Hash from hashcat sample hashes
result = extract(
result, _ = extract(
"admin::N46iSNekpT:08ca45b7d7ea58ee:88dcbe4446168966a153a0064958dac6:5c7830315c7830310000000000000b45c67103d07d7b95acd12ffa11230e0000000052920b85f78d013c31cdb3b92f5d765c783030",
"DUMMY")

Expand All @@ -30,7 +30,7 @@ def test_netntlmv2(self):

def test_dcc2(self):
# Hash from hashcat sample hashes, embedded in secrets dump output
result = extract(
result, _ = extract(
"""
[*] Dumping cached domain logon information (domain/username:hash)
DOMAIN.COMPANY.COM/tom:$DCC2$10240#tom#e4e938d12fe5974dc42a90120bd9c90f: (2024-10-17 12:31:41)
Expand All @@ -44,7 +44,7 @@ def test_dcc2(self):

def test_secretsdump_local_users_datestamped(self):
# Hash from hashcat sample hashes, embedded in secrets dump output
result = extract(
result, _ = extract(
"""
[2024-10-16 16:12:05] [*] Administrator:500:aad3b435b51404eeaad3b435b51404ee:0123456789abcdef0123456789abcdef:::
[2024-10-16 16:12:07] [*] Guest:501:aad3b435b51404eeaad3b435b51404ee:31d6cfe0d16ae931b73c59d7e0c089c0:::
Expand All @@ -62,7 +62,7 @@ def test_secretsdump_local_users_datestamped(self):

def test_secretsdump_machine_account(self):
# Hash from hashcat sample hashes, embedded in secrets dump output
result = extract(r"""
result, _ = extract(r"""
[2024-10-16 16:12:23] [*] DOMAIN\HOST$:aad3b435b51404eeaad3b435b51404ee:0123456789abcdef0123456789abcdef:::
""", "DUMMY")

Expand All @@ -73,7 +73,7 @@ def test_secretsdump_machine_account(self):

def test_sharpsccm_naa(self):
# Hash from hashcat sample hashes, embedded in secrets dump output
result = extract(r"""
result, _ = extract(r"""
[+] Decrypting network access account credentials
NetworkAccessUsername: DOMAIN\USER
Expand Down
4 changes: 2 additions & 2 deletions event_tracker/cred_extractor/test_extractors_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ class ExtractorTestCaseWithDB(django.test.TestCase):

def test_rubeus_kerberoast_crlf(self):
# Hash from HashCat example hashes
result = extract("""
result, _ = extract("""
[*] SamAccountName : USER\r
[*] DistinguishedName : CN=USER,OU=Service Accounts,OU=Non-Personal,OU=Accounts,OU=Brand,DC=domain,DC=local\r
[*] ServicePrincipalName : test/spn\r
Expand All @@ -28,7 +28,7 @@ def test_rubeus_kerberoast_crlf(self):

def test_rubeus_kerberoast_lf(self):
# Hash from HashCat example hashes
result = extract("""
result, _ = extract("""
[*] SamAccountName : USER
[*] DistinguishedName : CN=USER,OU=Service Accounts,OU=Non-Personal,OU=Accounts,OU=Brand,DC=domain,DC=local
[*] ServicePrincipalName : test/spn
Expand Down

0 comments on commit 6c70396

Please sign in to comment.