T6286: Implement null tokens (#257)

* make log4shell src_data optional

* fix coerce_to_float error handling

* stop reporting drop not found

* fix fortune response for web

* add legacy token support

* add unit test coverage exclusions

* overwrite legacy type in SMTP tokens

canarytokens/channel_dns.py

@@ -204,6 +204,7 @@ def query(self, query: Query, src_ip: str):  # noqa C901
         log.info(f"handling query:  {query.name}")
         try:
             canarydrop, src_data = handle_query_name(query_name=query.name)
+            # import rpdb; rpdb.Rpdb().set_trace()
         except NoCanarytokenFound:
             log.info(f"Query: {query.name} does not match a token.")
             return defer.succeed(self._do_dynamic_response(name=query.name.name))
@@ -231,7 +232,7 @@ def query(self, query: Query, src_ip: str):  # noqa C901
 
         self.dispatch(canarydrop=canarydrop, token_hit=token_hit)
 
-        if IS_NX_DOMAIN:
+        if IS_NX_DOMAIN:  # pragma: no cover
             if canarydrop.type not in [TokenTypes.ADOBE_PDF, TokenTypes.SIGNED_EXE]:
                 log.info(
                     "Token {token} hit the NX domain and is not a pdf. TokenType: {token_type}",
@@ -241,11 +242,11 @@ def query(self, query: Query, src_ip: str):  # noqa C901
             return defer.fail(error.DomainError())
         return defer.succeed(self._do_dynamic_response(name=query.name.name))
 
-    def lookupCAA(self, name, timeout):
+    def lookupCAA(self, name, timeout):  # pragma: no cover
         """Respond with NXdomain to a -t CAA lookup."""
         return defer.fail(error.DomainError())
 
-    def lookupAllRecords(self, name, timeout):
+    def lookupAllRecords(self, name, timeout):  # pragma: no cover
         """Respond with error to a -t ANY lookup."""
         return defer.fail(error.DomainError())
 
@@ -335,5 +336,5 @@ def lookupAllRecords(self, name, timeout):
 
     #     return additional_report
 
-    def _handleMySqlErr(self, result):
+    def _handleMySqlErr(self, result):  # pragma: no cover
         log.error(f"Error dispatching MySQL alert: {result}")

canarytokens/channel_http.py

@@ -13,12 +13,12 @@
 from canarytokens import queries
 from canarytokens.channel import InputChannel
 from canarytokens.constants import INPUT_CHANNEL_HTTP
-from canarytokens.exceptions import NoCanarytokenFound
+from canarytokens.exceptions import NoCanarytokenFound, NoCanarytokenPresent
 from canarytokens.models import AnyTokenHit, TokenTypes
 from canarytokens.queries import get_canarydrop
 from canarytokens.settings import FrontendSettings, SwitchboardSettings
 from canarytokens.switchboard import Switchboard
-from canarytokens.tokens import Canarytoken
+from canarytokens.tokens import Canarytoken, GIF
 from canarytokens.utils import coerce_to_float
 
 log = Logger()
@@ -87,8 +87,15 @@ def render_GET(self, request: Request):
             log.info(
                 f"HTTP GET on path {request.path} did not correspond to a token. Error: {e}"
             )
-            return
-        canarydrop = get_canarydrop(canarytoken)
+            request.setHeader("Content-Type", "image/gif")
+            return GIF
+
+        try:
+            canarydrop = get_canarydrop(canarytoken)
+        except NoCanarytokenPresent as e:
+            log.info(f"Error: {e}")
+            request.setHeader("Content-Type", "image/gif")
+            return GIF
 
         handler = getattr(Canarytoken, f"_get_info_for_{canarydrop.type}")
         http_general_info, src_data = handler(request)
@@ -129,7 +136,12 @@ def render_POST(self, request: Request):
         except NoCanarytokenFound as e:
             log.error(f"Failed to get token from {request.path=}. Error: {e}")
             return b"failed"
-        canarydrop = get_canarydrop(canarytoken=token)
+
+        try:
+            canarydrop = get_canarydrop(token)
+        except NoCanarytokenPresent as e:
+            log.info(f"Canarydrop not found for token {token.value()}. Error: {e}")
+            return b"failed"
         # if key and token args are present, we are either:
         #    -posting browser info
         #    -getting an aws trigger (key == aws_s3)

canarytokens/channel_input_smtp.py

@@ -16,8 +16,14 @@
 from canarytokens.channel import InputChannel
 from canarytokens.constants import INPUT_CHANNEL_SMTP
 from canarytokens.exceptions import NoCanarytokenFound, NoCanarytokenPresent
-from canarytokens.models import SMTPHeloField, SMTPMailField, SMTPTokenHit
-from canarytokens.queries import get_canarydrop
+from canarytokens.models import (
+    SMTPHeloField,
+    SMTPMailField,
+    SMTPTokenHistory,
+    SMTPTokenHit,
+    TokenTypes,
+)
+from canarytokens.queries import get_canarydrop, save_canarydrop
 from canarytokens.settings import FrontendSettings, SwitchboardSettings
 from canarytokens.switchboard import Switchboard
 from canarytokens.tokens import Canarytoken
@@ -162,6 +168,12 @@ def validateTo(self, user):
         try:
             canarytoken = Canarytoken(value=user.dest.local)
             self.canarydrop = get_canarydrop(canarytoken=canarytoken)
+            if self.canarydrop.type == TokenTypes.LEGACY:
+                self.canarydrop.type = TokenTypes.SMTP
+                self.canarydrop.triggered_details = SMTPTokenHistory(
+                    hits=self.canarydrop.triggered_details.hits
+                )
+                save_canarydrop(self.canarydrop)
             return lambda: CanaryMessage(esmtp=self)
         except (NoCanarytokenPresent, NoCanarytokenFound):
             log.warn(

canarytokens/loghandlers.py

@@ -1,3 +1,4 @@
+# pragma: no cover
 from twisted.logger import eventAsJSON, ILogObserver, Logger, LogLevel
 from twisted.web.iweb import IBodyProducer
 from twisted.web.client import Agent
@@ -9,7 +10,6 @@
 import os
 import json
 
-
 log = Logger()
 
 
@@ -21,18 +21,18 @@
 
 @implementer(IBodyProducer)
 class BytesProducer:
-    def __init__(self, body):
+    def __init__(self, body):  # pragma: no cover
         self.body = body
         self.length = len(body)
 
-    def startProducing(self, consumer):
+    def startProducing(self, consumer):  # pragma: no cover
         consumer.write(self.body)
         return succeed(None)
 
-    def pauseProducing(self):
+    def pauseProducing(self):  # pragma: no cover
         pass
 
-    def stopProducing(self):
+    def stopProducing(self):  # pragma: no cover
         pass
 
 
@@ -42,15 +42,15 @@ class errorsToWebhookLogObserver(object):
     Log observer that sends errors out to a Slack endpoint.
     """
 
-    def __init__(self, formatEvent):
+    def __init__(self, formatEvent):  # pragma: no cover
         """
         @param formatEvent: A callable that formats an event.
         @type formatEvent: L{callable} that takes an C{event} argument and
             returns a formatted event as L{unicode}.
         """
         self.formatEvent = formatEvent
 
-    def __call__(self, event):
+    def __call__(self, event):  # pragma: no cover
         """
         Check if log_level Error or higher, if so post to webhook
 
@@ -75,7 +75,7 @@ def __call__(self, event):
             _ = httpRequest(postdata)
 
 
-def httpRequest(postdata):
+def httpRequest(postdata):  # pragma: no cover
     agent = Agent(reactor)
     headers = {b"Content-Type": [b"application/x-www-form-urlencoded"]}
     data_str = json.dumps(postdata)
@@ -95,7 +95,7 @@ def handle_response(response):
     return d
 
 
-def webhookLogObserver(recordSeparator="\x1e"):
+def webhookLogObserver(recordSeparator="\x1e"):  # pragma: no cover
     """
     Create a L{errorsToWebhookLogObserver} that emits error and critical
     loglines' text to a specified webhook URL by doing a HTTP POST.

canarytokens/models.py

@@ -296,6 +296,7 @@ class TokenTypes(str, enum.Enum):
     CMD = "cmd"
     CC = "cc"
     SLACK_API = "slack_api"
+    LEGACY = "legacy"
 
     def __str__(self) -> str:
         return str(self.value)
@@ -336,6 +337,7 @@ def __str__(self) -> str:
     TokenTypes.CMD: "sensitive command",
     TokenTypes.CC: "credit card",
     TokenTypes.SLACK_API: "Slack API",
+    TokenTypes.LEGACY: "legacy",
 }
 
 GeneralHistoryTokenType = Literal[
@@ -1271,7 +1273,6 @@ def get_additional_data_for_notification(self) -> Dict[str, Any]:
                 "src_ip",
                 "is_tor_relay",
                 "input_channel",
-                # 'src_data',
                 "token_type",
             ),
         )
@@ -1483,7 +1484,7 @@ class FastRedirectTokenHit(TokenHit):
 
 class Log4ShellTokenHit(TokenHit):
     token_type: Literal[TokenTypes.LOG4SHELL] = TokenTypes.LOG4SHELL
-    src_data: dict
+    src_data: Optional[dict]
 
 
 class QRCodeTokenHit(TokenHit):
@@ -1512,6 +1513,24 @@ class WireguardTokenHit(TokenHit):
     src_data: WireguardSrcData
 
 
+class LegacyTokenHit(TokenHit):
+    token_type: Literal[TokenTypes.LEGACY] = TokenTypes.LEGACY
+    # zip;
+    src_data: Optional[dict]
+    # excel; word; image; QR;
+    useragent: Optional[str]
+    # web
+    request_headers: Optional[dict]
+    request_args: Optional[dict]
+    # web; image
+    additional_info: Optional[AdditionalInfo] = AdditionalInfo()
+    # cloned_web
+    referer: Optional[Union[str, bytes]]
+    location: Optional[Union[str, bytes]]
+    # smtp
+    mail: Optional[SMTPMailField]
+
+
 AnyTokenHit = Annotated[
     Union[
         CCTokenHit,
@@ -1539,6 +1558,7 @@ class WireguardTokenHit(TokenHit):
         WindowsDirectoryTokenHit,
         SQLServerTokenHit,
         KubeconfigTokenHit,
+        LegacyTokenHit,
     ],
     Field(discriminator="token_type"),
 ]
@@ -1727,6 +1747,11 @@ class SvnTokenHistory(TokenHistory[SvnTokenHit]):
     hits: List[SvnTokenHit] = []
 
 
+class LegacyTokenHistory(TokenHistory[LegacyTokenHit]):
+    token_type: Literal[TokenTypes.LEGACY] = TokenTypes.LEGACY
+    hits: List[LegacyTokenHit] = []
+
+
 # AnyTokenHistory is used to type annotate functions that
 # handle any token history. It makes use of an annotated type
 # that discriminates on `token_type` so pydantic can parse
@@ -1758,6 +1783,7 @@ class SvnTokenHistory(TokenHistory[SvnTokenHit]):
         WindowsDirectoryTokenHistory,
         SQLServerTokenHistory,
         KubeconfigTokenHistory,
+        LegacyTokenHistory,
     ],
     Field(discriminator="token_type"),
 ]