Prompt Layer Rework (#688)

backend/danswer/chat/chat_llm.py

@@ -35,7 +35,7 @@
 from danswer.indexing.models import InferenceChunk
 from danswer.llm.factory import get_default_llm
 from danswer.llm.interfaces import LLM
-from danswer.llm.utils import get_default_llm_tokenizer
+from danswer.llm.utils import get_default_llm_token_encode
 from danswer.llm.utils import translate_danswer_msg_to_langchain
 from danswer.search.access_filters import build_access_filters_for_user
 from danswer.search.models import IndexFilters
@@ -259,7 +259,7 @@ def llm_contextless_chat_answer(
         prompt_msgs = [translate_danswer_msg_to_langchain(msg) for msg in messages]
 
         if system_text:
-            tokenizer = tokenizer or get_default_llm_tokenizer()
+            tokenizer = tokenizer or get_default_llm_token_encode()
             system_tokens = len(tokenizer(system_text))
             system_msg = SystemMessage(content=system_text)
 

backend/danswer/chat/chat_prompts.py

@@ -2,12 +2,12 @@
 from langchain.schema.messages import HumanMessage
 from langchain.schema.messages import SystemMessage
 
-from danswer.configs.constants import CODE_BLOCK_PAT
 from danswer.configs.constants import MessageType
 from danswer.db.models import ChatMessage
 from danswer.db.models import ToolInfo
 from danswer.indexing.models import InferenceChunk
 from danswer.llm.utils import translate_danswer_msg_to_langchain
+from danswer.prompts.constants import CODE_BLOCK_PAT
 
 DANSWER_TOOL_NAME = "Current Search"
 DANSWER_TOOL_DESCRIPTION = (
@@ -176,7 +176,7 @@ def format_danswer_chunks_for_chat(chunks: list[InferenceChunk]) -> str:
         return "No Results Found"
 
     return "\n".join(
-        f"DOCUMENT {ind}:{CODE_BLOCK_PAT.format(chunk.content)}"
+        f"DOCUMENT {ind}:\n{CODE_BLOCK_PAT.format(chunk.content)}\n"
         for ind, chunk in enumerate(chunks, start=1)
     )
 

backend/danswer/configs/app_configs.py

@@ -212,6 +212,9 @@
     "DYNAMIC_CONFIG_STORE", "FileSystemBackedDynamicConfigStore"
 )
 DYNAMIC_CONFIG_DIR_PATH = os.environ.get("DYNAMIC_CONFIG_DIR_PATH", "/home/storage")
+# For selecting a different LLM question-answering prompt format
+# Valid values: default, cot, weak
+QA_PROMPT_OVERRIDE = os.environ.get("QA_PROMPT_OVERRIDE") or None
 # notset, debug, info, warning, error, or critical
 LOG_LEVEL = os.environ.get("LOG_LEVEL", "info")
 # NOTE: Currently only supported in the Confluence and Google Drive connectors +

backend/danswer/configs/constants.py

@@ -36,20 +36,6 @@
 DEFAULT_BOOST = 0
 SESSION_KEY = "session"
 
-# Prompt building constants:
-GENERAL_SEP_PAT = "\n-----\n"
-CODE_BLOCK_PAT = "\n```\n{}\n```\n"
-DOC_SEP_PAT = "---NEW DOCUMENT---"
-DOC_CONTENT_START_PAT = "DOCUMENT CONTENTS:\n"
-QUESTION_PAT = "Query:"
-THOUGHT_PAT = "Thought:"
-ANSWER_PAT = "Answer:"
-FINAL_ANSWER_PAT = "Final Answer:"
-UNCERTAINTY_PAT = "?"
-QUOTE_PAT = "Quote:"
-QUOTES_PAT_PLURAL = "Quotes:"
-INVALID_PAT = "Invalid:"
-
 
 class DocumentSource(str, Enum):
     SLACK = "slack"

backend/danswer/direct_qa/answer_question.py

@@ -9,9 +9,9 @@
 from danswer.configs.constants import IGNORE_FOR_QA
 from danswer.db.feedback import create_query_event
 from danswer.db.models import User
+from danswer.direct_qa.factory import get_default_qa_model
 from danswer.direct_qa.interfaces import DanswerAnswerPiece
 from danswer.direct_qa.interfaces import StreamingError
-from danswer.direct_qa.llm_utils import get_default_qa_model
 from danswer.direct_qa.models import LLMMetricsContainer
 from danswer.direct_qa.qa_utils import get_usable_chunks
 from danswer.document_index.factory import get_default_document_index

backend/danswer/direct_qa/llm_utils.py → backend/danswer/direct_qa/factory.py

@@ -1,21 +1,35 @@
+from danswer.configs.app_configs import QA_PROMPT_OVERRIDE
 from danswer.configs.app_configs import QA_TIMEOUT
 from danswer.direct_qa.interfaces import QAModel
 from danswer.direct_qa.qa_block import QABlock
 from danswer.direct_qa.qa_block import QAHandler
 from danswer.direct_qa.qa_block import SingleMessageQAHandler
 from danswer.direct_qa.qa_block import SingleMessageScratchpadHandler
+from danswer.direct_qa.qa_block import WeakLLMQAHandler
 from danswer.llm.factory import get_default_llm
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
 
 
-# TODO introduce the prompt choice parameter
-def get_default_qa_handler(real_time_flow: bool = True) -> QAHandler:
-    return (
-        SingleMessageQAHandler() if real_time_flow else SingleMessageScratchpadHandler()
-    )
-    # return SimpleChatQAHandler()
+def get_default_qa_handler(
+    real_time_flow: bool = True,
+    user_selection: str | None = QA_PROMPT_OVERRIDE,
+) -> QAHandler:
+    if user_selection:
+        if user_selection.lower() == "default":
+            return SingleMessageQAHandler()
+        if user_selection.lower() == "cot":
+            return SingleMessageScratchpadHandler()
+        if user_selection.lower() == "weak":
+            return WeakLLMQAHandler()
+
+        raise ValueError("Invalid Question-Answering prompt selected")
+
+    if not real_time_flow:
+        return SingleMessageScratchpadHandler()
+
+    return SingleMessageQAHandler()
 
 
 def get_default_qa_model(

backend/danswer/direct_qa/interfaces.py

@@ -52,7 +52,6 @@ class QAModel:
     def requires_api_key(self) -> bool:
         """Is this model protected by security features
         Does it need an api key to access the model for inference"""
-        # TODO, this should be false for custom request model and gpt4all
         return True
 
     def warm_up_model(self) -> None: