From 48ec4cee9adec63bb94ab53d03c0e8e4c0cd86c1 Mon Sep 17 00:00:00 2001 From: kun Date: Mon, 11 Mar 2024 13:53:35 +0800 Subject: [PATCH 1/6] add support for anthropic claude --- pyproject.toml | 3 +- src/vanna/claude/__init__.py | 0 src/vanna/claude/claude_chat.py | 78 +++++++++++++++++++++++++++++++++ tests/test_vanna.py | 33 +++++++++++--- 4 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 src/vanna/claude/__init__.py create mode 100644 src/vanna/claude/claude_chat.py diff --git a/pyproject.toml b/pyproject.toml index e21937e1..54c11a86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,10 +31,11 @@ mysql = ["PyMySQL"] bigquery = ["google-cloud-bigquery"] snowflake = ["snowflake-connector-python"] duckdb = ["duckdb"] -all = ["psycopg2-binary", "db-dtypes", "PyMySQL", "google-cloud-bigquery", "snowflake-connector-python", "duckdb", "openai", "mistralai", "chromadb"] +all = ["psycopg2-binary", "db-dtypes", "PyMySQL", "google-cloud-bigquery", "snowflake-connector-python", "duckdb", "openai", "mistralai", "chromadb", "anthropic"] test = ["tox"] chromadb = ["chromadb"] openai = ["openai"] mistralai = ["mistralai"] +anthropic = ["anthropic"] gemini = ["google-generativeai"] marqo = ["marqo"] diff --git a/src/vanna/claude/__init__.py b/src/vanna/claude/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/vanna/claude/claude_chat.py b/src/vanna/claude/claude_chat.py new file mode 100644 index 00000000..ac533e72 --- /dev/null +++ b/src/vanna/claude/claude_chat.py @@ -0,0 +1,78 @@ +import os + +import anthropic + +from ..base import VannaBase + + +class Claude_Chat(VannaBase): + def __init__(self, client=None, config=None): + VannaBase.__init__(self, config=config) + + if client is not None: + self.client = client + return + + if config is None and client is None: + self.client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) + return + + # default parameters - can be overrided using config + self.temperature = 0.7 + self.max_tokens = 500 + + if "temperature" in config: + self.temperature = config["temperature"] + + if "max_tokens" in config: + self.max_tokens = config["max_tokens"] + + if "api_key" in config: + self.client = anthropic.Anthropic(api_key=config["api_key"]) + + def system_message(self, message: str) -> any: + return {"role": "system", "content": message} + + def user_message(self, message: str) -> any: + return {"role": "user", "content": message} + + def assistant_message(self, message: str) -> any: + return {"role": "assistant", "content": message} + + def submit_prompt(self, prompt, **kwargs) -> str: + if prompt is None: + raise Exception("Prompt is None") + + if len(prompt) == 0: + raise Exception("Prompt is empty") + + # Count the number of tokens in the message log + # Use 4 as an approximation for the number of characters per token + num_tokens = 0 + for message in prompt: + num_tokens += len(message["content"]) / 4 + + if self.config is not None and "model" in self.config: + print( + f"Using model {self.config['model']} for {num_tokens} tokens (approx)" + ) + # claude required system message is a single filed + # https://docs.anthropic.com/claude/reference/messages_post + system_message = '' + no_system_prompt = [] + for prompt_message in prompt: + role = prompt_message['role'] + if role == 'system': + system_message = prompt_message['content'] + else: + no_system_prompt.append({"role": role, "content": prompt_message['content']}) + + response = self.client.messages.create( + model=self.config["model"], + messages=no_system_prompt, + system=system_message, + max_tokens=self.max_tokens, + temperature=self.temperature, + ) + + return response.content[0].text diff --git a/tests/test_vanna.py b/tests/test_vanna.py index 1bf0e40a..958631ad 100644 --- a/tests/test_vanna.py +++ b/tests/test_vanna.py @@ -1,10 +1,10 @@ -from vanna.openai.openai_chat import OpenAI_Chat -from vanna.vannadb.vannadb_vector import VannaDB_VectorStore +import os + +from vanna.claude.claude_chat import Claude_Chat from vanna.mistral.mistral import Mistral +from vanna.openai.openai_chat import OpenAI_Chat from vanna.remote import VannaDefault - - -import os +from vanna.vannadb.vannadb_vector import VannaDB_VectorStore try: print("Trying to load .env") @@ -15,9 +15,11 @@ pass MY_VANNA_MODEL = 'chinook' +ANTHROPIC_Model = 'claude-3-sonnet-20240229' MY_VANNA_API_KEY = os.environ['VANNA_API_KEY'] OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] MISTRAL_API_KEY = os.environ['MISTRAL_API_KEY'] +ANTHROPIC_API_KEY = os.environ['ANTHROPIC_API_KEY'] class VannaOpenAI(VannaDB_VectorStore, OpenAI_Chat): def __init__(self, config=None): @@ -53,8 +55,9 @@ def test_vn_default(): df = vn_default.run_sql(sql) assert len(df) == 6 -from vanna.openai.openai_chat import OpenAI_Chat from vanna.chromadb.chromadb_vector import ChromaDB_VectorStore +from vanna.openai.openai_chat import OpenAI_Chat + class MyVanna(ChromaDB_VectorStore, OpenAI_Chat): def __init__(self, config=None): @@ -72,4 +75,20 @@ def test_vn_chroma(): sql = vn_chroma.generate_sql("What are the top 7 customers by sales?") df = vn_chroma.run_sql(sql) - assert len(df) == 7 \ No newline at end of file + assert len(df) == 7 + + +class VannaClaude(VannaDB_VectorStore, Claude_Chat): + def __init__(self, config=None): + VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) + Claude_Chat.__init__(self, config={'api_key': ANTHROPIC_API_KEY, 'model': ANTHROPIC_Model}) + + +vn_claude = VannaClaude() +vn_claude.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') + + +def test_vn_claude(): + sql = vn_claude.generate_sql("What are the top 5 customers by sales?") + df = vn_claude.run_sql(sql) + assert len(df) == 5 From 077ce4cb5b4a28e6a192d0c777bf06000f83d204 Mon Sep 17 00:00:00 2001 From: kun Date: Mon, 11 Mar 2024 16:21:20 +0800 Subject: [PATCH 2/6] rename the claude to anthropic --- src/vanna/{claude => anthropic}/__init__.py | 0 .../anthropic_chat.py} | 2 +- tests/test_vanna.py | 126 +++++++++--------- 3 files changed, 64 insertions(+), 64 deletions(-) rename src/vanna/{claude => anthropic}/__init__.py (100%) rename src/vanna/{claude/claude_chat.py => anthropic/anthropic_chat.py} (98%) diff --git a/src/vanna/claude/__init__.py b/src/vanna/anthropic/__init__.py similarity index 100% rename from src/vanna/claude/__init__.py rename to src/vanna/anthropic/__init__.py diff --git a/src/vanna/claude/claude_chat.py b/src/vanna/anthropic/anthropic_chat.py similarity index 98% rename from src/vanna/claude/claude_chat.py rename to src/vanna/anthropic/anthropic_chat.py index ac533e72..fd2333ee 100644 --- a/src/vanna/claude/claude_chat.py +++ b/src/vanna/anthropic/anthropic_chat.py @@ -5,7 +5,7 @@ from ..base import VannaBase -class Claude_Chat(VannaBase): +class Anthropic_Chat(VannaBase): def __init__(self, client=None, config=None): VannaBase.__init__(self, config=config) diff --git a/tests/test_vanna.py b/tests/test_vanna.py index 958631ad..9911e52c 100644 --- a/tests/test_vanna.py +++ b/tests/test_vanna.py @@ -1,6 +1,6 @@ import os -from vanna.claude.claude_chat import Claude_Chat +from vanna.anthropic.anthropic_chat import Anthropic_Chat from vanna.mistral.mistral import Mistral from vanna.openai.openai_chat import OpenAI_Chat from vanna.remote import VannaDefault @@ -17,71 +17,71 @@ MY_VANNA_MODEL = 'chinook' ANTHROPIC_Model = 'claude-3-sonnet-20240229' MY_VANNA_API_KEY = os.environ['VANNA_API_KEY'] -OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] -MISTRAL_API_KEY = os.environ['MISTRAL_API_KEY'] +# OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] +# MISTRAL_API_KEY = os.environ['MISTRAL_API_KEY'] ANTHROPIC_API_KEY = os.environ['ANTHROPIC_API_KEY'] - -class VannaOpenAI(VannaDB_VectorStore, OpenAI_Chat): - def __init__(self, config=None): - VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) - OpenAI_Chat.__init__(self, config=config) - -vn_openai = VannaOpenAI(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) -vn_openai.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') - -def test_vn_openai(): - sql = vn_openai.generate_sql("What are the top 4 customers by sales?") - df = vn_openai.run_sql(sql) - assert len(df) == 4 - -class VannaMistral(VannaDB_VectorStore, Mistral): - def __init__(self, config=None): - VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) - Mistral.__init__(self, config={'api_key': MISTRAL_API_KEY, 'model': 'mistral-tiny'}) - -vn_mistral = VannaMistral() -vn_mistral.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') - -def test_vn_mistral(): - sql = vn_mistral.generate_sql("What are the top 5 customers by sales?") - df = vn_mistral.run_sql(sql) - assert len(df) == 5 - -vn_default = VannaDefault(model=MY_VANNA_MODEL, api_key=MY_VANNA_API_KEY) -vn_default.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') - -def test_vn_default(): - sql = vn_default.generate_sql("What are the top 6 customers by sales?") - df = vn_default.run_sql(sql) - assert len(df) == 6 - -from vanna.chromadb.chromadb_vector import ChromaDB_VectorStore -from vanna.openai.openai_chat import OpenAI_Chat - - -class MyVanna(ChromaDB_VectorStore, OpenAI_Chat): - def __init__(self, config=None): - ChromaDB_VectorStore.__init__(self, config=config) - OpenAI_Chat.__init__(self, config=config) - -vn_chroma = MyVanna(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) -vn_chroma.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') - -def test_vn_chroma(): - df_ddl = vn_chroma.run_sql("SELECT type, sql FROM sqlite_master WHERE sql is not null") - - for ddl in df_ddl['sql'].to_list(): - vn_chroma.train(ddl=ddl) - - sql = vn_chroma.generate_sql("What are the top 7 customers by sales?") - df = vn_chroma.run_sql(sql) - assert len(df) == 7 - - -class VannaClaude(VannaDB_VectorStore, Claude_Chat): +# +# class VannaOpenAI(VannaDB_VectorStore, OpenAI_Chat): +# def __init__(self, config=None): +# VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) +# OpenAI_Chat.__init__(self, config=config) +# +# vn_openai = VannaOpenAI(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) +# vn_openai.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') +# +# def test_vn_openai(): +# sql = vn_openai.generate_sql("What are the top 4 customers by sales?") +# df = vn_openai.run_sql(sql) +# assert len(df) == 4 +# +# class VannaMistral(VannaDB_VectorStore, Mistral): +# def __init__(self, config=None): +# VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) +# Mistral.__init__(self, config={'api_key': MISTRAL_API_KEY, 'model': 'mistral-tiny'}) +# +# vn_mistral = VannaMistral() +# vn_mistral.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') +# +# def test_vn_mistral(): +# sql = vn_mistral.generate_sql("What are the top 5 customers by sales?") +# df = vn_mistral.run_sql(sql) +# assert len(df) == 5 +# +# vn_default = VannaDefault(model=MY_VANNA_MODEL, api_key=MY_VANNA_API_KEY) +# vn_default.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') +# +# def test_vn_default(): +# sql = vn_default.generate_sql("What are the top 6 customers by sales?") +# df = vn_default.run_sql(sql) +# assert len(df) == 6 +# +# from vanna.chromadb.chromadb_vector import ChromaDB_VectorStore +# from vanna.openai.openai_chat import OpenAI_Chat +# +# +# class MyVanna(ChromaDB_VectorStore, OpenAI_Chat): +# def __init__(self, config=None): +# ChromaDB_VectorStore.__init__(self, config=config) +# OpenAI_Chat.__init__(self, config=config) +# +# vn_chroma = MyVanna(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) +# vn_chroma.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') +# +# def test_vn_chroma(): +# df_ddl = vn_chroma.run_sql("SELECT type, sql FROM sqlite_master WHERE sql is not null") +# +# for ddl in df_ddl['sql'].to_list(): +# vn_chroma.train(ddl=ddl) +# +# sql = vn_chroma.generate_sql("What are the top 7 customers by sales?") +# df = vn_chroma.run_sql(sql) +# assert len(df) == 7 + + +class VannaClaude(VannaDB_VectorStore, Anthropic_Chat): def __init__(self, config=None): VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) - Claude_Chat.__init__(self, config={'api_key': ANTHROPIC_API_KEY, 'model': ANTHROPIC_Model}) + Anthropic_Chat.__init__(self, config={'api_key': ANTHROPIC_API_KEY, 'model': ANTHROPIC_Model}) vn_claude = VannaClaude() From a2575f70e9edede48d4e5a35957f91d6e9a756b2 Mon Sep 17 00:00:00 2001 From: kun Date: Mon, 11 Mar 2024 16:24:23 +0800 Subject: [PATCH 3/6] rename the claude to anthropic --- tests/test_vanna.py | 116 ++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/tests/test_vanna.py b/tests/test_vanna.py index 9911e52c..be453b85 100644 --- a/tests/test_vanna.py +++ b/tests/test_vanna.py @@ -17,65 +17,65 @@ MY_VANNA_MODEL = 'chinook' ANTHROPIC_Model = 'claude-3-sonnet-20240229' MY_VANNA_API_KEY = os.environ['VANNA_API_KEY'] -# OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] -# MISTRAL_API_KEY = os.environ['MISTRAL_API_KEY'] +OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] +MISTRAL_API_KEY = os.environ['MISTRAL_API_KEY'] ANTHROPIC_API_KEY = os.environ['ANTHROPIC_API_KEY'] -# -# class VannaOpenAI(VannaDB_VectorStore, OpenAI_Chat): -# def __init__(self, config=None): -# VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) -# OpenAI_Chat.__init__(self, config=config) -# -# vn_openai = VannaOpenAI(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) -# vn_openai.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') -# -# def test_vn_openai(): -# sql = vn_openai.generate_sql("What are the top 4 customers by sales?") -# df = vn_openai.run_sql(sql) -# assert len(df) == 4 -# -# class VannaMistral(VannaDB_VectorStore, Mistral): -# def __init__(self, config=None): -# VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) -# Mistral.__init__(self, config={'api_key': MISTRAL_API_KEY, 'model': 'mistral-tiny'}) -# -# vn_mistral = VannaMistral() -# vn_mistral.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') -# -# def test_vn_mistral(): -# sql = vn_mistral.generate_sql("What are the top 5 customers by sales?") -# df = vn_mistral.run_sql(sql) -# assert len(df) == 5 -# -# vn_default = VannaDefault(model=MY_VANNA_MODEL, api_key=MY_VANNA_API_KEY) -# vn_default.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') -# -# def test_vn_default(): -# sql = vn_default.generate_sql("What are the top 6 customers by sales?") -# df = vn_default.run_sql(sql) -# assert len(df) == 6 -# -# from vanna.chromadb.chromadb_vector import ChromaDB_VectorStore -# from vanna.openai.openai_chat import OpenAI_Chat -# -# -# class MyVanna(ChromaDB_VectorStore, OpenAI_Chat): -# def __init__(self, config=None): -# ChromaDB_VectorStore.__init__(self, config=config) -# OpenAI_Chat.__init__(self, config=config) -# -# vn_chroma = MyVanna(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) -# vn_chroma.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') -# -# def test_vn_chroma(): -# df_ddl = vn_chroma.run_sql("SELECT type, sql FROM sqlite_master WHERE sql is not null") -# -# for ddl in df_ddl['sql'].to_list(): -# vn_chroma.train(ddl=ddl) -# -# sql = vn_chroma.generate_sql("What are the top 7 customers by sales?") -# df = vn_chroma.run_sql(sql) -# assert len(df) == 7 + +class VannaOpenAI(VannaDB_VectorStore, OpenAI_Chat): + def __init__(self, config=None): + VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) + OpenAI_Chat.__init__(self, config=config) + +vn_openai = VannaOpenAI(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) +vn_openai.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') + +def test_vn_openai(): + sql = vn_openai.generate_sql("What are the top 4 customers by sales?") + df = vn_openai.run_sql(sql) + assert len(df) == 4 + +class VannaMistral(VannaDB_VectorStore, Mistral): + def __init__(self, config=None): + VannaDB_VectorStore.__init__(self, vanna_model=MY_VANNA_MODEL, vanna_api_key=MY_VANNA_API_KEY, config=config) + Mistral.__init__(self, config={'api_key': MISTRAL_API_KEY, 'model': 'mistral-tiny'}) + +vn_mistral = VannaMistral() +vn_mistral.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') + +def test_vn_mistral(): + sql = vn_mistral.generate_sql("What are the top 5 customers by sales?") + df = vn_mistral.run_sql(sql) + assert len(df) == 5 + +vn_default = VannaDefault(model=MY_VANNA_MODEL, api_key=MY_VANNA_API_KEY) +vn_default.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') + +def test_vn_default(): + sql = vn_default.generate_sql("What are the top 6 customers by sales?") + df = vn_default.run_sql(sql) + assert len(df) == 6 + +from vanna.chromadb.chromadb_vector import ChromaDB_VectorStore +from vanna.openai.openai_chat import OpenAI_Chat + + +class MyVanna(ChromaDB_VectorStore, OpenAI_Chat): + def __init__(self, config=None): + ChromaDB_VectorStore.__init__(self, config=config) + OpenAI_Chat.__init__(self, config=config) + +vn_chroma = MyVanna(config={'api_key': OPENAI_API_KEY, 'model': 'gpt-3.5-turbo'}) +vn_chroma.connect_to_sqlite('https://vanna.ai/Chinook.sqlite') + +def test_vn_chroma(): + df_ddl = vn_chroma.run_sql("SELECT type, sql FROM sqlite_master WHERE sql is not null") + + for ddl in df_ddl['sql'].to_list(): + vn_chroma.train(ddl=ddl) + + sql = vn_chroma.generate_sql("What are the top 7 customers by sales?") + df = vn_chroma.run_sql(sql) + assert len(df) == 7 class VannaClaude(VannaDB_VectorStore, Anthropic_Chat): From 8127d4ab60ec9fb85dbc8085cd475f6ea9c603c8 Mon Sep 17 00:00:00 2001 From: Youri Aubry <76754000+Molrn@users.noreply.github.com> Date: Fri, 22 Mar 2024 09:46:05 +0100 Subject: [PATCH 4/6] fix(flask): no followup questions error --- src/vanna/flask/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vanna/flask/__init__.py b/src/vanna/flask/__init__.py index 2a80e354..c81f8d79 100644 --- a/src/vanna/flask/__init__.py +++ b/src/vanna/flask/__init__.py @@ -319,6 +319,7 @@ def generate_followup_questions(id: str, df, question, sql): } ) else: + cache.set(id=id, field="followup_questions", value=[]) return jsonify( { "type": "question_list", From e470dec98ce266b29235ccc56f1c02b821376d67 Mon Sep 17 00:00:00 2001 From: Zain Hoda <7146154+zainhoda@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:41:54 -0400 Subject: [PATCH 5/6] Update tests.yml --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 687caf95..36790669 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,4 +26,5 @@ jobs: VANNA_API_KEY: ${{ secrets.VANNA_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: tox -e py310 From e78cabbfb76b54c104647f0ee1e199a5691709a3 Mon Sep 17 00:00:00 2001 From: Zain Hoda <7146154+zainhoda@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:46:30 -0400 Subject: [PATCH 6/6] Fix generate_sql --- src/vanna/base/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vanna/base/base.py b/src/vanna/base/base.py index 0e9e9cb6..7e065ed2 100644 --- a/src/vanna/base/base.py +++ b/src/vanna/base/base.py @@ -103,7 +103,10 @@ def generate_sql(self, question: str, **kwargs) -> str: Returns: str: The SQL query that answers the question. """ - initial_prompt = self.config.get("initial_prompt", None) + if self.config is not None: + initial_prompt = self.config.get("initial_prompt", None) + else: + initial_prompt = None question_sql_list = self.get_similar_question_sql(question, **kwargs) ddl_list = self.get_related_ddl(question, **kwargs) doc_list = self.get_related_documentation(question, **kwargs)