Merge branch 'main' into feat/llm-agent-limit-tool-call-depth

libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py

@@ -143,7 +143,7 @@ async def add_datasource(self, assistant: dict, data: dict):
         data["index_name"] = await self._get_unique_index_name(data, assistant)
 
         await self._add_superrag_tool(assistant, data)
-        self.superrag_service.ingest(data=data)
+        await self.superrag_service.aingest(data=data)
 
     async def delete_datasource(self, assistant: dict, datasource: dict):
         tool = await self.agent_manager.get_tool(
@@ -156,7 +156,7 @@ async def delete_datasource(self, assistant: dict, datasource: dict):
             tool_metadata = json.loads(tool.metadata)
 
             await self._delete_tool(assistant, datasource)
-            self.superrag_service.delete(
+            await self.superrag_service.adelete(
                 {
                     **datasource,
                     "index_name": tool_metadata.get("index_name"),

libs/superagent/app/tools/superrag.py

@@ -58,7 +58,7 @@ async def _arun(
 
         credentials = get_superrag_compatible_credentials(provider.options)
 
-        res = self.superrag_service.query(
+        res = await self.superrag_service.aquery(
             {
                 "vector_database": {"type": database_provider, "config": credentials},
                 "index_name": index_name,

libs/superagent/lint-and-format.sh

@@ -3,6 +3,9 @@
 # getting changed files (only staged)
 changes=$(git diff --name-only --cached | grep '^libs/superagent.*\.py$' | sed 's|^libs/superagent/||')
 
+# Filter deleted files
+changes=$(echo "$changes" | while read -r file; do [ -e "$file" ] && echo "$file"; done)
+
 lint() {
     poetry run black $changes
     # sort imports 

libs/superagent/pyproject.toml

@@ -50,7 +50,7 @@ openai = "^1.1.1"
 langchain-experimental = "^0.0.37"
 pydub = "^0.25.1"
 algoliasearch = "^3.0.0"
-litellm = "1.35.21"
+litellm = "1.35.33"
 weaviate-client = "^3.25.3"
 qdrant-client = "^1.6.9"
 vecs = "^0.4.2"

libs/superagent/services/superrag.py

@@ -2,7 +2,7 @@
 from typing import Optional
 from urllib.parse import unquote, urlparse
 
-import requests
+from aiohttp import ClientSession
 from decouple import config
 from pydantic import BaseModel
 
@@ -42,10 +42,10 @@ def suffix(self) -> str:
 
 class File(BaseModel):
     url: str
-    name: str | None = None
+    name: Optional[str] = None
 
     @property
-    def type(self) -> FileType | None:
+    def type(self) -> Optional[FileType]:
         url = self.url
         if url:
             parsed_url = urlparse(url)
@@ -79,25 +79,29 @@ def __init__(self, url: Optional[str] = None):
         if not self.url:
             raise ValueError("SUPERRAG_API_URL is not set")
 
-    def _request(self, method, endpoint, data):
-        return requests.request(method, f"{self.url}/{endpoint}", json=data).json()
+    async def _arequest(self, method, endpoint, data):
+        async with ClientSession() as session:
+            async with session.request(
+                method, f"{self.url}/{endpoint}", json=data
+            ) as response:
+                return await response.json()
 
-    def ingest(self, data):
-        return self._request(
+    async def aingest(self, data):
+        return await self._arequest(
             "POST",
             "ingest",
             data,
         )
 
-    def delete(self, data):
-        return self._request(
+    async def adelete(self, data):
+        return await self._arequest(
             "DELETE",
             "delete",
             data,
         )
 
-    def query(self, data):
-        return self._request(
+    async def aquery(self, data):
+        return await self._arequest(
             "POST",
             "query",
             data,