diff --git a/scripts/export_data_graph.py b/scripts/export_data_graph.py index c1bd9a7..9f7bd00 100644 --- a/scripts/export_data_graph.py +++ b/scripts/export_data_graph.py @@ -1,6 +1,7 @@ -from neo4j import GraphDatabase import json +from neo4j import GraphDatabase + # Replace these credentials with your Neo4j database info uri = "bolt://localhost:7687" username = "neo4j" diff --git a/unoplat-code-confluence/.isort.cfg b/unoplat-code-confluence/.isort.cfg new file mode 100644 index 0000000..7bbd40b --- /dev/null +++ b/unoplat-code-confluence/.isort.cfg @@ -0,0 +1,7 @@ +[settings] +known_third_party = marko,pydantic,ruff,loguru,pygithub,pypdf,pydantic_settings,litellm,pytest,dspy_ai,packaging,progiter,sentence_transformers,einops,rich,neo4j,neomodel,requirements_parser,tomlkit,stdlib_list,pytest_cov,gitpython +import_heading_stdlib = Standard Library +import_heading_thirdparty = Third Party +import_heading_firstparty = First Party +import_heading_localfolder = Local +py_version = 311 # For Python 3.12 \ No newline at end of file diff --git a/unoplat-code-confluence/unoplat_code_confluence/__main__.py b/unoplat-code-confluence/unoplat_code_confluence/__main__.py index 4c7f0e8..3a2bd66 100755 --- a/unoplat-code-confluence/unoplat_code_confluence/__main__.py +++ b/unoplat-code-confluence/unoplat_code_confluence/__main__.py @@ -1,26 +1,41 @@ +# Standard Library import argparse import asyncio -import os -from loguru import logger import datetime -from unoplat_code_confluence.codebaseparser.arc_guard_handler import ArchGuardHandler +import os import re +import warnings + +# Third Party +from loguru import logger +from packaging import version + +# First Party +from unoplat_code_confluence.codebaseparser.arc_guard_handler import \ + ArchGuardHandler from unoplat_code_confluence.configuration.external_config import AppConfig -from unoplat_code_confluence.data_models.chapi_unoplat_codebase import UnoplatCodebase -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import DspyUnoplatCodebaseSummary -from unoplat_code_confluence.database.graph.unoplat_graph_processing import UnoplatGraphProcessing +from unoplat_code_confluence.data_models.chapi_unoplat_codebase import \ + UnoplatCodebase +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import \ + DspyUnoplatCodebaseSummary +from unoplat_code_confluence.database.graph.unoplat_graph_processing import \ + UnoplatGraphProcessing from unoplat_code_confluence.downloader.downloader import Downloader -from unoplat_code_confluence.dspy_class_summary import CodeConfluenceClassModule -from unoplat_code_confluence.dspy_codebase_summary import CodeConfluenceCodebaseModule -from unoplat_code_confluence.dspy_function_summary import CodeConfluenceFunctionModule -from unoplat_code_confluence.dspy_package_summary import CodeConfluencePackageModule +from unoplat_code_confluence.dspy_class_summary import \ + CodeConfluenceClassModule +from unoplat_code_confluence.dspy_codebase_summary import \ + CodeConfluenceCodebaseModule +from unoplat_code_confluence.dspy_function_summary import \ + CodeConfluenceFunctionModule +from unoplat_code_confluence.dspy_package_summary import \ + CodeConfluencePackageModule from unoplat_code_confluence.loader import iload_json, iparse_json from unoplat_code_confluence.loader.json_loader import JsonLoader from unoplat_code_confluence.loader.parse_json import JsonParser -from unoplat_code_confluence.markdownparser.markdownsummariser import MarkdownSummariser -from unoplat_code_confluence.summary_parser.codebase_summary import CodebaseSummaryParser -import warnings -from packaging import version +from unoplat_code_confluence.markdownparser.markdownsummariser import \ + MarkdownSummariser +from unoplat_code_confluence.summary_parser.codebase_summary import \ + CodebaseSummaryParser async def start_pipeline(): diff --git a/unoplat-code-confluence/unoplat_code_confluence/codebaseparser/arc_guard_handler.py b/unoplat-code-confluence/unoplat_code_confluence/codebaseparser/arc_guard_handler.py index 81c03c8..bbca9b4 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/codebaseparser/arc_guard_handler.py +++ b/unoplat-code-confluence/unoplat_code_confluence/codebaseparser/arc_guard_handler.py @@ -1,8 +1,14 @@ +# Standard Library import os import subprocess -from unoplat_code_confluence.utility.total_file_count import TotalFileCount + +# Third Party from loguru import logger +# First Party +from unoplat_code_confluence.utility.total_file_count import TotalFileCount + + class ArchGuardHandler: def __init__(self, jar_path, language, codebase_path,codebase_name, output_path,extension:str): self.jar_path = jar_path diff --git a/unoplat-code-confluence/unoplat_code_confluence/configuration/external_config.py b/unoplat-code-confluence/unoplat_code_confluence/configuration/external_config.py index 36b6739..f01ac11 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/configuration/external_config.py +++ b/unoplat-code-confluence/unoplat_code_confluence/configuration/external_config.py @@ -1,13 +1,11 @@ +# Standard Library from enum import Enum from typing import Any, Dict, List -from pydantic import BaseModel, ValidationInfo, field_validator, Field +# Third Party +from pydantic import BaseModel, Field, ValidationInfo, field_validator - - - - class ProgrammingLanguage(Enum): PYTHON = 'python' JAVA = 'java' diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation.py index 4637e9c..e9c8c50 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation.py @@ -1,8 +1,13 @@ -from unoplat_code_confluence.data_models.chapi_unoplat_annotation_key_val import ChapiUnoplatAnnotationKeyVal -from unoplat_code_confluence.data_models.chapi_unoplat_position import Position -from pydantic import BaseModel, Field +# Standard Library from typing import Optional +# Third Party +from pydantic import BaseModel, Field + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_annotation_key_val import \ + ChapiUnoplatAnnotationKeyVal +from unoplat_code_confluence.data_models.chapi_unoplat_position import Position class Annotation(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation_key_val.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation_key_val.py index 2cba7ab..ca652dd 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation_key_val.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_annotation_key_val.py @@ -1,8 +1,10 @@ +# Standard Library from typing import Optional +# Third Party from pydantic import BaseModel, Field diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_class_fieldmodel.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_class_fieldmodel.py index cc66b44..3d6481d 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_class_fieldmodel.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_class_fieldmodel.py @@ -1,8 +1,12 @@ -from pydantic import BaseModel, Field +# Standard Library from typing import List, Optional -from unoplat_code_confluence.data_models.chapi_unoplat_annotation import Annotation +# Third Party +from pydantic import BaseModel, Field +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_annotation import \ + Annotation class ClassFieldModel(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_codebase.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_codebase.py index 7c6f99e..b0d9d7d 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_codebase.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_codebase.py @@ -1,8 +1,12 @@ +# Standard Library from typing import List, Optional +# Third Party from pydantic import BaseModel, Field -from unoplat_code_confluence.data_models.chapi_unoplat_package import UnoplatPackage +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_package import \ + UnoplatPackage class UnoplatCodebase(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_function.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_function.py index 7455d14..d8d8b2b 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_function.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_function.py @@ -1,10 +1,18 @@ +# Standard Library +from typing import List, Optional + +# Third Party from pydantic import BaseModel, Field -from typing import Optional, List -from unoplat_code_confluence.data_models.chapi_unoplat_functioncall import FunctionCall -from unoplat_code_confluence.data_models.unoplat_function_field_model import UnoplatFunctionFieldModel -from unoplat_code_confluence.data_models.chapi_unoplat_annotation import Annotation +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_annotation import \ + Annotation +from unoplat_code_confluence.data_models.chapi_unoplat_functioncall import \ + FunctionCall from unoplat_code_confluence.data_models.chapi_unoplat_position import Position +from unoplat_code_confluence.data_models.unoplat_function_field_model import \ + UnoplatFunctionFieldModel + class ChapiUnoplatFunction(BaseModel): name: Optional[str] = Field(default=None, alias="Name") diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_functioncall.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_functioncall.py index 906cf30..5e9e0e0 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_functioncall.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_functioncall.py @@ -1,12 +1,17 @@ +# Standard Library from typing import List, Optional -from unoplat_code_confluence.data_models.chapi_unoplat_position import Position -from unoplat_code_confluence.data_models.chapi_unoplat_parameter import Parameter +# Third Party from pydantic import BaseModel, Field +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_parameter import \ + Parameter +from unoplat_code_confluence.data_models.chapi_unoplat_position import Position + class FunctionCall(BaseModel): """ diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_import.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_import.py index c403983..03678ea 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_import.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_import.py @@ -1,6 +1,8 @@ -from pydantic import BaseModel, Field -from typing import Optional, List +# Standard Library +from typing import List, Optional +# Third Party +from pydantic import BaseModel, Field class Import(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_node.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_node.py index 2b063ee..a3cff94 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_node.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_node.py @@ -1,13 +1,20 @@ +# Standard Library +from typing import List, Optional + +# Third Party from pydantic import BaseModel, Field -from typing import Optional, List -from unoplat_code_confluence.data_models.chapi_unoplat_annotation import Annotation -from unoplat_code_confluence.data_models.chapi_unoplat_class_fieldmodel import ClassFieldModel + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_annotation import \ + Annotation +from unoplat_code_confluence.data_models.chapi_unoplat_class_fieldmodel import \ + ClassFieldModel +from unoplat_code_confluence.data_models.chapi_unoplat_function import \ + ChapiUnoplatFunction from unoplat_code_confluence.data_models.chapi_unoplat_import import Import -from unoplat_code_confluence.data_models.chapi_unoplat_function import ChapiUnoplatFunction from unoplat_code_confluence.data_models.chapi_unoplat_position import Position - class ChapiUnoplatNode(BaseModel): node_name: Optional[str] = Field(default=None, alias="NodeName") type: Optional[str] = Field(default=None, alias="Type") diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_package.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_package.py index d1ab550..3b54448 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_package.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_package.py @@ -1,7 +1,12 @@ +# Standard Library from typing import Dict, List, Optional + +# Third Party from pydantic import BaseModel, Field -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode class UnoplatPackage(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_parameter.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_parameter.py index 81355d5..44bd2c9 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_parameter.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_parameter.py @@ -1,4 +1,7 @@ +# Standard Library from typing import Optional + +# Third Party from pydantic import BaseModel, Field diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_position.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_position.py index 92a410e..8434716 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_position.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/chapi_unoplat_position.py @@ -1,5 +1,8 @@ +# Standard Library from typing import Optional + +# Third Party from pydantic import BaseModel, Field diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_codebase_summary.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_codebase_summary.py index 9793eb0..27ea5fe 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_codebase_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_codebase_summary.py @@ -1,7 +1,13 @@ +# Standard Library from typing import Dict, Optional -from pydantic import BaseModel,Field -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary +# Third Party +from pydantic import BaseModel, Field + +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import \ + DspyUnoplatPackageSummary + class DspyUnoplatCodebaseSummary(BaseModel): codebase_summary: Optional[str] = Field(default=None, description="A summary of the codebase") diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_function_summary.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_function_summary.py index 45aed96..a2dc20d 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_function_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_function_summary.py @@ -1,6 +1,13 @@ -from typing import Optional, List -from pydantic import BaseModel,Field -from unoplat_code_confluence.data_models.chapi_unoplat_function import ChapiUnoplatFunction +# Standard Library +from typing import List, Optional + +# Third Party +from pydantic import BaseModel, Field + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_function import \ + ChapiUnoplatFunction + class DspyUnoplatFunctionSummary(ChapiUnoplatFunction): qualified_name: str = Field(required=True, alias="QualifiedName",description="The qualified name of the function that contains the entire hierarchy of the class") diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_node_summary.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_node_summary.py index f5025a5..8bf00f7 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_node_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_node_summary.py @@ -1,9 +1,17 @@ +# Standard Library from typing import Any, Dict, List, Optional -from pydantic import BaseModel,Field -from unoplat_code_confluence.data_models.chapi_unoplat_class_fieldmodel import ClassFieldModel -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import DspyUnoplatFunctionSummary +# Third Party +from pydantic import BaseModel, Field + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_class_fieldmodel import \ + ClassFieldModel +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import \ + DspyUnoplatFunctionSummary + class DspyUnoplatNodeSummary(ChapiUnoplatNode): qualified_name: str = Field(required=True, alias="QualifiedName",description="The qualified name of the class that contains the entire hierarchy of the class") diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_package_summary.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_package_summary.py index b272ea5..dfca493 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_package_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/dspy/dspy_unoplat_package_summary.py @@ -1,9 +1,12 @@ +# Standard Library from typing import Dict, List, Optional -from pydantic import BaseModel,Field - -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary +# Third Party +from pydantic import BaseModel, Field +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import \ + DspyUnoplatNodeSummary class DspyUnoplatPackageSummary(BaseModel): diff --git a/unoplat-code-confluence/unoplat_code_confluence/data_models/unoplat_function_field_model.py b/unoplat-code-confluence/unoplat_code_confluence/data_models/unoplat_function_field_model.py index 53c1ccf..4edb0f9 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/data_models/unoplat_function_field_model.py +++ b/unoplat-code-confluence/unoplat_code_confluence/data_models/unoplat_function_field_model.py @@ -1,4 +1,7 @@ +# Standard Library from typing import Optional + +# Third Party from pydantic import BaseModel, Field diff --git a/unoplat-code-confluence/unoplat_code_confluence/database/graph/__init__.py b/unoplat-code-confluence/unoplat_code_confluence/database/graph/__init__.py index 4f6bf67..5aede6b 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/database/graph/__init__.py +++ b/unoplat-code-confluence/unoplat_code_confluence/database/graph/__init__.py @@ -1,5 +1,9 @@ +# Standard Library +from typing import Any, Dict + +# Third Party from neo4j import GraphDatabase -from typing import Dict, Any + class Neo4jConnector: def __init__(self, uri: str, username: str, password: str): diff --git a/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_ingestion.py b/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_ingestion.py index d2b7841..2b74332 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_ingestion.py +++ b/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_ingestion.py @@ -1,7 +1,13 @@ +# Standard Library +from typing import Any, Dict + +# Third Party from neo4j import GraphDatabase -from typing import Dict, Any +from neomodel import config, db + +# First Party from unoplat_code_confluence.configuration.external_config import AppConfig -from neomodel import config,db + class UnoplatGraphIngestion: def __init__(self, app_config: AppConfig): diff --git a/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_processing.py b/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_processing.py index d5bc001..1ee2326 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_processing.py +++ b/unoplat-code-confluence/unoplat_code_confluence/database/graph/unoplat_graph_processing.py @@ -1,18 +1,35 @@ -from unoplat_code_confluence.data_models.chapi_unoplat_codebase import UnoplatCodebase -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import DspyUnoplatFunctionSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary -from unoplat_code_confluence.database.graph.unoplat_graph_ingestion import UnoplatGraphIngestion -from unoplat_code_confluence.embedding.unoplat_embedding_gen import UnoplatEmbeddingGenerator -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import DspyUnoplatCodebaseSummary -from unoplat_code_confluence.configuration.external_config import AppConfig -from unoplat_code_confluence_commons.graph_models.confluence_codebase import ConfluenceCodebase -from unoplat_code_confluence_commons.graph_models.confluence_package import ConfluencePackage -from unoplat_code_confluence_commons.graph_models.confluence_class import ConfluenceClass -from unoplat_code_confluence_commons.graph_models.confluence_method import ConfluenceMethod -from neomodel import config, db +# Standard Library import logging +# Third Party +from neomodel import config, db +from unoplat_code_confluence_commons.graph_models.confluence_class import \ + ConfluenceClass +from unoplat_code_confluence_commons.graph_models.confluence_codebase import \ + ConfluenceCodebase +from unoplat_code_confluence_commons.graph_models.confluence_method import \ + ConfluenceMethod +from unoplat_code_confluence_commons.graph_models.confluence_package import \ + ConfluencePackage + +# First Party +from unoplat_code_confluence.configuration.external_config import AppConfig +from unoplat_code_confluence.data_models.chapi_unoplat_codebase import \ + UnoplatCodebase +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import \ + DspyUnoplatCodebaseSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import \ + DspyUnoplatFunctionSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import \ + DspyUnoplatNodeSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import \ + DspyUnoplatPackageSummary +from unoplat_code_confluence.database.graph.unoplat_graph_ingestion import \ + UnoplatGraphIngestion +from unoplat_code_confluence.embedding.unoplat_embedding_gen import \ + UnoplatEmbeddingGenerator + + class UnoplatGraphProcessing: def __init__(self, app_config: AppConfig): self.unoplat_graph_ingestion = UnoplatGraphIngestion(app_config) diff --git a/unoplat-code-confluence/unoplat_code_confluence/downloader/downloader.py b/unoplat-code-confluence/unoplat_code_confluence/downloader/downloader.py index e3b406c..9d78bd6 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/downloader/downloader.py +++ b/unoplat-code-confluence/unoplat_code_confluence/downloader/downloader.py @@ -1,10 +1,14 @@ +# Standard Library import os +import re # Import regex module + +# Third Party import requests -from tqdm import tqdm from github import Github -import re # Import regex module from loguru import logger from packaging import version +from tqdm import tqdm + class Downloader: @staticmethod diff --git a/unoplat-code-confluence/unoplat_code_confluence/dspy_class_summary.py b/unoplat-code-confluence/unoplat_code_confluence/dspy_class_summary.py index 6039c8c..15d0d12 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/dspy_class_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/dspy_class_summary.py @@ -1,11 +1,18 @@ +# Standard Library from typing import Dict, List + +# Third Party import dspy +from loguru import logger +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import \ + DspyUnoplatFunctionSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import \ + DspyUnoplatNodeSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import DspyUnoplatFunctionSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary -from loguru import logger -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode #TODO: optimise using gpt4 judge and miprov2 class CodeConfluenceClassSummarySignature(dspy.Signature): diff --git a/unoplat-code-confluence/unoplat_code_confluence/dspy_codebase_summary.py b/unoplat-code-confluence/unoplat_code_confluence/dspy_codebase_summary.py index c303277..60fac27 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/dspy_codebase_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/dspy_codebase_summary.py @@ -1,6 +1,12 @@ +# Standard Library from typing import Dict + +# Third Party import dspy -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary + +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import \ + DspyUnoplatPackageSummary #TODO: optimise using gpt4 judge and miprov2 diff --git a/unoplat-code-confluence/unoplat_code_confluence/dspy_function_summary.py b/unoplat-code-confluence/unoplat_code_confluence/dspy_function_summary.py index fa65c2c..72ccaa9 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/dspy_function_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/dspy_function_summary.py @@ -1,8 +1,14 @@ +# Third Party import dspy -from unoplat_code_confluence.data_models.chapi_unoplat_function import ChapiUnoplatFunction -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode from loguru import logger +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_function import \ + ChapiUnoplatFunction +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode + + #TODO: optimise using gpt4 judge and miprov2/textgrad class CodeConfluenceFunctionSummary(dspy.Signature): """This signature takes in metadata about function in a class and returns unoplat_function_summary with all important details.""" diff --git a/unoplat-code-confluence/unoplat_code_confluence/dspy_package_summary.py b/unoplat-code-confluence/unoplat_code_confluence/dspy_package_summary.py index 91c8908..6d1d712 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/dspy_package_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/dspy_package_summary.py @@ -1,9 +1,16 @@ +# Standard Library from typing import Dict, List + +# Third Party import dspy -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary from loguru import logger +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import \ + DspyUnoplatNodeSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import \ + DspyUnoplatPackageSummary + #TODO: optimise using gpt4 judge and miprov2s class CodeConfluenceSubPackageSignature(dspy.Signature): diff --git a/unoplat-code-confluence/unoplat_code_confluence/embedding/unoplat_embedding_gen.py b/unoplat-code-confluence/unoplat_code_confluence/embedding/unoplat_embedding_gen.py index d971152..47a9456 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/embedding/unoplat_embedding_gen.py +++ b/unoplat-code-confluence/unoplat_code_confluence/embedding/unoplat_embedding_gen.py @@ -1,7 +1,13 @@ -from sentence_transformers import SentenceTransformer +# Standard Library from typing import List + +# Third Party +from sentence_transformers import SentenceTransformer + +# First Party from unoplat_code_confluence.configuration.external_config import AppConfig + class UnoplatEmbeddingGenerator: def __init__(self, config: AppConfig): self.model = SentenceTransformer(config.sentence_transformer_model, trust_remote_code=True) diff --git a/unoplat-code-confluence/unoplat_code_confluence/loader/iload_json.py b/unoplat-code-confluence/unoplat_code_confluence/loader/iload_json.py index 53d4b3c..fb63803 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/loader/iload_json.py +++ b/unoplat-code-confluence/unoplat_code_confluence/loader/iload_json.py @@ -1,5 +1,7 @@ -from abc import ABC, abstractmethod +# Standard Library import json +from abc import ABC, abstractmethod + class ILoadJson(ABC): @abstractmethod diff --git a/unoplat-code-confluence/unoplat_code_confluence/loader/iparse_json.py b/unoplat-code-confluence/unoplat_code_confluence/loader/iparse_json.py index d29889e..64da065 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/loader/iparse_json.py +++ b/unoplat-code-confluence/unoplat_code_confluence/loader/iparse_json.py @@ -1,6 +1,11 @@ +# Standard Library from abc import ABC, abstractmethod from typing import List -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode + class IParseJson(ABC): @abstractmethod diff --git a/unoplat-code-confluence/unoplat_code_confluence/loader/json_loader.py b/unoplat-code-confluence/unoplat_code_confluence/loader/json_loader.py index 47462dd..267b1f1 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/loader/json_loader.py +++ b/unoplat-code-confluence/unoplat_code_confluence/loader/json_loader.py @@ -1,8 +1,13 @@ +# Standard Library import json -from unoplat_code_confluence.loader.iload_json import ILoadJson +# Third Party from loguru import logger +# First Party +from unoplat_code_confluence.loader.iload_json import ILoadJson + + class JsonLoader(ILoadJson): @logger.catch def load_json_from_file(self, file_path: str) -> json: diff --git a/unoplat-code-confluence/unoplat_code_confluence/loader/parse_json.py b/unoplat-code-confluence/unoplat_code_confluence/loader/parse_json.py index cafae67..c5fe632 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/loader/parse_json.py +++ b/unoplat-code-confluence/unoplat_code_confluence/loader/parse_json.py @@ -1,12 +1,22 @@ +# Standard Library import os from typing import Dict, List + +# Third Party +from loguru import logger from pydantic import ValidationError -from unoplat_code_confluence.data_models.chapi_unoplat_codebase import UnoplatCodebase -from unoplat_code_confluence.data_models.chapi_unoplat_package import UnoplatPackage + +# First Party +from unoplat_code_confluence.data_models.chapi_unoplat_codebase import \ + UnoplatCodebase +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode +from unoplat_code_confluence.data_models.chapi_unoplat_package import \ + UnoplatPackage from unoplat_code_confluence.loader.iparse_json import IParseJson -from loguru import logger from unoplat_code_confluence.markdownparser.isummariser import ISummariser -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode + + class JsonParser(IParseJson): def parse_json_to_nodes(self, json_data: dict, local_workspace_path: str, programming_language: str) -> UnoplatCodebase: """Concrete implementation of the parse_json_to_nodes method.""" diff --git a/unoplat-code-confluence/unoplat_code_confluence/markdownparser/isummariser.py b/unoplat-code-confluence/unoplat_code_confluence/markdownparser/isummariser.py index d9c7325..808d0ef 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/markdownparser/isummariser.py +++ b/unoplat-code-confluence/unoplat_code_confluence/markdownparser/isummariser.py @@ -1,5 +1,10 @@ +# Standard Library from abc import ABC, abstractmethod -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import DspyUnoplatCodebaseSummary + +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import \ + DspyUnoplatCodebaseSummary + class ISummariser(ABC): diff --git a/unoplat-code-confluence/unoplat_code_confluence/markdownparser/markdownsummariser.py b/unoplat-code-confluence/unoplat_code_confluence/markdownparser/markdownsummariser.py index 06d2616..32f8d02 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/markdownparser/markdownsummariser.py +++ b/unoplat-code-confluence/unoplat_code_confluence/markdownparser/markdownsummariser.py @@ -1,6 +1,8 @@ -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import DspyUnoplatCodebaseSummary +# First Party +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import \ + DspyUnoplatCodebaseSummary from unoplat_code_confluence.markdownparser.isummariser import ISummariser diff --git a/unoplat-code-confluence/unoplat_code_confluence/summary_parser/codebase_summary.py b/unoplat-code-confluence/unoplat_code_confluence/summary_parser/codebase_summary.py index 7e7ad0c..1880956 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/summary_parser/codebase_summary.py +++ b/unoplat-code-confluence/unoplat_code_confluence/summary_parser/codebase_summary.py @@ -1,28 +1,43 @@ +# Standard Library import asyncio -from collections import deque import datetime -from itertools import cycle import sys +from collections import deque +from itertools import cycle from typing import Dict, List -from unoplat_code_confluence.configuration.external_config import AppConfig -from unoplat_code_confluence.data_models.chapi_unoplat_codebase import UnoplatCodebase -from unoplat_code_confluence.data_models.chapi_unoplat_package import UnoplatPackage -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import DspyUnoplatCodebaseSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import DspyUnoplatFunctionSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import DspyUnoplatNodeSummary -from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import DspyUnoplatPackageSummary -from unoplat_code_confluence.dspy_class_summary import CodeConfluenceClassModule -from unoplat_code_confluence.dspy_codebase_summary import CodeConfluenceCodebaseModule, CodeConfluenceCodebaseObjectiveSignature -from unoplat_code_confluence.dspy_function_summary import CodeConfluenceFunctionModule -from unoplat_code_confluence.dspy_package_summary import CodeConfluencePackageModule + +# Third Party import dspy from loguru import logger from progiter import ProgIter from progiter.manager import ProgressManager -from unoplat_code_confluence.data_models.chapi_unoplat_node import ChapiUnoplatNode -from unoplat_code_confluence.data_models.chapi_unoplat_function import ChapiUnoplatFunction - +# First Party +from unoplat_code_confluence.configuration.external_config import AppConfig +from unoplat_code_confluence.data_models.chapi_unoplat_codebase import \ + UnoplatCodebase +from unoplat_code_confluence.data_models.chapi_unoplat_function import \ + ChapiUnoplatFunction +from unoplat_code_confluence.data_models.chapi_unoplat_node import \ + ChapiUnoplatNode +from unoplat_code_confluence.data_models.chapi_unoplat_package import \ + UnoplatPackage +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_codebase_summary import \ + DspyUnoplatCodebaseSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_function_summary import \ + DspyUnoplatFunctionSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_node_summary import \ + DspyUnoplatNodeSummary +from unoplat_code_confluence.data_models.dspy.dspy_unoplat_package_summary import \ + DspyUnoplatPackageSummary +from unoplat_code_confluence.dspy_class_summary import \ + CodeConfluenceClassModule +from unoplat_code_confluence.dspy_codebase_summary import ( + CodeConfluenceCodebaseModule, CodeConfluenceCodebaseObjectiveSignature) +from unoplat_code_confluence.dspy_function_summary import \ + CodeConfluenceFunctionModule +from unoplat_code_confluence.dspy_package_summary import \ + CodeConfluencePackageModule class CodebaseSummaryParser: diff --git a/unoplat-code-confluence/unoplat_code_confluence/utility/total_file_count.py b/unoplat-code-confluence/unoplat_code_confluence/utility/total_file_count.py index 48ae284..b425657 100644 --- a/unoplat-code-confluence/unoplat_code_confluence/utility/total_file_count.py +++ b/unoplat-code-confluence/unoplat_code_confluence/utility/total_file_count.py @@ -1,7 +1,11 @@ -import os +# Standard Library import glob +import os + +# Third Party from loguru import logger + class TotalFileCount: def __init__(self, directory, extension): self.directory = directory