diff --git a/pyproject.toml b/pyproject.toml index 570e14104..0f1a9f0d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,9 @@ dependencies = ["chardet", "numba>=0.59.0; python_version=='3.12'", "scipy", "scitrack", + "stevedore", "tqdm", - "typing_extensions", - "stevedore"] + "typing_extensions"] # remember to update version in requires-python and classifiers requires-python = ">=3.9,<3.13" classifiers = [ diff --git a/src/cogent3/app/__init__.py b/src/cogent3/app/__init__.py index 7e5beafab..e71f8f196 100644 --- a/src/cogent3/app/__init__.py +++ b/src/cogent3/app/__init__.py @@ -14,9 +14,8 @@ from .composable import is_app, is_app_composable from .io import open_data_store - -# from stevedore.extension import ExtensionManager - +# Entry_point for apps to register themselves as plugins +APP_ENTRY_POINT = "cogent3.app" def _get_extension_attr(extension): """ @@ -60,29 +59,22 @@ def _make_types(app) -> dict: __apps = None -def apps(force: bool = False) -> stevedore.ExtensionManager: +def get_app_manager(force: bool = False) -> stevedore.ExtensionManager: """ Lazy load a stevedore ExtensionManager to collect apps. - Parameters - ---------- - force : bool, optional - If set to True, forces the re-creation of the ExtensionManager, - even if it already exists. Default is False. - - Returns - ------- - ExtensionManager - The ExtensionManager instance that collects apps. + Notes + ----- + If force is set, the app_manager will refresh it's cache of apps. """ global __apps if force: importlib.invalidate_caches() - importlib.reload(stevedore) - stevedore.extension.ExtensionManager.ENTRY_POINT_CACHE.clear() + if APP_ENTRY_POINT in stevedore.extension.ExtensionManager.ENTRY_POINT_CACHE: + del stevedore.extension.ExtensionManager.ENTRY_POINT_CACHE[APP_ENTRY_POINT] if not __apps or force: __apps = stevedore.ExtensionManager( - namespace="cogent3.app", invoke_on_load=False + namespace=APP_ENTRY_POINT, invoke_on_load=False ) return __apps @@ -90,13 +82,9 @@ def apps(force: bool = False) -> stevedore.ExtensionManager: def available_apps(name_filter: str | None = None, force: bool = False) -> Table: """returns Table listing the available apps - Parameters - ---------- - name_filter - include apps whose name includes name_filter - force : bool, optional - If set to True, forces the re-creation of the ExtensionManager, - even if it already exists. Default is False. + Notes + ----- + If force is set, the app_manager will refresh it's cache of apps. """ from cogent3.util.table import Table @@ -105,7 +93,7 @@ def available_apps(name_filter: str | None = None, force: bool = False) -> Table rows = [] - extensions = apps(force) + extensions = get_app_manager(force) for extension in extensions: if any(extension.name.startswith(d) for d in deprecated): @@ -189,7 +177,7 @@ def _get_app_matching_name(name: str): else: modname = None - extensions_matching = [extension for extension in apps() if extension.name == name] + extensions_matching = [extension for extension in get_app_manager() if extension.name == name] if not extensions_matching: raise ValueError(f"App {name!r} not found. Please check for typos.") @@ -210,24 +198,10 @@ def _get_app_matching_name(name: str): def get_app(_app_name: str, *args, **kwargs): """returns app instance, use app_help() to display arguments - Parameters - ---------- - _app_name - app name, e.g. 'minlength', or can include module information, - e.g. 'cogent3.app.sample.minlength' or 'sample.minlength'. Use the - latter (qualified class name) style when there are multiple matches - to name. - *args, **kwargs - positional and keyword arguments passed through to the app - - Returns - ------- - cogent3 app instance - Raises ------ NameError when multiple apps have the same name. In that case use a - qualified class name, as shown above. + qualified class name. """ return _get_app_matching_name(_app_name)(*args, **kwargs) diff --git a/tests/test_app/test_init.py b/tests/test_app/test_init.py index 25b32ef8a..de20510a7 100644 --- a/tests/test_app/test_init.py +++ b/tests/test_app/test_init.py @@ -137,27 +137,6 @@ def test_app_help(capsys): assert "Options" in got assert got.count("bytes") >= 2 # both input and output types are bytes - -@pytest.mark.xfail(reason="Constructing apps on the fly is no longer supported") -def test_app_help_doctest_examples(capsys): - app_doc = "A line of text describing the app." - init_doc = "\n Parameters\n ----------\n arg\n arg description\n\n Examples\n --------\n How to use the app\n\n >>>blah(arg)\n output\n" - blah.__doc__ = app_doc - blah.__init__.__doc__ = init_doc - app_help("blah") - got = capsys.readouterr().out - # Two new lines after the end of Parameters should be preserved - expect1 = "arg description\n\nExamples\n--------\n" - # Two new lines within Examples are preserved - expect2 = "How to use the app\n\n>>>blah(arg)\noutput\n" - # Two new lines at the end of Examples preserved - expect3 = "output\n\nInput type" - - assert expect1 in got - assert expect2 in got - assert expect3 in got - - @pytest.mark.parametrize( "app_name", ("bootstrap", "from_primitive", "load_db", "take_named_seqs")[:1] ) diff --git a/tests/test_app/test_install_plugins.py b/tests/test_app/test_install_plugins.py index 0bda1f3a6..aded89c51 100644 --- a/tests/test_app/test_install_plugins.py +++ b/tests/test_app/test_install_plugins.py @@ -10,42 +10,65 @@ # def _requires_imports(cls): # imports required by the app # return ["from cogent3.app import typing as c3types", "import json"] # -# Note: Due to the fact that it uses import_lib caches stevedore will see the first -# installed app immediately, but subsequent apps may not be seen until the cache is -# updated or the interpreter reloaded. +# Note: Due to the fact that it uses importlib, stevedore will see the first +# installed app immediately, but subsequent apps may not be seen until the +# importlib cache is eventually updated or the interpreter reloaded. +# importlib.invalidate_caches() does not appear to work as expected. import importlib import json import os +from pathlib import Path import site import sys import time -from contextlib import contextmanager, suppress -from importlib.metadata import distributions -from inspect import getsourcelines, isclass -from os import path -from shutil import rmtree +from contextlib import contextmanager +from inspect import getsourcelines from subprocess import check_call -from tempfile import mkdtemp -import pytest -import cogent3 +import pytest -from cogent3.app import apps, available_apps, get_app +from cogent3.app import APP_ENTRY_POINT, get_app_manager, get_app from cogent3.app.composable import define_app +from cogent3.app import typing as c3types - -def is_package_installed(package_name): +def is_package_installed(package_name: str) -> bool: """Check if a package is installed""" site_packages_dir = site.getsitepackages()[1] installed_packages = os.listdir(site_packages_dir) - return package_name in installed_packages + return any(package_name in pkg for pkg in installed_packages) +def is_package_imported(package_name: str) -> bool: + """Check if a package is imported""" + try: + importlib.import_module(package_name) + return True + except ImportError as e: + return False + +def is_app_installed(module_name: str, app_name: str) -> bool: + """Check if app is installed""" + app_manager = get_app_manager(force=True) + return any(ext.entry_point.value == f"{module_name}:{app_name}" for ext in app_manager) + +@contextmanager +def persist_for(seconds:int, operation_name:str="Operation"): + """A context manager that yields until the operation is complete or the time limit is reached.""" + start_time = time.time() + try: + def perform_operation(operation): + while time.time() - start_time < seconds and not operation(): + time.sleep(1) + yield perform_operation + finally: + elapsed_time = time.time() - start_time + if elapsed_time > seconds: + raise TimeoutError(f"{operation_name} timed out after {seconds} seconds.") -def install_app(cls, temp_dir, mod=None): +def install_app(cls, temp_dir: Path, mod=None): """ Installs a temporary app created from a class, by writing the source code into a new Python file in the temporary directory, creating a setup.py file @@ -53,20 +76,31 @@ def install_app(cls, temp_dir, mod=None): """ # Get the source code of the function lines, _ = getsourcelines(cls) - source_code = "\n".join(lines) + + # Find the first line of the class definition + class_line = next(i for i, line in enumerate(lines) if line.strip().startswith("class")) + + # Calculate the number of leading spaces in the first line of the class definition + leading_spaces = len(lines[class_line]) - len(lines[class_line].lstrip()) + + + # Remove leading spaces from all lines + source_code = "\n".join(line[leading_spaces:] for line in lines) if not mod: mod = f"mod_{cls.__name__}" - + # Write the source code into a new Python file in the temporary directory - with open(path.join(temp_dir, f"{mod}.py"), "w") as f: - f.write("from cogent3.app.composable import define_app\n") + package_path = temp_dir / f"{mod}.py" + with open(package_path, "w") as package_file: + package_file.write("from cogent3.app.composable import define_app\n") if hasattr(cls, "_requires_imports"): for import_statement in cls._requires_imports(): - f.write(f"{import_statement}\n") - f.write(source_code) + package_file.write(f"{import_statement}\n") + package_file.write(source_code) # Create a setup.py file in the temporary directory that references the new module - setup_py = f""" + setup_path = temp_dir / "setup.py" + setup_contents = f""" from setuptools import setup setup( @@ -74,62 +108,26 @@ def install_app(cls, temp_dir, mod=None): version='0.1', py_modules=['{mod}'], entry_points={{ - "cogent3.app": [ + "{APP_ENTRY_POINT}": [ "{cls.__name__} = {mod}:{cls.__name__}", ], }}, ) """ - with open(path.join(temp_dir, "setup.py"), "w") as f: - f.write(setup_py) + with open(setup_path, "w") as setup_file: + setup_file.write(setup_contents) # Use subprocess to run pip install . in the temporary directory check_call([sys.executable, "-m", "pip", "install", "."], cwd=temp_dir) - # Wait for the installation to complete - timeout = 60 # maximum time to wait in seconds - start_time = time.time() - - start_time = time.time() - package_name = mod - - while True: - if is_package_installed(package_name=package_name): - break - elif time.time() - start_time > timeout: - raise TimeoutError( - f"Package {package_name!r} not found after waiting for {timeout} seconds." - ) - time.sleep(1) + with persist_for(seconds=60, operation_name=f"Installing package {mod}") as repeat_until: + repeat_until(lambda: is_package_installed(package_name=mod)) - # check we can import it - timeout = 60 # maximum time to wait in seconds - start_time = time.time() - - while True: - with suppress(ImportError): - importlib.import_module(package_name) - break - if time.time() - start_time > timeout: - raise TimeoutError( - f"Package {package_name!r} not importable after waiting for {timeout} seconds." - ) - time.sleep(1) - - # check it's in the stevedore cache - timeout = 60 # maximum time to wait in seconds - start_time = time.time() - - while True: - appcache = apps(force=True) - if any(ext.entry_point.value == f"{mod}:{cls.__name__}" for ext in appcache): - break - elif time.time() - start_time > timeout: - raise TimeoutError( - f"App {mod}.{cls.__name__} not available after waiting for {timeout} seconds." - ) - time.sleep(1) + with persist_for(seconds=60, operation_name=f"Importing package {mod}") as repeat_until: + repeat_until(lambda: is_package_imported(package_name=mod)) + with persist_for(seconds=60, operation_name=f"Importing package {mod}") as repeat_until: + repeat_until(lambda: is_app_installed(module_name=mod,app_name=cls.__name__)) def uninstall_app(cls, mod=None): """ @@ -137,120 +135,49 @@ def uninstall_app(cls, mod=None): """ if not mod: mod = f"mod_{cls.__name__}" - # Use subprocess to run pip uninstall -y in the temporary directory - package_name = mod.replace("_", "-") # replace underscores with hyphens + package_name = mod.replace("_", "-") # underscores in package names are replaced with hyphens check_call([sys.executable, "-m", "pip", "uninstall", "-y", package_name]) - # force the app cache to reload - _ = available_apps(force=True) - + _ = get_app_manager(force=True) # force the app cache to reload @contextmanager -def temp_app(cls, module_name: str = None): +def load_app(app_class, tmp_path, module_name: str = None): """ - A context manager that creates a temporary app from a class, installs it, - and then uninstalls it after testing. - - Parameters - ---------- - cls : object - The class from which to create the app. - module_name : str, optional - The name of the module in which the app is defined. If None, a name - is generated based on the class name. - - Yields - ------ - None - - Raises - ------ - pytest.fail - If a TimeoutError occurs during app installation. - - Notes - ----- - The app is installed in a temporary directory, which is deleted after testing. + A context manager that creates a temporary app from a class, in a passed in path, + installs it, yields for tests then uninstalls the app. Usage ----- - with temp_app(cls): + with temp_app(cls,tmp_path): myapp = get_app(cls.__name__) - # test myapp + # test myapp ... """ if module_name is None: - module_name = f"mod_{cls.__name__}" - temp_dir = mkdtemp() + module_name = f"mod_{app_class.__name__}" try: try: - install_app(cls, temp_dir=temp_dir, mod=module_name) + install_app(app_class, temp_dir=tmp_path, mod=module_name) except TimeoutError as e: pytest.fail(e.args[0]) except Exception as e: pytest.fail(f"App installation failed: {e}") yield finally: - uninstall_app(cls, mod=module_name) - rmtree(temp_dir) - - -@pytest.fixture(scope="function") -def install_temp_app(request: pytest.FixtureRequest): - """ - A pytest fixture that creates a temporary app from a class, installs it, - and then uninstalls it after testing. - - Parameters - ---------- - request : pytest.FixtureRequest - The fixture request object. The class from which to create the app - should be passed as a parameter to the test function using this fixture. - - Yields - ------ - None - - Raises - ------ - TypeError - If the parameter passed to the test function is not a class. + uninstall_app(app_class, mod=module_name) - Notes - ----- - The app is installed in a temporary directory, which is deleted after testing. - - Usage - ----- - - @pytest.mark.parametrize("install_temp_app", [MyAppClass], indirect=True) - def test_function(install_temp_app): - myapp = get_app('MyAppClass') - # test myapp - """ - cls = request.param - if not isclass(cls): - raise TypeError(f"Expected a class, but got {type(cls).__name__}") - - with temp_app(cls): - yield - - -@pytest.mark.xfail( - reason="This test is expected to fail due to a bug with changing from using pkg_resources to importlib" -) -def test_install_temp_app(): +def test_install_temp_app(tmp_path: Path): @define_app class MyAppClass: - def main(self, data: cogent3.app.typing.SerialisableType) -> str: + def main(self, data: c3types.SerialisableType) -> str: return json.dumps(data) @classmethod def _requires_imports(cls): return ["from cogent3.app import typing as c3types", "import json"] - with temp_app(MyAppClass): + with load_app(MyAppClass, tmp_path): myapp = get_app("MyAppClass") data = {"key": "value", "number": 42, "list": [1, 2, 3], "boolean": True} assert ( myapp(data) == '{"key": "value", "number": 42, "list": [1, 2, 3], "boolean": true}' - ) + ) \ No newline at end of file diff --git a/tests/test_app/test_plugins.py b/tests/test_app/test_plugins.py index ae84700e3..1d5d4852a 100644 --- a/tests/test_app/test_plugins.py +++ b/tests/test_app/test_plugins.py @@ -62,7 +62,7 @@ def create_extension( ) -def test_Install_app_class(mock_extension_manager): +def test_install_app_class(mock_extension_manager): @define_app class uppercase: """Test app that converts a string to uppercase""" @@ -77,7 +77,7 @@ def main(self, data: str) -> str: assert appercase.__doc__ in _make_apphelp_docstring(appercase.__class__) -def test_Install_app_function(mock_extension_manager): +def test_install_app_function(mock_extension_manager): @define_app def uppercase(data: str) -> str: """Test function that converts a string to uppercase""" @@ -105,7 +105,7 @@ def main(self, val: int) -> int: mock_extension_manager([create_extension(documented_app)]) - assert cogent3.app.apps().names() == ["documented_app"] + assert cogent3.app.get_app_manager().names() == ["documented_app"] app = get_app("documented_app") app.__class__.__doc__ = app_doc app.__class__.__init__.__doc__ = init_doc @@ -133,7 +133,7 @@ def main(self, data: str) -> str: ] ) - assert cogent3.app.apps().names() == ["app1", "app1"] + assert cogent3.app.get_app_manager().names() == ["app1", "app1"] with pytest.raises(NameError): _ = get_app( @@ -159,5 +159,5 @@ def dummy(val: int) -> int: mock_extension_manager([create_extension(dummy)]) apps = available_apps() assert isinstance(apps, Table) - apps.filtered(lambda x: dummy.__name__ == x, columns="name") + apps = apps.filtered(lambda x: dummy.__name__ == x, columns="name") assert apps.shape[0] == 1