diff --git a/README.md b/README.md index f0f9f8b..79922f7 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,10 @@ Options: --odoo-version TEXT Project's Odoo version (e.g. 16.0) [required] --project-name TEXT Name of the project, will be the name of category of local addons (default: Local) - --oca-category Add category for third party addons coming from OCA + --oca-category Add category for third party addons coming from OCA. + The category is set as 'OCA/repository_name'. + If the repository can not be identified, + it falls into the default 'OCA' category. --reset-cache Purge cache used to identify OCA addons --help Show this message and exit. ``` diff --git a/pyproject.toml b/pyproject.toml index f36ca1e..5705748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ dependencies = [ "diskcache", "requests", "platformdirs", + "packaging", + "mousebender", ] [project.urls] diff --git a/src/odoo_sort_manifest_depends/sort_manifest_deps.py b/src/odoo_sort_manifest_depends/sort_manifest_deps.py index 81f9bb1..172f681 100644 --- a/src/odoo_sort_manifest_depends/sort_manifest_deps.py +++ b/src/odoo_sort_manifest_depends/sort_manifest_deps.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT from pathlib import Path -from re import DOTALL, sub +from re import DOTALL, search, sub import click from click import command, option @@ -12,13 +12,25 @@ from manifestoo_core.core_addons import is_core_ce_addon, is_core_ee_addon from manifestoo_core.metadata import addon_name_to_distribution_name from manifestoo_core.odoo_series import OdooSeries +from mousebender import simple +from packaging.specifiers import SpecifierSet +from packaging.utils import parse_wheel_filename from platformdirs import user_cache_dir -from requests import head +from requests import get, head NAME_DEFAULT_CATEGORY = "Default" OCA_ADDONS_INDEX_URL = "https://wheelhouse.odoo-community.org/oca-simple/" REQUEST_TIMEOUT = 2 # s +PYPI_SIMPLE_INDEX_URL = "https://pypi.org/simple/" +PAGE_NOT_FOUND = 404 + + +def _get_pypi_url(distribution_name: str, version: str) -> str: + return f"https://pypi.org/pypi/{distribution_name}/{version}/json" + + +oca_category_repo_not_found = "OCA" other_addons_category_cache = Cache(user_cache_dir("odoo-sort-manifest-depends", "Acsone")) @@ -42,8 +54,8 @@ def _get_addons_by_name(addons_dir: Path) -> dict[str, Addon]: return local_addons -def _identify_oca_addons(addon_names: list[str], odoo_series: OdooSeries) -> tuple[list[str], list[str]]: - oca_addons, other_addons = [], [] +def _identify_oca_addons(addon_names: list[str], odoo_series: OdooSeries) -> tuple[dict[str, list[str]], list[str]]: + oca_addons_by_category, other_addons = {}, [] with other_addons_category_cache as cache: for addon_name in addon_names: @@ -53,17 +65,65 @@ def _identify_oca_addons(addon_names: list[str], odoo_series: OdooSeries) -> tup distribution_name = addon_name_to_distribution_name(addon_name, odoo_series).replace("_", "-") res = head(f"{OCA_ADDONS_INDEX_URL}{distribution_name}", timeout=REQUEST_TIMEOUT) if res: - category = "oca" + category = get_oca_repository_name(addon_name, odoo_series) or oca_category_repo_not_found else: category = "other" cache[addon_name] = category - if category == "oca": - oca_addons.append(addon_name) - else: + if category == "other": other_addons.append(addon_name) + else: + oca_addons_by_category.setdefault(category, []).append(addon_name) + + return oca_addons_by_category, other_addons + + +def get_oca_repository_name(addon_name: str, odoo_series: OdooSeries) -> str | None: + specifier = SpecifierSet(f"=={odoo_series.value}.*") + distribution_name = addon_name_to_distribution_name(addon_name, odoo_series) + # get avaialble releases + project_url = simple.create_project_url(PYPI_SIMPLE_INDEX_URL, distribution_name) + response = get(project_url, headers={"Accept": simple.ACCEPT_JSON_V1}, timeout=REQUEST_TIMEOUT) + if response.status_code == PAGE_NOT_FOUND: + # project not found + return None + response.raise_for_status() + data = response.text + content_type = response.headers["Content-Type"] + project_details = simple.parse_project_details(data, content_type, distribution_name) + # find the first version that matches the requested Odoo version; + # we assume all releases come from the same repo for a given Odoo series + for file in project_details["files"]: + if file.get("yanked"): + continue + filename = file["filename"] + if not filename.endswith(".whl"): + continue + _, version, _, _ = parse_wheel_filename(filename) + if specifier.contains(version, prereleases=True): + # found a release that matches the requested Odoo version + break + else: + # no release found that matches the requested Odoo version + return None + + pypi_json_url = _get_pypi_url(distribution_name, version) + response = get(pypi_json_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + repo_url = response.json().get("info").get("home_page") + + if not repo_url: + return None + return search("OCA/.*", repo_url).group() + - return oca_addons, other_addons +def _add_oca_categories( + categories: dict[str, list[str]], other: list[str], odoo_series: OdooSeries +) -> tuple[dict[str, list[str]], list[str]]: + oca_addons_by_category, other = _identify_oca_addons(other, odoo_series) + for category, oca_addons in {key: sorted(value) for key, value in sorted(oca_addons_by_category.items())}.items(): + categories[category] = oca_addons + return categories, other def do_sorting(addons_dir: Path, odoo_version: str, project_name: str, *, oca_category: bool) -> None: @@ -121,10 +181,8 @@ def do_sorting(addons_dir: Path, odoo_version: str, project_name: str, *, oca_ca "Odoo Enterprise": odoo_ee, } - # Third party if oca_category: - oca, other = _identify_oca_addons(other, odoo_series) - categories["OCA"] = oca + categories, other = _add_oca_categories(categories, other, odoo_series) categories["Third-party"] = other @@ -164,7 +222,9 @@ def do_sorting(addons_dir: Path, odoo_version: str, project_name: str, *, oca_ca @option( "--oca-category", is_flag=True, - help="Add category for third party addons coming from OCA", + help="Add category for third party addons coming from OCA. " + "The category is set as 'OCA/repository_name'. " + "If the repository can not be identified, it falls into the default 'OCA' category.", ) @option( "--reset-cache", @@ -181,5 +241,11 @@ def sort_manifest_deps( ) -> None: if reset_cache: other_addons_category_cache.clear() + elif other_addons_category_cache: + # Remove addons from cache that have 'oca_category_repo_not_found' as category + with other_addons_category_cache as cache: + # 'oca' is for retrocompatibility with cache created in versions < v1.4 + cache.evict(oca_category_repo_not_found) + cache.evict("oca") do_sorting(Path(local_addons_dir), odoo_version, project_name, oca_category=oca_category)