From c8e3fd33ac037d4c64b76420d5881b84a7327c8f Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 17 Apr 2024 16:29:48 +0200 Subject: [PATCH 1/2] Bug 1891921 - part 1: Bump mozilla-version --- api/poetry.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 0453ae39f..a308e4d83 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1469,13 +1469,13 @@ six = "*" [[package]] name = "mozilla-version" -version = "2.1.0" +version = "3.0.0" description = "Process Firefox versions numbers. Tells whether they are valid or not, whether they are nightlies or regular releases, whether this version precedes that other." optional = false python-versions = "*" files = [ - {file = "mozilla-version-2.1.0.tar.gz", hash = "sha256:564bd0787951eb34c648af5e9785b38db394039f727530f10e69bb28063016e9"}, - {file = "mozilla_version-2.1.0-py3-none-any.whl", hash = "sha256:168ed5ff22b0bf577b09176e370bad7fe433c91b1141f3f765d5d7987526b71b"}, + {file = "mozilla_version-3.0.0-py3-none-any.whl", hash = "sha256:63c25a8cec007e8ca401e0a9aa343fe873679354819d4db6ad1e30c60b85787f"}, + {file = "mozilla_version-3.0.0.tar.gz", hash = "sha256:93162faa1768f14c39c7f347efd0dab74e7ba75a63f78f765f60eb567a1b3154"}, ] [package.dependencies] @@ -1976,6 +1976,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, From a9b27c1004e385f65433bce360bb8101926916ab Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 17 Apr 2024 12:37:17 +0200 Subject: [PATCH 2/2] Bug 1891921 - part 2: Use mozilla-version to determine the category of a release --- api/src/shipit_api/admin/product_details.py | 99 ++++----------------- api/src/shipit_api/common/product.py | 14 +++ 2 files changed, 33 insertions(+), 80 deletions(-) diff --git a/api/src/shipit_api/admin/product_details.py b/api/src/shipit_api/admin/product_details.py index a9eaff669..fdf8259bf 100644 --- a/api/src/shipit_api/admin/product_details.py +++ b/api/src/shipit_api/admin/product_details.py @@ -13,7 +13,6 @@ import logging import os import pathlib -import re import shutil import typing import urllib.parse @@ -32,7 +31,7 @@ import shipit_api.common.config import shipit_api.common.models from shipit_api.admin.release import parse_version -from shipit_api.common.product import Product, ProductCategory +from shipit_api.common.product import Product, ProductCategory, get_product_category logger = logging.getLogger(__name__) @@ -259,44 +258,6 @@ def get_releases_from_db(db_session: sqlalchemy.orm.Session, breakpoint_version: return query.all() -def get_product_categories(product: Product, version: str) -> typing.List[ProductCategory]: - # typically, these are dot releases that are considered major - SPECIAL_FIREFOX_MAJORS = ["14.0.1", "125.0.1"] - SPECIAL_THUNDERBIRD_MAJORS = ["14.0.1", "38.0.1"] - - def patternize_versions(versions): - if not versions: - return "" - return "|" + "|".join([v.replace(r".", r"\.") for v in versions]) - - categories = [] - categories_mapping: typing.List[typing.Tuple[ProductCategory, str]] = [] - - if product is Product.THUNDERBIRD: - special_majors = patternize_versions(SPECIAL_THUNDERBIRD_MAJORS) - else: - special_majors = patternize_versions(SPECIAL_FIREFOX_MAJORS) - - categories_mapping.append((ProductCategory.MAJOR, r"([0-9]+\.[0-9]+%s)$" % special_majors)) - categories_mapping.append((ProductCategory.MAJOR, r"([0-9]+\.[0-9]+(esr|)%s)$" % special_majors)) - categories_mapping.append((ProductCategory.STABILITY, r"([0-9]+\.[0-9]+\.[0-9]+$|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$)")) - categories_mapping.append((ProductCategory.STABILITY, r"([0-9]+\.[0-9]+\.[0-9]+(esr|)$|[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(esr|)$)")) - # We had 38.0.5b2 - categories_mapping.append((ProductCategory.DEVELOPMENT, r"([0-9]+\.[0-9]|[0-9]+\.[0-9]+\.[0-9])(b|rc|build|plugin)[0-9]+$")) - - # Ugly hack to manage the next ESR (when we have two overlapping esr) - if shipit_api.common.config.ESR_NEXT: - categories_mapping.append((ProductCategory.ESR, shipit_api.common.config.ESR_NEXT + r"(\.[0-9]+){1,2}esr$")) - else: - categories_mapping.append((ProductCategory.ESR, shipit_api.common.config.CURRENT_ESR + r"(\.[0-9]+){1,2}esr$")) - - for product_category, version_pattern in categories_mapping: - if re.match(version_pattern, version): - categories.append(product_category) - - return categories - - def get_releases( breakpoint_version: int, products: Products, releases: typing.List[shipit_api.common.models.Release], old_product_details: ProductDetails ) -> Releases: @@ -358,20 +319,22 @@ def get_releases( for release in releases: if release.product != product.value: continue - categories = get_product_categories(Product(release.product), release.version) + release_version = release.version - for category in categories: - if release_version.endswith("esr"): - release_version = release_version[: -len("esr")] - details[f"{release.product}-{release.version}"] = dict( - category=category.value, - product=release.product, - build_number=release.build_number, - description=None, - is_security_driven=False, # TODO: we don't have this field anymore - version=release_version, - date=with_default(release.completed, functools.partial(to_format, format="YYYY-MM-DD"), default=""), - ) + version = parse_version(product, release_version) + category = get_product_category(version) + + if release_version.endswith("esr"): + release_version = release_version[: -len("esr")] + details[f"{release.product}-{release.version}"] = dict( + category=category.value, + product=release.product, + build_number=release.build_number, + description=None, + is_security_driven=False, # TODO: we don't have this field anymore + version=release_version, + date=with_default(release.completed, functools.partial(to_format, format="YYYY-MM-DD"), default=""), + ) return dict(releases=details) @@ -438,6 +401,8 @@ def get_release_history( version = parse_version(product, version_string) if version.major_number >= breakpoint_version: continue + if get_product_category(version) != product_category: + continue history[version_string] = old_history[version_string] # @@ -454,33 +419,7 @@ def get_release_history( if release_version is None or release_version.major_number < breakpoint_version: continue - # short term hack: 125.0.1 is a major release. we should replace this with - # something that uses MozillaVersion to determine categories - if ( - product_category is ProductCategory.MAJOR - and release_version.major_number == 125 - and release_version.patch_number == 1 - and release_version.beta_number is None - and not release_version.is_esr - ): - # history_version is a copy of stuff further down - we need it now, before - # this release gets skipped - history_version = release.version - if history_version.endswith("esr"): - history_version = history_version[: -len("esr")] - history[history_version] = with_default(release.completed, functools.partial(to_format, format="YYYY-MM-DD"), default="") - continue - - # skip all releases which don't fit into product category - if product_category is ProductCategory.MAJOR and ( - release_version.patch_number is not None or release_version.beta_number is not None or release_version.is_esr - ): - continue - - elif product_category is ProductCategory.DEVELOPMENT and (release_version.beta_number is None or release_version.is_esr): - continue - - elif product_category is ProductCategory.STABILITY and (release_version.beta_number is not None or release_version.patch_number is None): + if get_product_category(release_version) != product_category: continue history_version = release.version diff --git a/api/src/shipit_api/common/product.py b/api/src/shipit_api/common/product.py index ac6740e8a..f276aaffa 100644 --- a/api/src/shipit_api/common/product.py +++ b/api/src/shipit_api/common/product.py @@ -1,5 +1,7 @@ import enum +from mozilla_version.gecko import GeckoVersion + from backend_common import get_product_names @@ -11,6 +13,18 @@ class ProductCategory(enum.Enum): ESR = "esr" +def get_product_category(version: GeckoVersion): + if version.is_major: + return ProductCategory.MAJOR + elif version.is_development: + return ProductCategory.DEVELOPMENT + elif version.is_stability: + return ProductCategory.STABILITY + elif version.is_esr: + return ProductCategory.ESR + raise ValueError(f"Unknown category for version: {version}") + + # Keys of Product will have underscores where the product name may have hyphens. # So all hyphens are translated to underscores as a rule. def get_key(name):