From 4f560676e1bed7b8cd939dd6770938c7a5c4cc4e Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 10 Jan 2022 03:37:05 +0100 Subject: [PATCH] feat(updater): improve generic update strategy --- ops2deb.yml | 30 +++++++------- poetry.lock | 20 ++++++++- pyproject.toml | 1 + src/ops2deb/generator.py | 2 +- src/ops2deb/jinja.py | 5 --- src/ops2deb/parser.py | 2 +- src/ops2deb/templates.py | 5 +++ src/ops2deb/updater.py | 88 +++++++++++++++++++++++++++------------- tests/test_updater.py | 65 +++++++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 52 deletions(-) delete mode 100644 src/ops2deb/jinja.py create mode 100644 tests/test_updater.py diff --git a/ops2deb.yml b/ops2deb.yml index 18529bb..5d4757d 100644 --- a/ops2deb.yml +++ b/ops2deb.yml @@ -1,5 +1,5 @@ - name: helm - version: 3.7.1 + version: 3.7.2 homepage: https://helm.sh/ summary: The Kubernetes package manager description: |- @@ -10,14 +10,14 @@ fetch: url: https://get.helm.sh/helm-v{{version}}-linux-{{goarch}}.tar.gz sha256: - amd64: 6cd6cad4b97e10c33c978ff3ac97bb42b68f79766f1d2284cfd62ec04cd177f4 - armhf: e035e0022cf5c49d08c08371364aae9eebe614a20035856e7370a85ded8db790 - arm64: 57875be56f981d11957205986a57c07432e54d0b282624d68b1aeac16be70704 + amd64: 4ae30e48966aba5f807a4e140dad6736ee1a392940101e4d79ffb4ee86200a9e + armhf: ab73727f1c00903aff010a3557ab4366a1a13ce2d243c9cb191e703fbb76c915 + arm64: b0214eabbb64791f563bd222d17150ce39bf4e2f5de49f49fdb456ce9ae8162f script: - mv linux-*/helm {{src}}/usr/bin/ - name: helmfile - version: 0.142.0 + version: 0.143.0 homepage: https://github.com/roboll/helmfile summary: Deploy Kubernetes Helm Charts description: |- @@ -32,13 +32,13 @@ fetch: url: https://github.com/roboll/helmfile/releases/download/v{{version}}/helmfile_linux_{{goarch}} sha256: - amd64: ec01b884cbb9d072f4a6784b33a29f3311248e85e714ebacabba85a79a3ba3db - arm64: 92e1da5dc8628830c725c39a2e991b7c1bf957396b7dd8127ceb34ec6607324e + amd64: 829b2b27aa4d7111f6c5b047ca162d2a9aef76b6646bb31c895850683a86c2c1 + arm64: 91fae17f3b43dcfd34b0cc8b0d69eb3937b13d0e3ddca96dcada58bdafc1ccf4 script: - install -m 755 helmfile_linux_* {{src}}/usr/bin/helmfile - name: istioctl - version: 1.12.0 + version: 1.12.1 homepage: https://istio.io/ summary: Istio service mesh CLI description: Istio is an open platform to connect, manage, and secure microservices. @@ -46,14 +46,14 @@ - kubectl fetch: url: https://github.com/istio/istio/releases/download/{{version}}/istio-{{version}}-linux-amd64.tar.gz - sha256: 7b219d9d0f48bd92901ef4cacb048c10c9f812dd4af120b0e3b353c006b3226e + sha256: 7b1279810a77590bd7af60d6d26074a89c32e6aff7512fdc37f38b093e34e382 script: - install -d {{src}}/opt/istio - mv * {{src}}/opt/istio/ - ln -s /opt/istio/istio-{{version}}/bin/istioctl {{src}}/usr/bin/istioctl - name: kubectl - version: 1.22.4 + version: 1.23.1 homepage: https://github.com/kubernetes/kubectl summary: Command line client for controlling a Kubernetes cluster description: |- @@ -61,7 +61,7 @@ clusters. fetch: url: https://storage.googleapis.com/kubernetes-release/release/v{{version}}/bin/linux/amd64/kubectl - sha256: 21f24aa723002353eba1cc2668d0be22651f9063f444fd01626dce2b6e1c568c + sha256: 156fd5e7ebbedf3c482fd274089ad75a448b04cf42bc53f370e4e4ea628f705e script: - mv kubectl {{src}}/usr/bin/ @@ -79,7 +79,7 @@ - install -m 755 kubeseal-linux-amd64 {{src}}/usr/bin/kubeseal - name: kustomize - version: 3.8.10 + version: 4.4.1 homepage: https://kustomize.io/ summary: Kubernetes native configuration management description: |- @@ -92,7 +92,7 @@ - kubectl fetch: url: https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v{{version}}/kustomize_v{{version}}_linux_amd64.tar.gz - sha256: 10281b6cd16a50fcbb4a762652bf5ab333633d37035fc7f76ee7b941b50b511d + sha256: 2d5927efec40ba32a121c49f6df9955b8b8a296ef1dec4515a46fc84df158798 script: - mv kustomize {{src}}/usr/bin/ @@ -116,7 +116,7 @@ - install -m 755 minikube-linux-amd64 {{src}}/usr/bin/minikube - name: ops2deb - version: 0.15.0 + version: 0.16.0 homepage: https://github.com/upciti/ops2deb summary: Debian packaging tool for portable applications description: |- @@ -131,6 +131,6 @@ - debhelper fetch: url: https://github.com/upciti/ops2deb/releases/download/{{version}}/ops2deb_linux_amd64.tar.gz - sha256: 6ff97d7359d414106d4b8246356a4547a9731a36b6b1f514b241ec7175d8304a + sha256: 211bc67e965cda9ab7cdcb227cf114f5114eaa85689eda55a3833d297aea698c script: - mv ops2deb {{src}}/usr/bin/ diff --git a/poetry.lock b/poetry.lock index 092d16f..c9de98d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -488,6 +488,20 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.16.0" +description = "Pytest support for asyncio." +category = "dev" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +pytest = ">=5.4.0" + +[package.extras] +testing = ["coverage", "hypothesis (>=5.7.1)"] + [[package]] name = "pytest-cov" version = "3.0.0" @@ -719,7 +733,7 @@ pyinstaller = ["pyinstaller"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "1dc253c595c752cd581342686f63b2724eef8343c86977b08aa79af93098ff39" +content-hash = "b8d5c5e7526fb97c55b1c4f1a584afa29a1169a6d7f3f3f7b9bb66f8de6010d0" [metadata.files] aiofiles = [ @@ -1086,6 +1100,10 @@ pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] +pytest-asyncio = [ + {file = "pytest-asyncio-0.16.0.tar.gz", hash = "sha256:7496c5977ce88c34379df64a66459fe395cd05543f0a2f837016e7144391fcfb"}, + {file = "pytest_asyncio-0.16.0-py3-none-any.whl", hash = "sha256:5f2a21273c47b331ae6aa5b36087047b4899e40f03f18397c0e65fa5cca54e9b"}, +] pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, diff --git a/pyproject.toml b/pyproject.toml index 698501c..6fe64f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ isort = "*" safety = "*" types-aiofiles = "*" types-PyYAML = "*" +pytest-asyncio = "^0.16.0" [tool.taskipy.tasks] check = """ diff --git a/src/ops2deb/generator.py b/src/ops2deb/generator.py index f433143..85dd580 100644 --- a/src/ops2deb/generator.py +++ b/src/ops2deb/generator.py @@ -7,8 +7,8 @@ from .apt import DebianRepositoryPackage, sync_list_repository_packages from .exceptions import Ops2debGeneratorError, Ops2debGeneratorScriptError from .fetcher import Fetcher, FetchResult, FetchResultOrError -from .jinja import environment from .parser import Blueprint, extend +from .templates import environment from .utils import separate_successes_from_errors diff --git a/src/ops2deb/jinja.py b/src/ops2deb/jinja.py deleted file mode 100644 index 1664d3b..0000000 --- a/src/ops2deb/jinja.py +++ /dev/null @@ -1,5 +0,0 @@ -from jinja2 import Environment, FunctionLoader - -from .templates import template_loader - -environment = Environment(loader=FunctionLoader(template_loader)) diff --git a/src/ops2deb/parser.py b/src/ops2deb/parser.py index b4fb930..f9804ca 100644 --- a/src/ops2deb/parser.py +++ b/src/ops2deb/parser.py @@ -5,7 +5,7 @@ from ruamel.yaml import YAML, YAMLError from .exceptions import Ops2debParserError -from .jinja import environment +from .templates import environment Architecture = Literal["all", "amd64", "arm64", "armhf"] diff --git a/src/ops2deb/templates.py b/src/ops2deb/templates.py index 5018437..3378305 100644 --- a/src/ops2deb/templates.py +++ b/src/ops2deb/templates.py @@ -1,5 +1,7 @@ from typing import Optional +from jinja2 import Environment, FunctionLoader + DEBIAN_CHANGELOG = """\ {{ package.name }} ({{ package.version }}) stable; urgency=medium @@ -61,3 +63,6 @@ def template_loader(name: str) -> Optional[str]: if variable_name in globals(): return template_content return None + + +environment = Environment(loader=FunctionLoader(template_loader)) diff --git a/src/ops2deb/updater.py b/src/ops2deb/updater.py index 119ce55..812a0f9 100644 --- a/src/ops2deb/updater.py +++ b/src/ops2deb/updater.py @@ -33,7 +33,7 @@ def is_new(self) -> bool: return self.blueprint.version != self.version -class BaseUpdater: +class BaseUpdateStrategy: def __init__(self, client: httpx.AsyncClient): self.client = client @@ -45,32 +45,58 @@ async def __call__(self, blueprint: Blueprint) -> str: raise NotImplementedError -class GenericUpdater(BaseUpdater): - async def _bump_and_poll( +class GenericUpdateStrategy(BaseUpdateStrategy): + """ + Tries a few blueprint fetch URLs with bumped versions to see if servers + replies with something else than a 404. More or less a brute force approach. + """ + + async def _try_version( + self, blueprint: Blueprint, version: Version + ) -> Optional[Version]: + if not (remote_file := blueprint.render_fetch(version=str(version))): + return None + url = remote_file.url + logger.debug(f"Trying {url}") + try: + response = await self.client.head(url) + except httpx.HTTPError as e: + raise Ops2debUpdaterError(f"Failed HEAD request to {url}. {str(e)}") + status = response.status_code + if status >= 500: + raise Ops2debUpdaterError(f"Server error when requesting {url}") + elif status >= 400: + return None + return version + + async def _try_a_few_patches( + self, blueprint: Blueprint, version: Version + ) -> Optional[Version]: + for i in range(0, 3): + version = version.bump_patch() + if await self._try_version(blueprint, version) is not None: + return version + return None + + async def _try_versions( self, blueprint: Blueprint, version: Version, - bump_patch: bool = False, + version_part: str, ) -> Version: - new_version = version - while True: - version = version.bump_patch() if bump_patch else version.bump_minor() - if not (remote_file := blueprint.render_fetch(version=str(version))): - break - url = remote_file.url - logger.debug(f"Trying {url}") - try: - response = await self.client.head(url) - except httpx.HTTPError as e: - raise Ops2debUpdaterError(f"Failed HEAD request to {url}. {str(e)}") - status = response.status_code - if status >= 500: - raise Ops2debUpdaterError(f"Server error when requesting {url}") - if status >= 400: - break + bumped_version = getattr(version, f"bump_{version_part}")() + if (result := await self._try_version(blueprint, bumped_version)) is None: + if version_part != "patch": + if ( + result := await self._try_a_few_patches(blueprint, bumped_version) + ) is not None: + return await self._try_versions(blueprint, result, version_part) + else: + return version else: - new_version = version - return new_version + return version + else: + return await self._try_versions(blueprint, result, version_part) @classmethod def is_blueprint_supported(cls, blueprint: Blueprint) -> bool: @@ -80,18 +106,22 @@ def is_blueprint_supported(cls, blueprint: Blueprint) -> bool: return True async def __call__(self, blueprint: Blueprint) -> str: - version = Version.parse(blueprint.version) - version = await self._bump_and_poll(blueprint, version, False) - version = await self._bump_and_poll(blueprint, version, True) + current_version = version = Version.parse(blueprint.version) + for version_part in ["minor", "patch"]: + version = await self._try_versions(blueprint, version, version_part) + if version == current_version: + version = await self._try_versions(blueprint, version, "major") return str(version) async def _find_latest_version(client: httpx.AsyncClient, blueprint: Blueprint) -> str: - updaters = [GenericUpdater(client)] - updaters = [u for u in updaters if u.is_blueprint_supported(blueprint)] - for updater in updaters: + strategies = [GenericUpdateStrategy(client)] + strategies = [u for u in strategies if u.is_blueprint_supported(blueprint)] + if not strategies: + return blueprint.version + for update_strategy in strategies: try: - return await updater(blueprint) + return await update_strategy(blueprint) except Ops2debUpdaterError as e: logger.debug(str(e)) continue diff --git a/tests/test_updater.py b/tests/test_updater.py new file mode 100644 index 0000000..5414684 --- /dev/null +++ b/tests/test_updater.py @@ -0,0 +1,65 @@ +from typing import List + +import pytest +from httpx import AsyncClient +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import Response + +from ops2deb.logger import enable_debug +from ops2deb.parser import Blueprint, RemoteFile +from ops2deb.updater import GenericUpdateStrategy + +enable_debug(True) + + +@pytest.fixture +def app_factory(): + def _app_response(request: Request): + return Response(status_code=200) + + def _app_factory(versions: List[str]): + app = Starlette(debug=True) + for version in versions: + app.add_route( + f"/releases/{version}/some-app.tar.gz", _app_response, ["HEAD", "GET"] + ) + return app + + return _app_factory + + +@pytest.fixture +def dummy_blueprint(): + return Blueprint( + name="some-app", + version="1.0.0", + summary="some summary", + description="some description", + fetch=RemoteFile( + url="http://test/releases/{{version}}/some-app.tar.gz", sha256="deadbeef" + ), + ) + + +@pytest.mark.parametrize( + "versions,expected_result", + [ + (["1.0.0", "1.1.0"], "1.1.0"), + (["1.0.0", "1.1.3"], "1.1.3"), + (["1.0.0", "1.0.1", "1.1.0"], "1.1.0"), + (["1.0.0", "1.1.1", "2.0.0"], "1.1.1"), + (["1.0.0", "2.0.0"], "2.0.0"), + (["1.0.0", "2.0.3"], "2.0.3"), + (["1.0.0", "1.1.0", "2.0.0"], "1.1.0"), + (["1.0.0", "1.0.1", "1.0.2", "1.1.0", "1.1.1"], "1.1.1"), + ], +) +@pytest.mark.asyncio +async def test_generic_update_strategy_should_find_expected_blueprint_release( + dummy_blueprint, app_factory, versions, expected_result +): + app = app_factory(versions) + async with AsyncClient(app=app) as client: + update_strategy = GenericUpdateStrategy(client) + assert await update_strategy(dummy_blueprint) == expected_result