feat(updater): improve generic update strategy

ops2deb.yml

@@ -1,5 +1,5 @@
 - name: helm
-  version: 3.7.1
+  version: 3.7.2
   homepage: https://helm.sh/
   summary: The Kubernetes package manager
   description: |-
@@ -10,14 +10,14 @@
   fetch:
     url: https://get.helm.sh/helm-v{{version}}-linux-{{goarch}}.tar.gz
     sha256:
-      amd64: 6cd6cad4b97e10c33c978ff3ac97bb42b68f79766f1d2284cfd62ec04cd177f4
-      armhf: e035e0022cf5c49d08c08371364aae9eebe614a20035856e7370a85ded8db790
-      arm64: 57875be56f981d11957205986a57c07432e54d0b282624d68b1aeac16be70704
+      amd64: 4ae30e48966aba5f807a4e140dad6736ee1a392940101e4d79ffb4ee86200a9e
+      armhf: ab73727f1c00903aff010a3557ab4366a1a13ce2d243c9cb191e703fbb76c915
+      arm64: b0214eabbb64791f563bd222d17150ce39bf4e2f5de49f49fdb456ce9ae8162f
   script:
     - mv linux-*/helm {{src}}/usr/bin/
 
 - name: helmfile
-  version: 0.142.0
+  version: 0.143.0
   homepage: https://github.com/roboll/helmfile
   summary: Deploy Kubernetes Helm Charts
   description: |-
@@ -32,36 +32,36 @@
   fetch:
     url: https://github.com/roboll/helmfile/releases/download/v{{version}}/helmfile_linux_{{goarch}}
     sha256:
-      amd64: ec01b884cbb9d072f4a6784b33a29f3311248e85e714ebacabba85a79a3ba3db
-      arm64: 92e1da5dc8628830c725c39a2e991b7c1bf957396b7dd8127ceb34ec6607324e
+      amd64: 829b2b27aa4d7111f6c5b047ca162d2a9aef76b6646bb31c895850683a86c2c1
+      arm64: 91fae17f3b43dcfd34b0cc8b0d69eb3937b13d0e3ddca96dcada58bdafc1ccf4
   script:
     - install -m 755 helmfile_linux_* {{src}}/usr/bin/helmfile
 
 - name: istioctl
-  version: 1.12.0
+  version: 1.12.1
   homepage: https://istio.io/
   summary: Istio service mesh CLI
   description: Istio is an open platform to connect, manage, and secure microservices.
   depends:
     - kubectl
   fetch:
     url: https://github.com/istio/istio/releases/download/{{version}}/istio-{{version}}-linux-amd64.tar.gz
-    sha256: 7b219d9d0f48bd92901ef4cacb048c10c9f812dd4af120b0e3b353c006b3226e
+    sha256: 7b1279810a77590bd7af60d6d26074a89c32e6aff7512fdc37f38b093e34e382
   script:
     - install -d {{src}}/opt/istio
     - mv * {{src}}/opt/istio/
     - ln -s /opt/istio/istio-{{version}}/bin/istioctl {{src}}/usr/bin/istioctl
 
 - name: kubectl
-  version: 1.22.4
+  version: 1.23.1
   homepage: https://github.com/kubernetes/kubectl
   summary: Command line client for controlling a Kubernetes cluster
   description: |-
     kubectl is a command line client for running commands against Kubernetes
     clusters.
   fetch:
     url: https://storage.googleapis.com/kubernetes-release/release/v{{version}}/bin/linux/amd64/kubectl
-    sha256: 21f24aa723002353eba1cc2668d0be22651f9063f444fd01626dce2b6e1c568c
+    sha256: 156fd5e7ebbedf3c482fd274089ad75a448b04cf42bc53f370e4e4ea628f705e
   script:
     - mv kubectl {{src}}/usr/bin/
 
@@ -79,7 +79,7 @@
     - install -m 755 kubeseal-linux-amd64 {{src}}/usr/bin/kubeseal
 
 - name: kustomize
-  version: 3.8.10
+  version: 4.4.1
   homepage: https://kustomize.io/
   summary: Kubernetes native configuration management
   description: |-
@@ -92,7 +92,7 @@
     - kubectl
   fetch:
     url: https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v{{version}}/kustomize_v{{version}}_linux_amd64.tar.gz
-    sha256: 10281b6cd16a50fcbb4a762652bf5ab333633d37035fc7f76ee7b941b50b511d
+    sha256: 2d5927efec40ba32a121c49f6df9955b8b8a296ef1dec4515a46fc84df158798
   script:
     - mv kustomize {{src}}/usr/bin/
 
@@ -116,7 +116,7 @@
     - install -m 755 minikube-linux-amd64 {{src}}/usr/bin/minikube
 
 - name: ops2deb
-  version: 0.15.0
+  version: 0.16.0
   homepage: https://github.com/upciti/ops2deb
   summary: Debian packaging tool for portable applications
   description: |-
@@ -131,6 +131,6 @@
     - debhelper
   fetch:
     url: https://github.com/upciti/ops2deb/releases/download/{{version}}/ops2deb_linux_amd64.tar.gz
-    sha256: 6ff97d7359d414106d4b8246356a4547a9731a36b6b1f514b241ec7175d8304a
+    sha256: 211bc67e965cda9ab7cdcb227cf114f5114eaa85689eda55a3833d297aea698c
   script:
     - mv ops2deb {{src}}/usr/bin/

pyproject.toml

@@ -48,6 +48,7 @@ isort = "*"
 safety = "*"
 types-aiofiles = "*"
 types-PyYAML = "*"
+pytest-asyncio = "^0.16.0"
 
 [tool.taskipy.tasks]
 check = """

src/ops2deb/generator.py

@@ -7,8 +7,8 @@
 from .apt import DebianRepositoryPackage, sync_list_repository_packages
 from .exceptions import Ops2debGeneratorError, Ops2debGeneratorScriptError
 from .fetcher import Fetcher, FetchResult, FetchResultOrError
-from .jinja import environment
 from .parser import Blueprint, extend
+from .templates import environment
 from .utils import separate_successes_from_errors
 
 

src/ops2deb/parser.py

@@ -5,7 +5,7 @@
 from ruamel.yaml import YAML, YAMLError
 
 from .exceptions import Ops2debParserError
-from .jinja import environment
+from .templates import environment
 
 Architecture = Literal["all", "amd64", "arm64", "armhf"]
 

src/ops2deb/templates.py

@@ -1,5 +1,7 @@
 from typing import Optional
 
+from jinja2 import Environment, FunctionLoader
+
 DEBIAN_CHANGELOG = """\
 {{ package.name }} ({{ package.version }}) stable; urgency=medium
 
@@ -61,3 +63,6 @@ def template_loader(name: str) -> Optional[str]:
     if variable_name in globals():
         return template_content
     return None
+
+
+environment = Environment(loader=FunctionLoader(template_loader))

src/ops2deb/updater.py

@@ -33,7 +33,7 @@ def is_new(self) -> bool:
         return self.blueprint.version != self.version
 
 
-class BaseUpdater:
+class BaseUpdateStrategy:
     def __init__(self, client: httpx.AsyncClient):
         self.client = client
 
@@ -45,32 +45,58 @@ async def __call__(self, blueprint: Blueprint) -> str:
         raise NotImplementedError
 
 
-class GenericUpdater(BaseUpdater):
-    async def _bump_and_poll(
+class GenericUpdateStrategy(BaseUpdateStrategy):
+    """
+    Tries a few blueprint fetch URLs with bumped versions to see if servers
+    replies with something else than a 404. More or less a brute force approach.
+    """
+
+    async def _try_version(
+        self, blueprint: Blueprint, version: Version
+    ) -> Optional[Version]:
+        if not (remote_file := blueprint.render_fetch(version=str(version))):
+            return None
+        url = remote_file.url
+        logger.debug(f"Trying {url}")
+        try:
+            response = await self.client.head(url)
+        except httpx.HTTPError as e:
+            raise Ops2debUpdaterError(f"Failed HEAD request to {url}. {str(e)}")
+        status = response.status_code
+        if status >= 500:
+            raise Ops2debUpdaterError(f"Server error when requesting {url}")
+        elif status >= 400:
+            return None
+        return version
+
+    async def _try_a_few_patches(
+        self, blueprint: Blueprint, version: Version
+    ) -> Optional[Version]:
+        for i in range(0, 3):
+            version = version.bump_patch()
+            if await self._try_version(blueprint, version) is not None:
+                return version
+        return None
+
+    async def _try_versions(
         self,
         blueprint: Blueprint,
         version: Version,
-        bump_patch: bool = False,
+        version_part: str,
     ) -> Version:
-        new_version = version
-        while True:
-            version = version.bump_patch() if bump_patch else version.bump_minor()
-            if not (remote_file := blueprint.render_fetch(version=str(version))):
-                break
-            url = remote_file.url
-            logger.debug(f"Trying {url}")
-            try:
-                response = await self.client.head(url)
-            except httpx.HTTPError as e:
-                raise Ops2debUpdaterError(f"Failed HEAD request to {url}. {str(e)}")
-            status = response.status_code
-            if status >= 500:
-                raise Ops2debUpdaterError(f"Server error when requesting {url}")
-            if status >= 400:
-                break
+        bumped_version = getattr(version, f"bump_{version_part}")()
+        if (result := await self._try_version(blueprint, bumped_version)) is None:
+            if version_part != "patch":
+                if (
+                    result := await self._try_a_few_patches(blueprint, bumped_version)
+                ) is not None:
+                    return await self._try_versions(blueprint, result, version_part)
+                else:
+                    return version
             else:
-                new_version = version
-        return new_version
+                return version
+        else:
+            return await self._try_versions(blueprint, result, version_part)
 
     @classmethod
     def is_blueprint_supported(cls, blueprint: Blueprint) -> bool:
@@ -80,18 +106,22 @@ def is_blueprint_supported(cls, blueprint: Blueprint) -> bool:
         return True
 
     async def __call__(self, blueprint: Blueprint) -> str:
-        version = Version.parse(blueprint.version)
-        version = await self._bump_and_poll(blueprint, version, False)
-        version = await self._bump_and_poll(blueprint, version, True)
+        current_version = version = Version.parse(blueprint.version)
+        for version_part in ["minor", "patch"]:
+            version = await self._try_versions(blueprint, version, version_part)
+        if version == current_version:
+            version = await self._try_versions(blueprint, version, "major")
         return str(version)
 
 
 async def _find_latest_version(client: httpx.AsyncClient, blueprint: Blueprint) -> str:
-    updaters = [GenericUpdater(client)]
-    updaters = [u for u in updaters if u.is_blueprint_supported(blueprint)]
-    for updater in updaters:
+    strategies = [GenericUpdateStrategy(client)]
+    strategies = [u for u in strategies if u.is_blueprint_supported(blueprint)]
+    if not strategies:
+        return blueprint.version
+    for update_strategy in strategies:
         try:
-            return await updater(blueprint)
+            return await update_strategy(blueprint)
         except Ops2debUpdaterError as e:
             logger.debug(str(e))
             continue

tests/test_updater.py

@@ -0,0 +1,65 @@
+from typing import List
+
+import pytest
+from httpx import AsyncClient
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import Response
+
+from ops2deb.logger import enable_debug
+from ops2deb.parser import Blueprint, RemoteFile
+from ops2deb.updater import GenericUpdateStrategy
+
+enable_debug(True)
+
+
+@pytest.fixture
+def app_factory():
+    def _app_response(request: Request):
+        return Response(status_code=200)
+
+    def _app_factory(versions: List[str]):
+        app = Starlette(debug=True)
+        for version in versions:
+            app.add_route(
+                f"/releases/{version}/some-app.tar.gz", _app_response, ["HEAD", "GET"]
+            )
+        return app
+
+    return _app_factory
+
+
+@pytest.fixture
+def dummy_blueprint():
+    return Blueprint(
+        name="some-app",
+        version="1.0.0",
+        summary="some summary",
+        description="some description",
+        fetch=RemoteFile(
+            url="http://test/releases/{{version}}/some-app.tar.gz", sha256="deadbeef"
+        ),
+    )
+
+
+@pytest.mark.parametrize(
+    "versions,expected_result",
+    [
+        (["1.0.0", "1.1.0"], "1.1.0"),
+        (["1.0.0", "1.1.3"], "1.1.3"),
+        (["1.0.0", "1.0.1", "1.1.0"], "1.1.0"),
+        (["1.0.0", "1.1.1", "2.0.0"], "1.1.1"),
+        (["1.0.0", "2.0.0"], "2.0.0"),
+        (["1.0.0", "2.0.3"], "2.0.3"),
+        (["1.0.0", "1.1.0", "2.0.0"], "1.1.0"),
+        (["1.0.0", "1.0.1", "1.0.2", "1.1.0", "1.1.1"], "1.1.1"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_generic_update_strategy_should_find_expected_blueprint_release(
+    dummy_blueprint, app_factory, versions, expected_result
+):
+    app = app_factory(versions)
+    async with AsyncClient(app=app) as client:
+        update_strategy = GenericUpdateStrategy(client)
+        assert await update_strategy(dummy_blueprint) == expected_result