Improve OCI to better support docker.io login and image change detection

CHANGES.d/20240214_133728_cz_HEAD.md

@@ -0,0 +1 @@
+- OCI: Support registries where the docker login is different than the registry used in referencing containers.

CHANGES.d/20240214_133815_cz_HEAD.md

@@ -0,0 +1 @@
+- OCI: Improve change detection of remote images (required for docker.io)

CHANGES.d/20240214_133847_cz_HEAD.md

@@ -0,0 +1 @@
+- OCI: The nix file does not contain sensitive data, so don’t mark it as such.

CHANGES.d/20240313_112633_cz_oci_codestlye.md

@@ -0,0 +1 @@
+- OCI: add support for extraOptions

src/batou_ext/nix.py

@@ -612,6 +612,8 @@ def value_to_nix(value):
         return nix_dict_to_nix(host_to_nix_dict(value))
     elif isinstance(value, Environment):
         return nix_dict_to_nix(environment_to_nix_dict(value))
+    elif isinstance(value, batou.utils.Timer):
+        return None  # ignore
     else:
         raise TypeError(f"unsupported type '{type(value)}'")
 

src/batou_ext/oci.py

@@ -1,62 +1,62 @@
-import hashlib
 import os
-import re
 import shlex
+from textwrap import dedent
 from typing import Optional
 
 import batou
-import pkg_resources
 from batou import UpdateNeeded
 from batou.component import Attribute, Component
 from batou.lib.file import File
-from batou.lib.service import Service
+from batou.utils import CmdExecutionError
 
 import batou_ext.nix
 
 
 @batou_ext.nix.rebuild
 class Container(Component):
-    """
-    A OCI Container component.
-    With this component you can dynamically schedule docker containers to be run on the target host
+    """A OCI Container component.
+
+    With this component you can dynamically schedule docker containers to be
+    run on the target host.
 
     Note: Docker image specifiers do not follow a properly resolvable pattern.
-        Therefore, container registries have to be specified seperately if you need to log in before.
-        If you do not provide a container registry, docker will use the default one for authentication.
-        You can choose to also append the image attribute with the registry but this module will do so
-        automatically.
-
-        ```
-        # the following two call are identical:
-        self += batou_ext.oci.Container(
-            image="foo",
-            registry_address="test.registry",
-            registry_user="foo",
-            registry_password="bar")
-
-        self += batou_ext.oci.Container(
-            image="test.registry/foo",
-            registry_address="test.registry",
-            registry_user="foo",
-            registry_password="bar")
-
-        # However, this will fail since docker will try to log into the default container registry
-        # with the provided credentials and then pull the image from the registry specified in the image string
-        self += batou_ext.oci.Container(
-            image="test.registry/foo",
-            registry_user="foo",
-            registry_password="bar")
-
-
-        # if you don't need to authenticate, not explicitly specifying the registry is fine of course
-        # and it will pull the image from the correct registry
-        self += batou_ext.oci.Container(image="test.registry/foo")
-        ```
+    Therefore, container registries have to be specified seperately if you need
+    to log in before. If you do not provide a container registry, docker will
+    use the default one for authentication. You can choose to also append the
+    image attribute with the registry but this module will do so automatically.
+
+    ```
+    # the following two call are identical:
+    self += batou_ext.oci.Container(
+        image="foo",
+        registry_address="test.registry",
+        registry_user="foo",
+        registry_password="bar")
+
+    self += batou_ext.oci.Container(
+        image="test.registry/foo",
+        registry_address="test.registry",
+        registry_user="foo",
+        registry_password="bar")
+
+    # However, this will fail since docker will try to log into the default container registry
+    # with the provided credentials and then pull the image from the registry specified in the image string
+    self += batou_ext.oci.Container(
+        image="test.registry/foo",
+        registry_user="foo",
+        registry_password="bar")
+
+
+    # if you don't need to authenticate, not explicitly specifying the registry is fine of course
+    # and it will pull the image from the correct registry
+    self += batou_ext.oci.Container(image="test.registry/foo")
+    ```
 
     Example:
     ```
     self += batou_ext.oci.Container(image = "mysql", version = "8.0")
     ```
+
     """
 
     # general options
@@ -71,6 +71,8 @@ class Container(Component):
     mounts: dict = {}
     ports: dict = {}
     env: dict = {}
+    depends_on: list = None
+    extra_options: list = None
 
     # secrets
     registry_address = Attribute(Optional[str], None)
@@ -85,15 +87,18 @@ def configure(self):
         if (
             self.registry_user or self.registry_password
         ) and not self.registry_address:
-            batou.output.annotate(
-                "WARN: you might want to specify the registry explicitly unless you really intend to log into the default docker registry"
+            self.log(
+                "WARN: you might want to specify the registry explicitly"
+                " unless you really intend to log into the default"
+                " docker registry"
             )
 
-        if self.version:
-            self.image = f"{self.image}:{self.version}"
-
-        if self.registry_address and not self.image.startswith(
+        if (
             self.registry_address
+            # This is for the strange case of index.docker.io where you have
+            # to set the registry_address to `https://index.docker.io/v1/`
+            and not self.registry_address.startswith("https://")
+            and not self.image.startswith(self.registry_address)
         ):
             self.image = f"{self.registry_address}{'/' if not self.registry_address.endswith('/') else ''}{self.image}"
 
@@ -116,9 +121,12 @@ def configure(self):
         if self.docker_cmd:
             self._docker_cmd_list = shlex.split(self.docker_cmd)
 
+        if not self.depends_on:
+            self.depends_on = []
+
         self += File(
             f"/etc/local/nixos/docker_{self.container_name}.nix",
-            sensitive_data=True,
+            sensitive_data=False,
             source=os.path.join(
                 os.path.dirname(__file__), "resources/oci-template.nix"
             ),
@@ -131,31 +139,51 @@ def verify(self):
         if self.registry_address:
             logintxt, _ = self.cmd(
                 self.expand(
-                    """
-                    docker login \\
-                        {%- if component.registry_user and component.registry_password %}
-                        -u {{component.registry_user}} \\
-                        -p {{component.registry_password}} \\
-                        {%- endif %}
-                        {{component.registry_address}}
-                """
+                    dedent(
+                        """\
+        docker login \\
+            {%- if component.registry_user and component.registry_password %}
+            -u {{component.registry_user}} \\
+            -p {{component.registry_password}} \\
+            {%- endif %}
+            {{component.registry_address}}
+        """
+                    )
                 )
             )
 
         local_digest, stderr = self.cmd(
-            "docker image inspect {{component.image}} | jq -r 'first | .RepoDigests | first | split(\"@\") | last' || echo image not available locally"
-        )
-        remote_digest, stderr = self.cmd(
-            "docker manifest inspect {{component.image}} -v | jq -r 'if type ==\"array\" then (. | first) else . end | .Descriptor.digest'"
+            dedent(
+                """\
+            docker image inspect {{component.image}}:{{component.version}} \
+                | jq -r 'first | .RepoDigests | first | split("@") | last' \
+                || echo image not available locally
+                """
+            )
         )
-
-        # `docker manifest inspect` silently raises an error, returns code 0 when unathorized
+        try:
+            self.cmd(
+                "docker manifest inspect"
+                f" {self.image}:{self.version}@{local_digest}"
+            )
+        except CmdExecutionError as e:
+            valid = False
+            error = e.stderr
+            if error.startswith("unsupported manifest format"):  # gitlab
+                batou.output.annotate(error, debug=True)
+                error = error[:50]
+        else:
+            valid = True
+
+        # `docker manifest inspect` silently raises an error, returns code 0
+        # when unathorized
         if stderr == "unauthorized":
             raise RuntimeError(
                 "Wrong credentials for remote container registry"
             )
 
-        if local_digest != remote_digest:
+        if not valid:
+            self.log("Update due digest verification error: %r", error)
             raise UpdateNeeded()
 
     def update(self):

src/batou_ext/resources/oci-template.nix

@@ -36,22 +36,34 @@
         # {% endif %}
       };
 
-      extraOptions = [ "--pull=always" ];
+      extraOptions = [
+        "--pull=always"
+        # {% for option in (component.extra_options or []) %}
+        "{{option}}"
+        # {% endfor %}
+      ];
 
       volumes = [
       # {% for key, value in component.mounts.items() | sort  %}
         "{{key}}:{{value}}"
       # {% endfor %}
       ];
 
-      image = "{{component.image}}";
+      image = "{{component.image}}:{{component.version}}";
       environmentFiles = [ {{component.envfile.path}} ];
 
       ports = [
       # {% for key, value in component.ports.items() | sort  %}
         "{{key}}:{{value}}"
       # {% endfor %}
       ];
+
+      dependsOn = [
+      # {% for value in component.depends_on  %}
+        "{{value}}"
+      # {% endfor %}
+
+      ];
     };
   };
 }