Custom env variables for model builder (#1100)

* pydantic

* WORKFLOW_GENERATOR_CUSTOM_MODEL_BUILDER_ENVS

* Black reformating

gordo/cli/workflow_generator.py

@@ -1,15 +1,17 @@
 import logging
 import time
 import pkg_resources
-import json
 import os
 
-from typing import Dict, Any
+from typing import Dict, Any, TypeVar, Type, List
 
 import click
+import json
 
+from pydantic import parse_obj_as, ValidationError
 from gordo import __version__
 from gordo.workflow.config_elements.normalized_config import NormalizedConfig
+from gordo.workflow.config_elements.schemas import CustomEnv
 from gordo.workflow.workflow_generator import workflow_generator as wg
 from gordo.cli.exceptions_reporter import ReportLevel
 from gordo.util.version import parse_version
@@ -42,6 +44,45 @@ def get_builder_exceptions_report_level(config: NormalizedConfig) -> ReportLevel
     return report_level
 
 
+T = TypeVar("T")
+
+
+def parse_json(value: str, schema: Type[T]):
+    try:
+        data = json.loads(value)
+    except json.JSONDecodeError as e:
+        raise click.ClickException('Malformed JSON string: "%s"' % str(e))
+    try:
+        obj = parse_obj_as(schema, data)
+    except ValidationError as e:
+        raise click.ClickException('Schema validation error: "%s"' % str(e))
+    return obj
+
+
+DEFAULT_CUSTOM_MODEL_BUILDER_ENVS = """
+[
+    {
+        "name": "DL_SERVICE_AUTH_STR",
+        "valueFrom": {
+            "secretKeyRef": {
+                "key": "tenant_id_secret",
+                "name": "dlserviceauth"
+            }
+        }
+    },
+    {
+        "name": "DL2_SERVICE_AUTH_STR",
+        "valueFrom": {
+            "secretKeyRef": {
+                "key": "tenant_id_secret",
+                "name": "dl2serviceauth"
+            }
+        }
+    }
+]
+"""
+
+
 @click.group("workflow")
 @click.pass_context
 def workflow_cli(gordo_ctx):
@@ -183,6 +224,12 @@ def workflow_cli(gordo_ctx):
     help="Default imagePullPolicy for all gordo's images",
     envvar=f"{PREFIX}_IMAGE_PULL_POLICY",
 )
+@click.option(
+    "--custom-model-builder-envs",
+    help="List of custom environment variables in ",
+    envvar=f"{PREFIX}_CUSTOM_MODEL_BUILDER_ENVS",
+    default=DEFAULT_CUSTOM_MODEL_BUILDER_ENVS,
+)
 @click.pass_context
 def workflow_generator_cli(gordo_ctx, **ctx):
     """
@@ -203,6 +250,13 @@ def workflow_generator_cli(gordo_ctx, **ctx):
     # Create normalized config
     config = NormalizedConfig(yaml_content, project_name=context["project_name"])
 
+    if context["custom_model_builder_envs"]:
+        result = parse_json(context["custom_model_builder_envs"], List[CustomEnv])
+        custom_model_builder_envs = []
+        for value in result:
+            custom_model_builder_envs.append(value.dict(exclude_defaults=True))
+        context["custom_model_builder_envs"] = custom_model_builder_envs
+
     version = parse_version(context["gordo_version"])
     if "image_pull_policy" not in context or not context["image_pull_policy"]:
         context["image_pull_policy"] = wg.default_image_pull_policy(version)

gordo/workflow/config_elements/schemas.py

@@ -0,0 +1,20 @@
+from typing import Optional, Union
+from pydantic import BaseModel
+
+ValueType = Union[str, int, float, bool]
+
+
+class RefValue(BaseModel):
+    name: str
+    key: str
+
+
+class ValueFrom(BaseModel):
+    configMapKeyRef: Optional[RefValue]
+    secretKeyRef: Optional[RefValue]
+
+
+class CustomEnv(BaseModel):
+    name: str
+    value: Optional[ValueType]
+    valueFrom: Optional[ValueFrom]

gordo/workflow/workflow_generator/resources/argo-workflow.yml.template

@@ -690,22 +690,13 @@ spec:
           configMapKeyRef:
             name: gordo-components-config-map  # TODO: Update in gordo-infrastructure
             key: azureml_workspace_str
-      - name: DL_SERVICE_AUTH_STR
-        valueFrom:
-          secretKeyRef:
-            name: dlserviceauth
-            key: tenant_id_secret
-      - name: DL2_SERVICE_AUTH_STR
-        valueFrom:
-          secretKeyRef:
-            name: dl2serviceauth
-            key: tenant_id_secret
       - name: GORDO_LOG_LEVEL
         value: "{{log_level}}"{% if builder_exceptions_report_file is defined %}
       - name: EXCEPTIONS_REPORTER_FILE
         value: "{{ builder_exceptions_report_file }}"{% endif %}
       - name: EXCEPTIONS_REPORT_LEVEL
-        value: "{{ builder_exceptions_report_level }}"
+        value: "{{ builder_exceptions_report_level }}"{% if custom_model_builder_envs %}
+{{ custom_model_builder_envs | yaml | indent(6, True) }}{% endif %}
       volumeMounts:
       - name: azurefile
         mountPath: /gordo
@@ -1191,19 +1182,9 @@ spec:
         value: "true"
       - name: GORDO_CLIENT_METADATA
         value: "version,{{project_revision}}"
-      - name: DL_SERVICE_AUTH_STR
-        valueFrom:
-          secretKeyRef:
-            name: dlserviceauth
-            key: tenant_id_secret
-      - name: DL2_SERVICE_AUTH_STR
-        valueFrom:
-          secretKeyRef:
-            name: dl2serviceauth
-            key: tenant_id_secret
       - name: GORDO_LOG_LEVEL
-        value: "{{log_level}}"
-
+        value: "{{log_level}}"{% if custom_model_builder_envs %}
+{{ custom_model_builder_envs | yaml | indent(6, True) }}{% endif %}
   - name: influx-cleanup
     activeDeadlineSeconds: 300
     retryStrategy:

gordo/workflow/workflow_generator/workflow_generator.py

@@ -7,7 +7,7 @@
 
 from gordo.util.version import Version, GordoRelease, GordoSpecial, GordoPR
 
-from typing import Union, cast
+from typing import Union, cast, Any
 
 logger = logging.getLogger(__name__)
 
@@ -97,6 +97,10 @@ def get_dict_from_yaml(config_file: Union[str, io.StringIO]) -> dict:
     return yaml_content
 
 
+def yaml_filter(data: Any) -> str:
+    return yaml.safe_dump(data)
+
+
 def load_workflow_template(workflow_template: str) -> jinja2.Template:
     """
     Loads the Jinja2 Template from a specified path
@@ -117,6 +121,7 @@ def load_workflow_template(workflow_template: str) -> jinja2.Template:
     templateEnv = jinja2.Environment(
         loader=jinja2.FileSystemLoader(template_dir), undefined=jinja2.StrictUndefined
     )
+    templateEnv.filters["yaml"] = yaml_filter
     return templateEnv.get_template(os.path.basename(workflow_template))
 
 

requirements/full_requirements.txt

@@ -105,6 +105,7 @@ pyarrow==0.17.1           # via gordo-dataset
 pyasn1-modules==0.2.8     # via google-auth
 pyasn1==0.4.8             # via ndg-httpsclient, pyasn1-modules, rsa
 pycparser==2.19           # via cffi
+pydantic==1.7.3           # via -r requirements.in
 pyjwt[crypto]==1.7.1      # via adal, azureml-core, msal
 pyopenssl==19.1.0         # via azureml-core, ndg-httpsclient
 pyparsing==2.4.6          # via matplotlib, packaging

requirements/requirements.in

@@ -29,3 +29,4 @@ PyYAML~=5.3.1
 gordo-dataset~=2.3.0
 jeepney>=0.6
 packaging~=20.7
+pydantic~=1.7.3