Remove kbatch, prefect and clearml (#2101)

Co-authored-by: Chuck McAndrew <[email protected]>

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -104,7 +104,6 @@ body:
       description: Is this issue related to any of the Nebari integrations?
       multiple: true
       options:
-        - "Prefect"
         - "Keycloak"
         - "conda-store"
         - "Dask"

.pre-commit-config.yaml

@@ -30,8 +30,6 @@ repos:
         exclude: "^docs-sphinx/cli.html"
       - id: check-json
       - id: check-yaml
-        # jinja2 templates for helm charts
-        exclude: "src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)"
         args: [--allow-multiple-documents]
       - id: check-toml
       # Lint: Checks that non-binary executables have a proper shebang.

scripts/helm-validate.py

@@ -215,7 +215,7 @@ def add_workflow_job_summary(chart_index: dict):
 if __name__ == "__main__":
     # charts = generate_index_of_helm_charts()
     STAGES_DIR = "nebari/template/stages"
-    SKIP_CHARTS = ["prefect", "clearml", "helm-extensions"]
+    SKIP_CHARTS = ["helm-extensions"]
 
     charts = HelmChartIndexer(
         stages_dir=STAGES_DIR, skip_charts=SKIP_CHARTS

src/_nebari/provider/cicd/github.py

@@ -108,12 +108,6 @@ def gha_env_vars(config: schema.Main):
     if os.environ.get("NEBARI_GH_BRANCH"):
         env_vars["NEBARI_GH_BRANCH"] = "${{ secrets.NEBARI_GH_BRANCH }}"
 
-    # This assumes that the user is using the omitting sensitive values configuration for the token.
-    if config.prefect.enabled:
-        env_vars[
-            "NEBARI_SECRET_prefect_token"
-        ] = "${{ secrets.NEBARI_SECRET_PREFECT_TOKEN }}"
-
     if config.provider == schema.ProviderEnum.aws:
         env_vars["AWS_ACCESS_KEY_ID"] = "${{ secrets.AWS_ACCESS_KEY_ID }}"
         env_vars["AWS_SECRET_ACCESS_KEY"] = "${{ secrets.AWS_SECRET_ACCESS_KEY }}"

src/_nebari/stages/kubernetes_ingress/__init__.py

@@ -24,17 +24,6 @@
 TIMEOUT = 10  # seconds
 
 
-def add_clearml_dns(zone_name, record_name, record_type, ip_or_hostname):
-    dns_records = [
-        f"app.clearml.{record_name}",
-        f"api.clearml.{record_name}",
-        f"files.clearml.{record_name}",
-    ]
-
-    for dns_record in dns_records:
-        update_record(zone_name, dns_record, record_type, ip_or_hostname)
-
-
 def provision_ingress_dns(
     stage_outputs: Dict[str, Dict[str, Any]],
     config: schema.Main,
@@ -60,13 +49,9 @@ def provision_ingress_dns(
             schema.ProviderEnum.azure,
         }:
             update_record(zone_name, record_name, "A", ip_or_hostname)
-            if config.clearml.enabled:
-                add_clearml_dns(zone_name, record_name, "A", ip_or_hostname)
 
         elif config.provider == schema.ProviderEnum.aws:
             update_record(zone_name, record_name, "CNAME", ip_or_hostname)
-            if config.clearml.enabled:
-                add_clearml_dns(zone_name, record_name, "CNAME", ip_or_hostname)
         else:
             logger.info(
                 f"Couldn't update the DNS record for cloud provider: {config.provider}"

src/_nebari/stages/kubernetes_services/__init__.py

@@ -37,13 +37,6 @@ def to_yaml(cls, representer, node):
         return representer.represent_str(node.value)
 
 
-class Prefect(schema.Base):
-    enabled: bool = False
-    image: typing.Optional[str]
-    overrides: typing.Dict = {}
-    token: typing.Optional[str]
-
-
 class DefaultImages(schema.Base):
     jupyterhub: str = f"quay.io/nebari/nebari-jupyterhub:{set_docker_image_tag()}"
     jupyterlab: str = f"quay.io/nebari/nebari-jupyterlab:{set_docker_image_tag()}"
@@ -195,20 +188,10 @@ class ArgoWorkflows(schema.Base):
     nebari_workflow_controller: NebariWorkflowController = NebariWorkflowController()
 
 
-class KBatch(schema.Base):
-    enabled: bool = True
-
-
 class Monitoring(schema.Base):
     enabled: bool = True
 
 
-class ClearML(schema.Base):
-    enabled: bool = False
-    enable_forward_auth: bool = False
-    overrides: typing.Dict = {}
-
-
 class JupyterHub(schema.Base):
     overrides: typing.Dict = {}
 
@@ -228,7 +211,6 @@ class JupyterLab(schema.Base):
 
 
 class InputSchema(schema.Base):
-    prefect: Prefect = Prefect()
     default_images: DefaultImages = DefaultImages()
     storage: Storage = Storage()
     theme: Theme = Theme()
@@ -249,11 +231,6 @@ class InputSchema(schema.Base):
                 "numpy=1.23.5",
                 "numba=0.56.4",
                 "pandas=1.5.3",
-                {
-                    "pip": [
-                        "kbatch==0.4.2",
-                    ],
-                },
             ],
         ),
         "environment-dashboard.yaml": CondaEnvironment(
@@ -301,9 +278,7 @@ class InputSchema(schema.Base):
     }
     conda_store: CondaStore = CondaStore()
     argo_workflows: ArgoWorkflows = ArgoWorkflows()
-    kbatch: KBatch = KBatch()
     monitoring: Monitoring = Monitoring()
-    clearml: ClearML = ClearML()
     jupyterhub: JupyterHub = JupyterHub()
     jupyterlab: JupyterLab = JupyterLab()
 
@@ -382,23 +357,6 @@ class ArgoWorkflowsInputVars(schema.Base):
     )
 
 
-class KBatchInputVars(schema.Base):
-    kbatch_enabled: bool = Field(alias="kbatch-enabled")
-
-
-class PrefectInputVars(schema.Base):
-    prefect_enabled: bool = Field(alias="prefect-enabled")
-    prefect_token: str = Field(None, alias="prefect-token")
-    prefect_image: str = Field(None, alias="prefect-image")
-    prefect_overrides: Dict = Field(alias="prefect-overrides")
-
-
-class ClearMLInputVars(schema.Base):
-    clearml_enabled: bool = Field(alias="clearml-enabled")
-    clearml_enable_forwardauth: bool = Field(alias="clearml-enable-forwardauth")
-    clearml_overrides: List[str] = Field(alias="clearml-overrides")
-
-
 class KubernetesServicesStage(NebariTerraformStage):
     name = "07-kubernetes-services"
     priority = 70
@@ -520,33 +478,13 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             keycloak_read_only_user_credentials=keycloak_read_only_user_credentials,
         )
 
-        kbatch_vars = KBatchInputVars(
-            kbatch_enabled=self.config.kbatch.enabled,
-        )
-
-        prefect_vars = PrefectInputVars(
-            prefect_enabled=self.config.prefect.enabled,
-            prefect_token=self.config.prefect.token,
-            prefect_image=self.config.prefect.image,
-            prefect_overrides=self.config.prefect.overrides,
-        )
-
-        clearml_vars = ClearMLInputVars(
-            clearml_enabled=self.config.clearml.enabled,
-            clearml_enable_forwardauth=self.config.clearml.enable_forward_auth,
-            clearml_overrides=[json.dumps(self.config.clearml.overrides)],
-        )
-
         return {
             **kubernetes_services_vars.dict(by_alias=True),
             **conda_store_vars.dict(by_alias=True),
             **jupyterhub_vars.dict(by_alias=True),
             **dask_gateway_vars.dict(by_alias=True),
             **monitoring_vars.dict(by_alias=True),
             **argo_workflows_vars.dict(by_alias=True),
-            **kbatch_vars.dict(by_alias=True),
-            **prefect_vars.dict(by_alias=True),
-            **clearml_vars.dict(by_alias=True),
         }
 
     def check(

src/_nebari/stages/kubernetes_services/template/jupyterhub.tf

@@ -112,8 +112,6 @@ module "jupyterhub" {
     "dask-gateway"
     ],
     (var.monitoring-enabled ? ["monitoring"] : []),
-    (var.prefect-enabled ? ["prefect"] : []),
-    (var.kbatch-enabled ? ["kbatch"] : [])
   )
 
   general-node-group = var.node_groups.general