Merge pull request #4519 from sgibson91/openscapes/dask-nodegroup-tag…

…ging [Openscapes] Put dask-gateway workers (and schedulers) on their own nodegroups
2i2c-org · Jul 30, 2024 · 555503f · 555503f
2 parents eca58ba + 5da9453
commit 555503f
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 2 deletions.
diff --git a/config/clusters/openscapes/prod.values.yaml b/config/clusters/openscapes/prod.values.yaml
@@ -20,3 +20,14 @@ basehub:
           oauth_callback_url: "https://openscapes.2i2c.cloud/hub/oauth_callback"
         CILogonOAuthenticator:
           oauth_callback_url: "https://openscapes.2i2c.cloud/hub/oauth_callback"
+
+dask-gateway:
+  gateway:
+    scheduler:
+      extraPodConfig:
+        nodeSelector:
+          2i2c/hub-name: prod
+    worker:
+      extraPodConfig:
+        nodeSelector:
+          2i2c/hub-name: prod
diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml
@@ -23,3 +23,15 @@ basehub:
           oauth_callback_url: "https://staging.openscapes.2i2c.cloud/hub/oauth_callback"
         CILogonOAuthenticator:
           oauth_callback_url: "https://staging.openscapes.2i2c.cloud/hub/oauth_callback"
+
+dask-gateway:
+  gateway:
+    backend:
+      scheduler:
+        extraPodConfig:
+          nodeSelector:
+            2i2c/hub-name: staging
+      worker:
+        extraPodConfig:
+          nodeSelector:
+            2i2c/hub-name: staging
diff --git a/config/clusters/openscapes/workshop.values.yaml b/config/clusters/openscapes/workshop.values.yaml
@@ -129,3 +129,14 @@ basehub:
                   image: "{value}"
               choices: {}
             resource_allocation: *profile_options_resource_allocation
+
+dask-gateway:
+  gateway:
+    scheduler:
+      extraPodConfig:
+        nodeSelector:
+          2i2c/hub-name: workshop
+    worker:
+      extraPodConfig:
+        nodeSelector:
+          2i2c/hub-name: workshop
diff --git a/eksctl/openscapes.jsonnet b/eksctl/openscapes.jsonnet
@@ -84,14 +84,31 @@ local daskNodes = [
     // Node definitions for dask worker nodes. Config here is merged
     // with our dask worker node definition, which uses spot instances.
     // A `node.kubernetes.io/instance-type label is set to the name of the
-    // *first* item in instanceDistribution.instanceTypes, to match
+    // *first* item in instancesDistribution.instanceTypes, to match
     // what we do with notebook nodes. Pods can request a particular
     // kind of node with a nodeSelector
     //
     // A not yet fully established policy is being developed about using a single
     // node pool, see https://github.com/2i2c-org/infrastructure/issues/2687.
     //
-    { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }},
+    {
+        namePrefix: "dask-staging",
+        labels+: { "2i2c/hub-name": "staging" },
+        tags+: { "2i2c:hub-name": "staging" },
+        instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }
+    },
+    {
+        namePrefix: "dask-prod",
+        labels+: { "2i2c/hub-name": "prod" },
+        tags+: { "2i2c:hub-name": "prod" },
+        instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }
+    },
+    {
+        namePrefix: "dask-workshop",
+        labels+: { "2i2c/hub-name": "workshop" },
+        tags+: { "2i2c:hub-name": "workshop" },
+        instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }
+    },
 ];
 
 
@@ -181,6 +198,9 @@ local daskNodes = [
             labels+: {
                 "k8s.dask.org/node-purpose": "worker"
             },
+            tags+: {
+                "2i2c:node-purpose": "worker"
+            },
             taints+: {
                 "k8s.dask.org_dedicated" : "worker:NoSchedule",
                 "k8s.dask.org/dedicated" : "worker:NoSchedule"