From 997c8944dd795b59dbddd347bbdcb48dc4be4ac8 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Mon, 29 Jul 2024 17:31:03 +0100 Subject: [PATCH 1/4] Add a dask nodegroup per hub on openscales cluster --- eksctl/openscapes.jsonnet | 42 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/eksctl/openscapes.jsonnet b/eksctl/openscapes.jsonnet index 624b51342..9fcbb287a 100644 --- a/eksctl/openscapes.jsonnet +++ b/eksctl/openscapes.jsonnet @@ -84,14 +84,49 @@ local daskNodes = [ // Node definitions for dask worker nodes. Config here is merged // with our dask worker node definition, which uses spot instances. // A `node.kubernetes.io/instance-type label is set to the name of the - // *first* item in instanceDistribution.instanceTypes, to match + // *first* item in instancesDistribution.instanceTypes, to match // what we do with notebook nodes. Pods can request a particular // kind of node with a nodeSelector // // A not yet fully established policy is being developed about using a single // node pool, see https://github.com/2i2c-org/infrastructure/issues/2687. // - { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }}, + { + namePrefix: "dask-staging", + labels+: { + "2i2c/hub-name": "staging" + }, + tags+: { + "2i2c:hub-name": "staging" + }, + instancesDistribution+: { + instanceTypes: ["r5.4xlarge"] + } + }, + { + namePrefix: "dask-prod", + labels+: { + "2i2c/hub-name": "prod" + }, + tags+: { + "2i2c:hub-name": "prod" + }, + instancesDistribution+: { + instanceTypes: ["r5.4xlarge"] + } + }, + { + namePrefix: "dask-workshop", + labels+: { + "2i2c/hub-name": "workshop" + }, + tags+: { + "2i2c:hub-name": "workshop" + }, + instancesDistribution+: { + instanceTypes: ["r5.4xlarge"] + } + }, ]; @@ -181,6 +216,9 @@ local daskNodes = [ labels+: { "k8s.dask.org/node-purpose": "worker" }, + tags+: { + "2i2c:node-purpose": "worker" + }, taints+: { "k8s.dask.org_dedicated" : "worker:NoSchedule", "k8s.dask.org/dedicated" : "worker:NoSchedule" From a2f72ca4642ba04e4536e4031ca15d18f4e7c7c7 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Mon, 29 Jul 2024 17:31:47 +0100 Subject: [PATCH 2/4] Add nodeSelectors to put dask workers and schedulers on hub-specific nodes --- config/clusters/openscapes/prod.values.yaml | 11 +++++++++++ config/clusters/openscapes/staging.values.yaml | 11 +++++++++++ config/clusters/openscapes/workshop.values.yaml | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/config/clusters/openscapes/prod.values.yaml b/config/clusters/openscapes/prod.values.yaml index 0e1e49ff1..5009c2d2d 100644 --- a/config/clusters/openscapes/prod.values.yaml +++ b/config/clusters/openscapes/prod.values.yaml @@ -18,3 +18,14 @@ basehub: oauth_callback_url: "https://openscapes.2i2c.cloud/hub/oauth_callback" CILogonOAuthenticator: oauth_callback_url: "https://openscapes.2i2c.cloud/hub/oauth_callback" + +dask-gateway: + gateway: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: prod + worker: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: prod diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml index a792003e8..1497d6380 100644 --- a/config/clusters/openscapes/staging.values.yaml +++ b/config/clusters/openscapes/staging.values.yaml @@ -23,3 +23,14 @@ basehub: oauth_callback_url: "https://staging.openscapes.2i2c.cloud/hub/oauth_callback" CILogonOAuthenticator: oauth_callback_url: "https://staging.openscapes.2i2c.cloud/hub/oauth_callback" + +dask-gateway: + gateway: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: staging + worker: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: staging diff --git a/config/clusters/openscapes/workshop.values.yaml b/config/clusters/openscapes/workshop.values.yaml index 476582dd4..d24a96a57 100644 --- a/config/clusters/openscapes/workshop.values.yaml +++ b/config/clusters/openscapes/workshop.values.yaml @@ -127,3 +127,14 @@ basehub: image: "{value}" choices: {} resource_allocation: *profile_options_resource_allocation + +dask-gateway: + gateway: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: workshop + worker: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: workshop From 71e3960d3df994c2c0aa2a7b76cb02a8add3e43f Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Tue, 30 Jul 2024 10:33:41 +0100 Subject: [PATCH 3/4] Tidy up jsonnet config --- eksctl/openscapes.jsonnet | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/eksctl/openscapes.jsonnet b/eksctl/openscapes.jsonnet index 9fcbb287a..5d40c512e 100644 --- a/eksctl/openscapes.jsonnet +++ b/eksctl/openscapes.jsonnet @@ -93,39 +93,21 @@ local daskNodes = [ // { namePrefix: "dask-staging", - labels+: { - "2i2c/hub-name": "staging" - }, - tags+: { - "2i2c:hub-name": "staging" - }, - instancesDistribution+: { - instanceTypes: ["r5.4xlarge"] - } + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" }, + instancesDistribution+: { instanceTypes: ["r5.4xlarge"] } }, { namePrefix: "dask-prod", - labels+: { - "2i2c/hub-name": "prod" - }, - tags+: { - "2i2c:hub-name": "prod" - }, - instancesDistribution+: { - instanceTypes: ["r5.4xlarge"] - } + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" }, + instancesDistribution+: { instanceTypes: ["r5.4xlarge"] } }, { namePrefix: "dask-workshop", - labels+: { - "2i2c/hub-name": "workshop" - }, - tags+: { - "2i2c:hub-name": "workshop" - }, - instancesDistribution+: { - instanceTypes: ["r5.4xlarge"] - } + labels+: { "2i2c/hub-name": "workshop" }, + tags+: { "2i2c:hub-name": "workshop" }, + instancesDistribution+: { instanceTypes: ["r5.4xlarge"] } }, ]; From 9c702176c1233fa9cb0e0aba88591b2d48f2bbe7 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Tue, 30 Jul 2024 10:33:58 +0100 Subject: [PATCH 4/4] Add missing backend key to dask-gateway config --- config/clusters/openscapes/staging.values.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/config/clusters/openscapes/staging.values.yaml b/config/clusters/openscapes/staging.values.yaml index 1497d6380..0e9b3edeb 100644 --- a/config/clusters/openscapes/staging.values.yaml +++ b/config/clusters/openscapes/staging.values.yaml @@ -26,11 +26,12 @@ basehub: dask-gateway: gateway: - scheduler: - extraPodConfig: - nodeSelector: - 2i2c/hub-name: staging - worker: - extraPodConfig: - nodeSelector: - 2i2c/hub-name: staging + backend: + scheduler: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: staging + worker: + extraPodConfig: + nodeSelector: + 2i2c/hub-name: staging