From befdd9fb583843058ba63ffdba4b43e0f5c56df5 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 15 Nov 2024 17:10:44 +0000 Subject: [PATCH 1/5] smithsonian: create hub-specific nodegroups --- eksctl/smithsonian.jsonnet | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/eksctl/smithsonian.jsonnet b/eksctl/smithsonian.jsonnet index f083bca48..a1ac2e373 100644 --- a/eksctl/smithsonian.jsonnet +++ b/eksctl/smithsonian.jsonnet @@ -25,12 +25,63 @@ local nodeAz = "us-east-2b"; // A `node.kubernetes.io/instance-type label is added, so pods // can request a particular kind of node with a nodeSelector local notebookNodes = [ - { instanceType: "r5.xlarge" }, - { instanceType: "r5.4xlarge" }, - { instanceType: "r5.16xlarge" }, + { + instanceType: "r5.xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" }, + }, + { + instanceType: "r5.xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" }, + }, + { + instanceType: "r5.4xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" }, + }, + { + instanceType: "r5.4xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" }, + }, + { + instanceType: "r5.16xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { "2i2c:hub-name": "staging" }, + }, + { + instanceType: "r5.16xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, + tags+: { "2i2c:hub-name": "prod" }, + }, + { + instanceType: "g4dn.xlarge", + namePrefix: "nb-staging", + labels+: { "2i2c/hub-name": "staging" }, + tags+: { + "2i2c:hub-name": "staging", + "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1" + }, + taints+: { + "nvidia.com/gpu": "present:NoSchedule" + }, + // Allow provisioning GPUs across all AZs, to prevent situation where all + // GPUs in a single AZ are in use and no new nodes can be spawned + availabilityZones: masterAzs, + }, { instanceType: "g4dn.xlarge", + namePrefix: "nb-prod", + labels+: { "2i2c/hub-name": "prod" }, tags+: { + "2i2c:hub-name": "prod", "k8s.io/cluster-autoscaler/node-template/resources/nvidia.com/gpu": "1" }, taints+: { From 305fcbacf23edca3955ec573e526e2415de51327 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 15 Nov 2024 17:11:19 +0000 Subject: [PATCH 2/5] smithsonian: upgrade to k8s 1.30 --- eksctl/smithsonian.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eksctl/smithsonian.jsonnet b/eksctl/smithsonian.jsonnet index a1ac2e373..bb1a05752 100644 --- a/eksctl/smithsonian.jsonnet +++ b/eksctl/smithsonian.jsonnet @@ -113,7 +113,7 @@ local daskNodes = [ metadata+: { name: "smithsonian", region: clusterRegion, - version: "1.29", + version: "1.30", }, availabilityZones: masterAzs, iam: { From f6354eee44abe287fd921ff67b14b133b89428c4 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 15 Nov 2024 17:11:40 +0000 Subject: [PATCH 3/5] smithsonian: add node-purpose tags --- eksctl/smithsonian.jsonnet | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eksctl/smithsonian.jsonnet b/eksctl/smithsonian.jsonnet index bb1a05752..aa5a7c1bf 100644 --- a/eksctl/smithsonian.jsonnet +++ b/eksctl/smithsonian.jsonnet @@ -157,6 +157,7 @@ local daskNodes = [ "hub.jupyter.org/node-purpose": "core", "k8s.dask.org/node-purpose": "core" }, + tags+: { "2i2c:node-purpose": "core" }, }, ] + [ ng + { @@ -172,6 +173,7 @@ local daskNodes = [ "hub.jupyter.org/node-purpose": "user", "k8s.dask.org/node-purpose": "scheduler" }, + tags+: { "2i2c:node-purpose": "user" }, taints+: { "hub.jupyter.org_dedicated": "user:NoSchedule", "hub.jupyter.org/dedicated": "user:NoSchedule" From 006ce7570e5d40ff34b7096126a82a3d1c83afad Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 15 Nov 2024 17:11:55 +0000 Subject: [PATCH 4/5] smithsonian: cycle core nodegroup --- eksctl/smithsonian.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eksctl/smithsonian.jsonnet b/eksctl/smithsonian.jsonnet index aa5a7c1bf..c19cab725 100644 --- a/eksctl/smithsonian.jsonnet +++ b/eksctl/smithsonian.jsonnet @@ -144,7 +144,7 @@ local daskNodes = [ [ ng + { namePrefix: 'core', - nameSuffix: 'a', + nameSuffix: 'b', nameIncludeInstanceType: false, availabilityZones: [nodeAz], ssh: { From 45ff5091cd642da113bf39129bc92a9249bd8d0d Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 15 Nov 2024 17:13:14 +0000 Subject: [PATCH 5/5] smithsonian: add node selectors --- config/clusters/smithsonian/prod.values.yaml | 3 +++ config/clusters/smithsonian/staging.values.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/config/clusters/smithsonian/prod.values.yaml b/config/clusters/smithsonian/prod.values.yaml index b56b3e853..2c7096b74 100644 --- a/config/clusters/smithsonian/prod.values.yaml +++ b/config/clusters/smithsonian/prod.values.yaml @@ -9,3 +9,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://smithsonian.2i2c.cloud/hub/oauth_callback + singleuser: + nodeSelector: + 2i2c/hub-name: prod diff --git a/config/clusters/smithsonian/staging.values.yaml b/config/clusters/smithsonian/staging.values.yaml index 1b30b413f..b743ff37c 100644 --- a/config/clusters/smithsonian/staging.values.yaml +++ b/config/clusters/smithsonian/staging.values.yaml @@ -9,3 +9,6 @@ basehub: config: GitHubOAuthenticator: oauth_callback_url: https://staging.smithsonian.2i2c.cloud/hub/oauth_callback + singleuser: + nodeSelector: + 2i2c/hub-name: staging