Merge branch 'main' into fix-import-plugin-cli-option

.github/workflows/test_local_integration.yaml

@@ -107,8 +107,8 @@ jobs:
         with:
           workloads: "" # all
           namespace: "dev"
-          timeout: 60
-          max-restarts: 0
+          timeout: 300
+          max-restarts: 3
 
       ### DEPLOYMENT TESTS
       - name: Deployment Pytests

.pre-commit-config.yaml

@@ -37,7 +37,7 @@ repos:
         exclude: "^src/_nebari/template/"
 
   - repo: https://github.com/crate-ci/typos
-    rev: v1.27.0
+    rev: typos-dict-v0.11.37
     hooks:
       - id: typos
 
@@ -61,7 +61,7 @@ repos:
         args: ["--line-length=88", "--exclude=/src/_nebari/template/"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.2
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: ["--fix"]

RELEASE.md

@@ -9,6 +9,55 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+## Release 2024.12.1 - December 13, 2024
+
+> NOTE: Support for DigitalOcean has been removed in this release. If you plan to deploy Nebari on DigitalOcean, you first need to independently create a Kubernetes cluster and then use the `existing` deployment option.
+
+### What's Changed
+- Precommit typos by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2731
+- fix typo in KubernetesCredentials by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2729
+- handle branch rename from develop to main in github actions by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2748
+- remove do integration test by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2765
+- Remove old develop branch references after default branch renaming by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2769
+- fix CICD issue with pre-commit action by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2775
+- fix CHECK_URL in kuberhealthy checks to respect namespaces by @dcmcand in https://github.com/nebari-dev/nebari/pull/2779
+- remove duplicate GCPPrivateClusterConfig class by @dcmcand in https://github.com/nebari-dev/nebari/pull/2786
+- Fix hub variable for jupyterhub_dashboard by @kenafoster in https://github.com/nebari-dev/nebari/pull/2721
+- Fix Pytest Tests failing on PRs updating src by @joneszc in https://github.com/nebari-dev/nebari/pull/2790
+- Add ability to add overrides to jhub-apps config by @aktech in https://github.com/nebari-dev/nebari/pull/2754
+- Remove leftover develop reference by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2792
+- fix bug where check_immutable_fields throws error with old version of Nebari by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2796
+- Fix immutable field validation error when a sub-schema is not Pydantic by @kenafoster in https://github.com/nebari-dev/nebari/pull/2797
+- Address issue with AWS instance type schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2787
+- add broken note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2802
+- Fix release notes formatting to restore docs syncing functionality by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2809
+- Refactor role creation for upgrade command path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2795
+- add test workflow for upgrade by @pmeier in https://github.com/nebari-dev/nebari/pull/2780
+- Add config option to enable the encryption of AWS EKS secrets by @joneszc in https://github.com/nebari-dev/nebari/pull/2788
+- remove digital ocean tests by @dcmcand in https://github.com/nebari-dev/nebari/pull/2813
+- Python3 13 upgrade dependencies by @dcmcand in https://github.com/nebari-dev/nebari/pull/2823
+- Test support for Python 3.13 in CI by @aktech in https://github.com/nebari-dev/nebari/pull/2774
+- remove unmaintained nix files by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2831
+- allow passing X.XX or X.XX.XX as k8s versions by @dcmcand in https://github.com/nebari-dev/nebari/pull/2840
+- Remove explicit branch inputs from cloud integration test workflows in GHA by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2837
+- Allow overriding of keycloak root credentials for `2024.11.1` upgrade path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2843
+- Added security group rule descriptions by @jcbolling in https://github.com/nebari-dev/nebari/pull/2850
+- Set `launch_template.ami_id` attrs to private by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2842
+- attempt to address paramiko connection errors by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2811
+- specify terraform registry for providers not in opentofu registry by @dcmcand in https://github.com/nebari-dev/nebari/pull/2852
+- Disable AWS `launch_template` from nebari-config schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2856
+- Remove Digital Ocean references by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2838
+- Use tofu binary instead of terraform one by @marcelovilla in https://github.com/nebari-dev/nebari/pull/2773
+- Add 2024.11.1 release notes and bump version by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2859
+- Disable `jupyterlab-jhub-apps` extension when jhub-apps is disabled by @krassowski in https://github.com/nebari-dev/nebari/pull/2804
+- Validate instance types for GCP by @blakerosenthal in https://github.com/nebari-dev/nebari/pull/2730
+- update gcp instance validation by @dcmcand in https://github.com/nebari-dev/nebari/pull/2875
+
+### New Contributors
+- @jcbolling made their first contribution in https://github.com/nebari-dev/nebari/pull/2850
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.11.1...2024.12.1
+
 ## Release 2024.11.1 - November 21, 2024 (Hotfix Release)
 
 > NOTE: This hotfix addresses several major bugs identified in the 2024.9.1 release. For a detailed overview, please refer to the related discussion at #2798. Users should upgrade directly from 2024.7.1 to 2024.11.1.

src/_nebari/constants.py

@@ -1,4 +1,4 @@
-CURRENT_RELEASE = "2024.11.1"
+CURRENT_RELEASE = "2024.12.1"
 
 HELM_VERSION = "v3.15.3"
 KUSTOMIZE_VERSION = "5.4.3"

src/_nebari/provider/cloud/google_cloud.py

@@ -51,6 +51,24 @@ def regions() -> Set[str]:
     return {region.name for region in response}
 
 
+@functools.lru_cache()
+def instances(region: str) -> set[str]:
+    """Return a set of available compute instances in a region."""
+    credentials, project_id = load_credentials()
+    zones_client = compute_v1.services.region_zones.RegionZonesClient(
+        credentials=credentials
+    )
+    instances_client = compute_v1.MachineTypesClient(credentials=credentials)
+    zone_list = zones_client.list(project=project_id, region=region)
+    zones = [zone for zone in zone_list]
+    instance_set: set[str] = set()
+    for zone in zones:
+        instance_list = instances_client.list(project=project_id, zone=zone.name)
+        for instance in instance_list:
+            instance_set.add(instance.name)
+    return instance_set
+
+
 @functools.lru_cache()
 def kubernetes_versions(region: str) -> List[str]:
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""

src/_nebari/stages/infrastructure/__init__.py

@@ -325,6 +325,21 @@ def _check_input(cls, data: Any) -> Any:
             raise ValueError(
                 f"\nInvalid `kubernetes-version` provided: {data['kubernetes_version']}.\nPlease select from one of the following supported Kubernetes versions: {available_kubernetes_versions} or omit flag to use latest Kubernetes version available."
             )
+
+        # check if instances are valid
+        available_instances = google_cloud.instances(data["region"])
+        if "node_groups" in data:
+            for _, node_group in data["node_groups"].items():
+                instance = (
+                    node_group["instance"]
+                    if hasattr(node_group, "__getitem__")
+                    else node_group.instance
+                )
+                if instance not in available_instances:
+                    raise ValueError(
+                        f"Google Cloud Platform instance {instance} not one of available instance types={available_instances}"
+                    )
+
         return data
 
 

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf

@@ -60,6 +60,17 @@ resource "local_file" "overrides_json" {
   filename = "${path.module}/files/jupyterlab/overrides.json"
 }
 
+resource "local_file" "page_config_json" {
+  content = jsonencode({
+    "disabledExtensions" : {
+      "jupyterlab-jhub-apps" : !var.jhub-apps-enabled
+    },
+    # `lockedExtensions` is an empty dict to signify that `jupyterlab-jhub-apps` is not being disabled and locked (but only disabled)
+    # which means users are still allowed to disable the jupyterlab-jhub-apps extension (if they have write access to page_config).
+    "lockedExtensions" : {}
+  })
+  filename = "${path.module}/files/jupyterlab/page_config.json"
+}
 
 resource "kubernetes_config_map" "etc-ipython" {
   metadata {
@@ -92,6 +103,9 @@ locals {
   etc-jupyterlab-settings = {
     "overrides.json" = local_file.overrides_json.content
   }
+  etc-jupyterlab-page-config = {
+    "page_config.json" = local_file.page_config_json.content
+  }
 }
 
 resource "kubernetes_config_map" "etc-jupyter" {
@@ -136,6 +150,20 @@ resource "kubernetes_config_map" "jupyterlab-settings" {
   data = local.etc-jupyterlab-settings
 }
 
+
+resource "kubernetes_config_map" "jupyterlab-page-config" {
+  depends_on = [
+    local_file.page_config_json
+  ]
+
+  metadata {
+    name      = "jupyterlab-page-config"
+    namespace = var.namespace
+  }
+
+  data = local.etc-jupyterlab-page-config
+}
+
 resource "kubernetes_config_map" "git_clone_update" {
   metadata {
     name      = "git-clone-update"

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf

@@ -104,6 +104,11 @@ resource "helm_release" "jupyterhub" {
               kind      = "configmap"
             }
 
+            "/etc/jupyter/labconfig" = {
+              name      = kubernetes_config_map.jupyterlab-page-config.metadata.0.name
+              namespace = kubernetes_config_map.jupyterlab-page-config.metadata.0.namespace
+              kind      = "configmap"
+            }
           }
         )
         environments = var.conda-store-environments

src/_nebari/upgrade.py

@@ -1390,6 +1390,31 @@ def _version_specific_upgrade(
         return config
 
 
+class Upgrade_2024_12_1(UpgradeStep):
+    """
+    Upgrade step for Nebari version 2024.12.1
+    """
+
+    version = "2024.12.1"
+
+    @override
+    def _version_specific_upgrade(
+        self, config, start_version, config_filename: Path, *args, **kwargs
+    ):
+        if config.get("provider", "") == "do":
+            rich.print(
+                "\n[red bold]Error: DigitalOcean is no longer supported as a provider[/red bold].",
+            )
+            rich.print(
+                "You can still deploy Nebari to a Kubernetes cluster on DigitalOcean by using 'existing' as the provider in the config file."
+            )
+            exit()
+
+        rich.print("Ready to upgrade to Nebari version [green]2024.12.1[/green].")
+
+        return config
+
+
 __rounded_version__ = str(rounded_ver_parse(__version__))
 
 # Manually-added upgrade steps must go above this line

tests/tests_unit/conftest.py

@@ -85,6 +85,11 @@ def _mock_return_value(return_value):
             "us-central1",
             "us-east1",
         ],
+        "_nebari.provider.cloud.google_cloud.instances": [
+            "e2-standard-4",
+            "e2-standard-8",
+            "e2-highmem-4",
+        ],
     }
 
     for attribute_path, return_value in MOCK_VALUES.items():

tests/tests_unit/test_upgrade.py

@@ -69,7 +69,7 @@ def mock_input(prompt, **kwargs):
             return "y"
         elif (
             prompt
-            == "[bold]Would you like Nebari to assign the corresponding role to all of your current groups automatically?[/bold]"
+            == "[bold]Would you like Nebari to assign the corresponding role/scopes to all of your current groups automatically?[/bold]"
         ):
             return "N"
         # All other prompts will be answered with "y"