Merge pull request #1 from MetroStar/CSG-1234-metrics-server-nebari-p…

…lugin

Csg 1234 metrics server nebari plugin

.github/workflows/publish.yaml

@@ -0,0 +1,28 @@
+name: build and publish
+'on':
+  workflow_dispatch: {}
+  push:
+    branches:
+    - main
+jobs:
+  pypi-publish:
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/nebari-plugin-metrics-server
+    permissions:
+      id-token: write
+    steps:
+    - name: Checkout Image
+      uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Install Hatch
+      run: pip install hatch
+    - name: Build
+      run: hatch build
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

README.md

@@ -1 +1,62 @@
-# nebari-metrics-server
+
+# Nebari Plugin - Metrics Server
+
+[![PyPI - Version](https://img.shields.io/pypi/v/nebari-plugin-metrics-server.svg)](https://pypi.org/project/nebari-plugin-metrics-server)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/nebari-plugin-metrics-server.svg)](https://pypi.org/project/nebari-plugin-metrics-server)
+
+-----
+
+## Overview
+This plugin integrates Metrics Server into the Nebari platform, allowing container resource metrics for Kubernetes built-in autoscaling pipelines within Nebari. Utilizing Python, Terraform, and Helm, the plugin provides a configurable deployment.
+
+## Design and Architecture
+The plugin follows a modular design, leveraging Terraform to define the deployment of Metrics Server within a Kubernetes cluster. Key component is:
+**Terraform Configuration**: Defines variables and Helm release for deployment.
+
+## Installation Instructions
+
+
+```console
+pip install nebari-plugin-metrics-server
+```
+
+
+## Usage Instructions
+**Configurations**: Various configurations are available, including name, namespace, and affinity settings.
+
+## Configuration Details
+
+### Public
+Configuration of the Metrics Server plugin is controlled through the `metrics_server` section of the `nebari-config.yaml` for the environment.
+
+``` yaml
+metrics_server:
+  # helm release name - default metrics-server
+  name: metrics-server
+  # target namespace - default kube-system
+  namespace: kube-system
+  # configure default affinity/selector for chart components
+  affinity:
+    enabled: true # default
+    selector: general # default
+    # -- or --
+    selector:
+      default: general
+      worker: worker
+      db: general
+      auth: general
+  # helm chart values overrides
+  values: {}
+```
+
+### Internal
+The following configuration values apply to the internally managed terraform module and are indirectly controlled through related values in `nebari-config.yaml`.
+
+- `name`: Chart name for Helm release.
+- `namespace`: Kubernetes namespace configuration.
+- `affinity`: Affinity configuration for Helm release.
+- `overrides`: Map for overriding default configurations.
+
+## License
+
+`nebari-plugin-metrics-server` is distributed under the terms of the [Apache](./LICENSE.md) license.

pyproject.toml

@@ -0,0 +1,168 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "nebari-plugin-metrics-server"
+dynamic = ["version"]
+description = "MetroStar Onyx - Metrics Server plugin for Nebari platform"
+readme = "README.md"
+requires-python = ">=3.8"
+license = "Apache-2.0"
+keywords = []
+authors = [
+  { name = "Pedram Adili", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = [
+  "nebari",
+]
+
+[project.urls]
+Documentation = "https://github.com/MetroStar/nebari-metrics-server#readme"
+Issues = "https://github.com/MetroStar/nebari-metrics-server/issues"
+Source = "https://github.com/MetroStar/nebari-metrics-server"
+
+[tool.hatch.build.targets.sdist]
+include = ["src"]
+
+[tool.hatch.build.targets.wheel]
+include = ["src"]
+
+[tool.hatch.version]
+path = "src/nebari_plugin_metrics_server/__about__.py"
+
+[tool.hatch.envs.default]
+dependencies = [
+  "coverage[toml]>=6.5",
+  "pytest",
+  "kubernetes",
+]
+[tool.hatch.envs.default.scripts]
+test = "pytest {args:tests}"
+test-cov = "coverage run -m pytest {args:tests}"
+cov-report = [
+  "- coverage combine",
+  "coverage report",
+]
+cov = [
+  "test-cov",
+  "cov-report",
+]
+
+[[tool.hatch.envs.all.matrix]]
+python = ["3.8", "3.9", "3.10", "3.11"]
+
+[tool.hatch.envs.lint]
+detached = true
+dependencies = [
+  "black>=23.1.0",
+  "mypy>=1.0.0",
+  "ruff>=0.0.243",
+]
+[tool.hatch.envs.lint.scripts]
+typing = "mypy --install-types --non-interactive {args:src/nebari_plugin_metrics_server tests}"
+style = [
+  "ruff {args:.}",
+  "black --check --diff {args:.}",
+]
+fmt = [
+  "black {args:.}",
+  "ruff --fix {args:.}",
+  "style",
+]
+all = [
+  "style",
+  "typing",
+]
+
+[tool.black]
+target-version = ["py38"]
+line-length = 120
+skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py38"
+line-length = 120
+select = [
+  "A",
+  "ARG",
+  "B",
+  "C",
+  "DTZ",
+  "E",
+  "EM",
+  "F",
+  "FBT",
+  "I",
+  "ICN",
+  "ISC",
+  "N",
+  "PLC",
+  "PLE",
+  "PLR",
+  "PLW",
+  "Q",
+  "RUF",
+  "S",
+  "T",
+  "TID",
+  "UP",
+  "W",
+  "YTT",
+]
+ignore = [
+  # Allow non-abstract empty methods in abstract base classes
+  "B027",
+  # Allow boolean positional values in function calls, like `dict.get(... True)`
+  "FBT003",
+  # Ignore checks for possible passwords
+  "S105", "S106", "S107",
+  # Ignore complexity
+  "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+]
+unfixable = [
+  # Don't touch unused imports
+  "F401",
+]
+
+[tool.ruff.isort]
+known-first-party = ["nebari_plugin_metrics_server"]
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.per-file-ignores]
+# Tests can use magic values, assertions, and relative imports
+"tests/**/*" = ["PLR2004", "S101", "TID252"]
+
+[tool.coverage.run]
+source_pkgs = ["nebari_plugin_metrics_server", "tests"]
+branch = true
+parallel = true
+omit = [
+  "src/nebari_plugin_metrics_server/__about__.py",
+]
+
+[tool.coverage.paths]
+nebari_plugin_metrics_server = ["src/nebari_plugin_metrics_server", "*/nebari-plugin-metrics-server/src/nebari_plugin_metrics_server"]
+tests = ["tests", "*/nebari-plugin-metrics-server/tests"]
+
+[tool.coverage.report]
+exclude_lines = [
+  "no cov",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:",
+]
+
+[project.entry-points.nebari]
+nebari-plugin-metrics-server = "src.nebari_plugin_metrics_server"

src/nebari_plugin_metrics_server/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

src/nebari_plugin_metrics_server/__init__.py

@@ -0,0 +1,10 @@
+from nebari.hookspecs import NebariStage, hookimpl
+from typing import List
+
+from .plugin import MetricsServerStage
+
+@hookimpl
+def nebari_stage() -> List[NebariStage]:
+    return [
+        MetricsServerStage,
+    ]

src/nebari_plugin_metrics_server/plugin.py

@@ -0,0 +1,47 @@
+from typing import Any, Dict, List, Optional, Union
+
+from nebari.schema import Base
+from _nebari.stages.base import NebariTerraformStage
+
+
+class MetricsServerAffinitySelectorConfig(Base):
+    default: str
+
+
+class MetricsServerAffinityConfig(Base):
+    enabled: Optional[bool] = True
+    selector: Union[MetricsServerAffinitySelectorConfig, str] = "general"
+
+class MetricsServerConfig(Base):
+    name: Optional[str] = "metrics-server"
+    namespace: Optional[str] = "kube-system"
+    affinity: MetricsServerAffinityConfig = MetricsServerAffinityConfig()
+    values: Optional[Dict[str, Any]] = {}
+
+
+class InputSchema(Base):
+    metrics_server: MetricsServerConfig = MetricsServerConfig()
+
+
+class MetricsServerStage(NebariTerraformStage):
+    name = "metrics-server"
+    priority = 100
+
+    input_schema = InputSchema
+
+    def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
+        chart_ns = self.config.metrics_server.namespace
+        if chart_ns == None or chart_ns == "" or chart_ns == self.config.namespace:
+            chart_ns = self.config.namespace
+
+        return {
+            "name": self.config.metrics_server.name,
+            "namespace": chart_ns,
+            "affinity": {
+                "enabled": self.config.metrics_server.affinity.enabled,
+                "selector": self.config.metrics_server.affinity.selector.__dict__
+                if isinstance(self.config.metrics_server.affinity.selector, MetricsServerAffinityConfig)
+                else self.config.metrics_server.affinity.selector,
+            },
+            "overrides": self.config.metrics_server.values,
+        }

src/nebari_plugin_metrics_server/template/main.tf

@@ -0,0 +1,52 @@
+locals {
+  name      = var.name
+  namespace = var.namespace
+  overrides = var.overrides
+
+  affinity = var.affinity != null && lookup(var.affinity, "enabled", false) ? {
+    enabled = true
+    selector = try(
+      { for k in ["default"] : k => length(var.affinity.selector[k]) > 0 ? var.affinity.selector[k] : var.affinity.selector.default },
+      {
+        default = var.affinity.selector
+      },
+    )
+    } : {
+    enabled  = false
+    selector = null
+  }
+
+}
+
+resource "helm_release" "metrics_server" {
+  name             = local.name
+  create_namespace = true
+  namespace        = local.namespace
+
+  repository = "https://kubernetes-sigs.github.io/metrics-server/"
+  chart      = "metrics-server"
+  version    = "3.11.0"
+
+  values = [
+    yamlencode({
+      affinity = local.affinity.enabled ? {
+        nodeAffinity = {
+          requiredDuringSchedulingIgnoredDuringExecution = {
+            nodeSelectorTerms = [
+              {
+                matchExpressions = [
+                  {
+                    key      = "eks.amazonaws.com/nodegroup"
+                    operator = "In"
+                    values   = [local.affinity.selector.default]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      } : {}
+    }),
+    yamlencode(lookup(local.overrides, "metrics-server", {})),
+  ]
+}