[IA-4996] WIP: Deploy Jupyter on AKS

Dockerfile

@@ -34,6 +34,7 @@ ENV CROMWELL_CHART_VERSION 0.2.523
 ENV HAIL_BATCH_CHART_VERSION 0.2.0
 ENV RSTUDIO_CHART_VERSION 0.12.0
 ENV SAS_CHART_VERSION 0.17.0
+ENV JUPYTER_CHART_VERSION 0.1.0
 
 RUN mkdir /leonardo
 COPY ./leonardo*.jar /leonardo
@@ -56,8 +57,11 @@ RUN helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx && \
 
 # .Files helm helper can't access files outside a chart. Hence in order to populate cert file properly, we're
 # pulling `terra-app-setup` locally and add cert files to the chart. As a result we need to pull all GKE
-# charts locally as well so they can acess the local cert files during the helm install step, see https://helm.sh/docs/chart_template_guide/accessing_files/
+# charts locally as well so they can access the local cert files during the helm install step, see https://helm.sh/docs/chart_template_guide/accessing_files/
 # Helm does not seem to support the direct installation of a chart located in OCI so let's pull it to a local directory for now.
+COPY ./jupyter-0.1.0.tgz /leonardo
+RUN tar -xzf /leonardo/jupyter-0.1.0.tgz -C /leonardo
+
 RUN cd /leonardo && \
     helm repo update && \
     helm pull terra-app-setup-charts/terra-app-setup --version $TERRA_APP_SETUP_VERSION --untar && \
@@ -68,6 +72,7 @@ RUN cd /leonardo && \
     helm pull terra-helm/rstudio --version $RSTUDIO_CHART_VERSION --untar && \
     helm pull terra-helm/sas --version $SAS_CHART_VERSION --untar && \
     helm pull oci://terradevacrpublic.azurecr.io/hail/hail-batch-terra-azure --version $HAIL_BATCH_CHART_VERSION --untar && \
+#     helm pull terra-helm/jupyter --version $JUPYTER_CHART_VERSION --untar && \
     cd /
 
 # Install https://github.com/apangin/jattach to get access to JDK tools

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/JsonCodec.scala

@@ -661,7 +661,7 @@ object JsonCodec {
     ComputeClass.stringToObject.get(s.toLowerCase).toRight(s"Invalid compute class ${s}")
   )
   implicit val autopilotDecoder: Decoder[Autopilot] =
-    Decoder.forProduct4("computeClass", "cpuInMillicores", "memoryInGb", "ephemeralStorageInGb")(Autopilot.apply)
+    Decoder.forProduct2("computeClass", "ephemeralStorageInGb")(Autopilot.apply)
 
   implicit val locationDecoder: Decoder[Location] = Decoder.decodeString.map(Location)
   implicit val kubeClusterIdDecoder: Decoder[KubernetesClusterLeoId] = Decoder.decodeLong.map(KubernetesClusterLeoId)

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/diskModels.scala

@@ -129,6 +129,10 @@ object FormattedBy extends Enum[FormattedBy] {
     override def asString: String = "CROMWELL"
   }
 
+  final case object Jupyter extends FormattedBy {
+    override def asString: String = "JUPYTER"
+  }
+
   final case object Allowed extends FormattedBy {
     override def asString: String = "ALLOWED"
   }

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/kubernetesModels.scala

@@ -364,6 +364,10 @@ object AppType {
     override def toString: String = "HAIL_BATCH"
   }
 
+  case object Jupyter extends AppType {
+    override def toString: String = "JUPYTER"
+  }
+
   // See more context in https://docs.google.com/document/d/1RaQRMqAx7ymoygP6f7QVdBbZC-iD9oY_XLNMe_oz_cs/edit
   case object Allowed extends AppType {
     override def toString: String = "ALLOWED"
@@ -377,16 +381,18 @@ object AppType {
   def stringToObject: Map[String, AppType] = values.map(v => v.toString -> v).toMap
 
   /**
-   * Disk formatting for an App. Currently, only Galaxy, RStudio and Custom app types
+   * Disk formatting for an App. Currently, only Galaxy, RStudio, Jupyter and Custom app types
    * support disk management. So we default all other app types to Cromwell,
    * but the field is unused.
    */
   def appTypeToFormattedByType(appType: AppType): FormattedBy =
     appType match {
       case Galaxy                                                        => FormattedBy.Galaxy
+      case Jupyter                                                       => FormattedBy.Jupyter
       case Custom                                                        => FormattedBy.Custom
       case Allowed                                                       => FormattedBy.Allowed
       case Cromwell | Wds | HailBatch | WorkflowsApp | CromwellRunnerApp => FormattedBy.Cromwell
+
     }
 }
 
@@ -439,9 +445,9 @@ final case class App(id: AppId,
                      descriptorPath: Option[Uri],
                      extraArgs: List[String],
                      sourceWorkspaceId: Option[WorkspaceId],
-                     numOfReplicas: Option[Int],
                      autodelete: Autodelete,
-                     autopilot: Option[Autopilot],
+                     autopilot: Boolean,
+                     computeProfile: ComputeProfile,
                      bucketNameToMount: Option[GcsBucketName]
 ) {
 
@@ -598,7 +604,13 @@ object ComputeClass {
   val stringToObject = values.map(v => v.toString.toLowerCase -> v).toMap
 }
 final case class Autodelete(autodeleteEnabled: Boolean, autodeleteThreshold: Option[AutodeleteThreshold])
-final case class Autopilot(computeClass: ComputeClass, cpuInMillicores: Int, memoryInGb: Int, ephemeralStorageInGb: Int)
+
+final case class ComputeProfile(numOfReplicas: Option[Int],
+                                cpuInMi: Option[Int],
+                                memoryInGb: Option[Int],
+                                computeClass: Option[ComputeClass],
+                                ephemeralStorageInGb: Option[Int]
+)
 
 final case class UpdateAppTableId(value: Long) extends AnyVal
 final case class UpdateAppJobId(value: UUID) extends AnyVal

http/src/main/resources/leo.conf

@@ -184,6 +184,10 @@ azure {
     enabled = ${?HAIL_BATCH_APP_ENABLED}
   }
 
+   jupyter-app-config {
+    enabled = ${?JUPYTER_APP_ENABLED}
+  }
+
   coa-app-config {
     instrumentation-enabled = ${?COA_INSTRUMENTATION_ENABLED}
     database-enabled  = ${?COA_DATABASE_ENABLED}

http/src/main/resources/reference.conf

@@ -455,6 +455,23 @@ azure {
      chart-versions-to-exclude-from-updates = []
    }
 
+  jupyter-app-config {
+    chart-name = "/leonardo/jupyter" // TODO (LM) this should be terra-helm/jupyter
+    chart-version = "0.1.0"
+    release-name-suffix = "jupyter-rls"
+    namespace-name-suffix = "jupyter-ns"
+    ksa-name = "jupyter-ksa"
+    services = [
+      {
+            name = "jupyter"
+            kind = "ClusterIP"
+         }
+    ]
+    enabled = true
+    # App developers - Please keep the list of non-backward compatible versions in the list below
+    chart-versions-to-exclude-from-updates = []
+  }
+
    # App types which are allowed to launch with WORKSPACE_SHARED access scope.
    allowed-shared-apps = [
      "WDS",

http/src/main/resources/swagger/api-docs.yaml

@@ -2988,6 +2988,7 @@ components:
         - ALLOWED
         - WORKFLOWS_APP
         - CROMWELL_RUNNER_APP
+        - JUPYTER
     AllowedChartName:
       type: string
       enum:

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/AppInstall.scala

@@ -15,7 +15,8 @@ import org.broadinstitute.dsde.workbench.leonardo.{
   LandingZoneResources,
   ManagedIdentityName,
   WorkspaceId,
-  WsmControlledDatabaseResource
+  WsmControlledDatabaseResource,
+  WsmControlledResourceId
 }
 import org.broadinstitute.dsp.Values
 import org.http4s.Uri
@@ -34,6 +35,10 @@ trait AppInstall[F[_]] {
 
   /** Checks status of the app. */
   def checkStatus(baseUri: Uri, authHeader: Authorization)(implicit ev: Ask[F, AppContext]): F[Boolean]
+
+//  /** Checks status of the app. */
+//  def checkStatus(cloudContext: CloudContext, runtimeName: RuntimeName)(implicit ev: Ask[F, AppContext]): F[Boolean]
+
 }
 
 object AppInstall {
@@ -43,13 +48,15 @@ object AppInstall {
                                 cromwellAppInstall: CromwellAppInstall[F],
                                 workflowsAppInstall: WorkflowsAppInstall[F],
                                 hailBatchAppInstall: HailBatchAppInstall[F],
-                                cromwellRunnerAppInstall: CromwellRunnerAppInstall[F]
+                                cromwellRunnerAppInstall: CromwellRunnerAppInstall[F],
+                                jupyterAppInstall: JupyterAppInstall[F]
   ): AppType => AppInstall[F] = _ match {
     case AppType.Wds               => wdsAppInstall
     case AppType.Cromwell          => cromwellAppInstall
     case AppType.WorkflowsApp      => workflowsAppInstall
     case AppType.HailBatch         => hailBatchAppInstall
     case AppType.CromwellRunnerApp => cromwellRunnerAppInstall
+    case AppType.Jupyter           => jupyterAppInstall
     case e                         => throw new IllegalArgumentException(s"Unexpected app type: ${e}")
   }
 
@@ -75,6 +82,7 @@ object Database {
 
 final case class BuildHelmOverrideValuesParams(app: App,
                                                workspaceId: WorkspaceId,
+                                               workspaceName: String,
                                                cloudContext: AzureCloudContext,
                                                billingProfileId: BillingProfileId,
                                                landingZoneResources: LandingZoneResources,
@@ -83,5 +91,6 @@ final case class BuildHelmOverrideValuesParams(app: App,
                                                ksaName: ServiceAccountName,
                                                managedIdentityName: ManagedIdentityName,
                                                databaseNames: List[WsmControlledDatabaseResource],
-                                               config: AKSInterpreterConfig
+                                               config: AKSInterpreterConfig,
+                                               diskWsmResourceId: Option[WsmControlledResourceId]
 )

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/JupyterAppInstall.scala

@@ -0,0 +1,70 @@
+package org.broadinstitute.dsde.workbench.leonardo.app
+import cats.effect.Async
+import cats.mtl.Ask
+import cats.syntax.all._
+import org.broadinstitute.dsde.workbench.leonardo.AppContext
+import org.broadinstitute.dsde.workbench.leonardo.config.JupyterAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.dao.JupyterDAO
+import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
+import org.broadinstitute.dsp.Values
+import org.http4s.Uri
+import org.http4s.headers.Authorization
+
+/**
+ * Jupyter app.
+ */
+class JupyterAppInstall[F[_]](config: JupyterAppConfig, jupyterDao: JupyterDAO[F])(implicit F: Async[F])
+    extends AppInstall[F] {
+  override def databases: List[Database] = List.empty
+
+  override def buildHelmOverrideValues(
+    params: BuildHelmOverrideValuesParams
+  )(implicit ev: Ask[F, AppContext]): F[Values] =
+    for {
+      ctx <- ev.ask
+      // Storage container is required for Cromwell app
+      storageContainer <- F.fromOption(
+        params.storageContainer,
+        AppCreationException("Storage container required for Jupyter app", Some(ctx.traceId))
+      )
+
+      disk <- F.fromOption(
+        params.app.appResources.disk,
+        AppCreationException("Disk required for Jupyter app", Some(ctx.traceId))
+      )
+
+//      diskResourceId <- F.fromOption(
+//        params.diskWsmResourceId,
+//        AppCreationException("Disk required for Jupyter app", Some(ctx.traceId))
+//      )
+
+      values =
+        List(
+          // workspace configs
+          raw"workspace.id=${params.workspaceId.value.toString}",
+          raw"workspace.name=${params.workspaceName}",
+          raw"workspace.storageContainer.url=https://${params.landingZoneResources.storageAccountName.value}.blob.core.windows.net/${storageContainer.name.value}",
+          raw"workspace.storageContainer.resourceId=${storageContainer.resourceId.value.toString}",
+          raw"workspace.cloudProvider=Azure",
+
+          // persistent disk configs
+          raw"persistence.diskName=${disk.name.value}",
+          raw"persistence.diskSize=${disk.size.gb}",
+          raw"persistence.subscriptionId=${params.cloudContext.subscriptionId.value}",
+          raw"persistence.resourceGroupName=${params.cloudContext.managedResourceGroupName.value}",
+
+          // app resource requests
+          raw"resources.cpu=100", // ${params.app.appResources}",
+          raw"resources.memory=128", // ${disk.size.gb}",
+
+          // misc
+          raw"serviceAccount.name=${params.ksaName.value}",
+          raw"relay.connectionName=${params.app.appName.value}"
+        )
+    } yield Values(values.mkString(","))
+
+  override def checkStatus(baseUri: Uri, authHeader: Authorization)(implicit
+    ev: Ask[F, AppContext]
+  ): F[Boolean] =
+    jupyterDao.getStatus(baseUri, authHeader).handleError(_ => false)
+}

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/auth/SamAuthProvider.scala

@@ -74,6 +74,7 @@ class SamAuthProvider[F[_]: OpenTelemetryMetrics](
             .info(Map("traceId" -> traceId.asString), e)(s"$action is not allowed for resource $samResource")
             .as(false)
       }
+      _ <- logger.info(s"result of hasPermission($samResource, $action): $res")
     } yield res
 
   override def hasPermissionWithProjectFallback[R, A](

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala

@@ -47,6 +47,7 @@ object KubernetesAppConfig {
       case (CromwellRunnerApp, CloudProvider.Azure) => Some(ConfigReader.appConfig.azure.cromwellRunnerAppConfig)
       case (Wds, CloudProvider.Azure)               => Some(ConfigReader.appConfig.azure.wdsAppConfig)
       case (HailBatch, CloudProvider.Azure)         => Some(ConfigReader.appConfig.azure.hailBatchAppConfig)
+      case (Jupyter, CloudProvider.Azure)           => Some(ConfigReader.appConfig.azure.jupyterAppConfig)
       case _                                        => None
     }
 }
@@ -202,6 +203,22 @@ final case class HailBatchAppConfig(chartName: ChartName,
   val appType: AppType = AppType.HailBatch
 }
 
+final case class JupyterAppConfig(chartName: ChartName,
+                                  chartVersion: ChartVersion,
+                                  releaseNameSuffix: ReleaseNameSuffix,
+                                  namespaceNameSuffix: NamespaceNameSuffix,
+                                  ksaName: KsaName,
+                                  services: List[ServiceConfig],
+                                  enabled: Boolean,
+                                  chartVersionsToExcludeFromUpdates: List[ChartVersion]
+) extends KubernetesAppConfig {
+  override val kubernetesServices: List[KubernetesService] = services.map(s => KubernetesService(ServiceId(-1), s))
+  override val serviceAccountName = ServiceAccountName(ksaName.value)
+
+  val cloudProvider: CloudProvider = CloudProvider.Azure
+  val appType: AppType = AppType.Jupyter
+}
+
 final case class ContainerRegistryUsername(asString: String) extends AnyVal
 final case class ContainerRegistryPassword(asString: String) extends AnyVal
 final case class ContainerRegistryCredentials(username: ContainerRegistryUsername, password: ContainerRegistryPassword)

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala

@@ -1,28 +1,50 @@
 package org.broadinstitute.dsde.workbench.leonardo.dao
 
 import cats.effect.Async
+import cats.mtl.Ask
 import cats.syntax.all._
 import io.circe.Decoder
 import org.broadinstitute.dsde.workbench.leonardo.dao.ExecutionState.{Idle, OtherState}
 import org.broadinstitute.dsde.workbench.leonardo.dao.HostStatus.HostReady
 import org.broadinstitute.dsde.workbench.leonardo.dao.HttpJupyterDAO._
 import org.broadinstitute.dsde.workbench.leonardo.dns.RuntimeDnsCache
-import org.broadinstitute.dsde.workbench.leonardo.{CloudContext, RuntimeName}
+import org.broadinstitute.dsde.workbench.leonardo.{AppContext, CloudContext, RuntimeName}
 import org.broadinstitute.dsde.workbench.model.google.GoogleProject
+import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics
 import org.http4s.circe.CirceEntityDecoder._
 import org.http4s.client.Client
-import org.http4s.{Header, Headers, Method, Request}
+import org.http4s.client.dsl.Http4sClientDsl
+import org.http4s.headers.Authorization
+import org.http4s.{Header, Headers, Method, Request, Uri}
 import org.typelevel.ci.CIString
 import org.typelevel.log4cats.Logger
 
 //Jupyter server API doc https://github.com/jupyter/jupyter/wiki/Jupyter-Notebook-Server-API
 class HttpJupyterDAO[F[_]](val runtimeDnsCache: RuntimeDnsCache[F], client: Client[F], samDAO: SamDAO[F])(implicit
   F: Async[F],
-  logger: Logger[F]
-) extends JupyterDAO[F] {
+  logger: Logger[F],
+  metrics: OpenTelemetryMetrics[F]
+) extends JupyterDAO[F]
+    with Http4sClientDsl[F] {
   private val SETDATEACCESSEDINSPECTOR_HEADER_IGNORE: Header.Raw =
     Header.Raw(CIString("X-SetDateAccessedInspector-Action"), "ignore")
 
+  def getStatus(baseUri: Uri, authHeader: Authorization)(implicit
+    ev: Ask[F, AppContext]
+  ): F[Boolean] = for {
+    _ <- metrics.incrementCounter("jupyter/status")
+    res <- client.status(
+      Request[F](
+        method = Method.GET,
+        uri = baseUri / "api" / "status", // TODO (LM) this may need to change
+        headers = Headers(authHeader)
+      )
+    )
+    _ <- logger.info(s"(LM) Jupyter endpoint: ${baseUri / "api" / "status"}")
+    _ <- logger.info(s"(LM) Jupyter status result: $res")
+
+  } yield res.isSuccess
+
   def isProxyAvailable(cloudContext: CloudContext, runtimeName: RuntimeName): F[Boolean] =
     for {
       hostStatus <- Proxy.getRuntimeTargetHost[F](runtimeDnsCache, cloudContext, runtimeName)
@@ -37,7 +59,9 @@ class HttpJupyterDAO[F[_]](val runtimeDnsCache: RuntimeDnsCache[F], client: Clie
           client
             .successful(
               Request[F](
-                method = Method.GET,
+                method =
+                  Method.GET, //    private def azureUri: Uri = Uri.unsafeFromString(s"https://${hostname.address()}/${path}")
+                // https://hostIp/runtimeName/api/status
                 uri = x.toNotebooksUri / "api" / "status",
                 headers = headers
               )
@@ -110,13 +134,6 @@ object HttpJupyterDAO {
   implicit val sessionDecoder: Decoder[Session] = Decoder.forProduct1("kernel")(Session)
 }
 
-trait JupyterDAO[F[_]] {
-  def isAllKernelsIdle(cloudContext: CloudContext, runtimeName: RuntimeName): F[Boolean]
-  def isProxyAvailable(cloudContext: CloudContext, runtimeName: RuntimeName): F[Boolean]
-  def createTerminal(googleProject: GoogleProject, runtimeName: RuntimeName): F[Unit]
-  def terminalExists(googleProject: GoogleProject, runtimeName: RuntimeName, terminalName: TerminalName): F[Boolean]
-}
-
 sealed abstract class ExecutionState
 object ExecutionState {
   case object Idle extends ExecutionState {