Merge branch 'develop' into update/commons-codec-1.17.1

.git-blame-ignore-revs

@@ -0,0 +1,4 @@
+# .git-blame-ignore-revs
+
+# scalafmt mass change
+3487e7a60194a66886572c899a6a93d7355dc376

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @broadinstitute/dsp-core-services

.github/workflows/format.yaml

@@ -0,0 +1,28 @@
+name: Check formatting for modified files with scalafmt
+
+on:
+  pull_request:
+    paths-ignore: ['**.md']
+
+jobs:
+  format:
+
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+          cache: sbt
+
+      - name: Check formatting for modified files
+        run: |
+          sbt scalafmtCheckAll

.scalafmt.conf

@@ -1,15 +1,17 @@
-version = 2.4.2
+version = 3.8.3
 align = none
 align.openParenCallSite = true
 align.openParenDefnSite = true
 maxColumn = 120
 continuationIndent.defnSite = 2
 assumeStandardLibraryStripMargin = true
-danglingParentheses = true
+danglingParentheses.preset = true
 rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers]
-docstrings = JavaDoc
+docstrings.style = keep
 project.excludeFilters = [
   Dependencies.scala,
   Settings.scala,
   build.sbt
 ]
+runner.dialect = scala213
+project.git = true

build.sbt

@@ -6,23 +6,28 @@ version := "0.2"
 
 organization := "org.broadinstitute"
 
-scalaVersion := "2.13.10"
+scalaVersion := "2.13.15"
 
 val akkaV = "2.6.18"
 val akkaHttpV = "10.2.7"
 val slickV = "3.3.3"
-val workbenchGoogleV = "0.28-3ad3700"
-val scalaTestV = "3.2.11"
+
+val scalaTestV = "3.2.19"
+
+val workbenchLibsHash = "9254729"
+val workbenchGoogleV = s"0.32-$workbenchLibsHash"
+val workbenchNotificationsV = s"0.7-$workbenchLibsHash"
+
 
 resolvers ++= Seq(
   "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/",
   "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/")
 
 libraryDependencies ++= Seq(
-  "org.webjars" % "swagger-ui" % "4.1.3",
+  "org.webjars" % "swagger-ui" % "5.17.14",
   "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5",
   "com.typesafe.akka" %% "akka-http-spray-json" % "10.2.9",
-  "com.google.protobuf" % "protobuf-java" % "4.0.0-rc-2",
+  "com.google.protobuf" % "protobuf-java" % "4.29.0-RC1",
   "io.sentry" % "sentry" % "6.9.2",
   "io.sentry" % "sentry-logback" % "6.9.2",
   "org.broadinstitute.dsde.workbench" %%  "workbench-google" % workbenchGoogleV
@@ -32,7 +37,7 @@ libraryDependencies ++= Seq(
     exclude("org.bouncycastle", "bcprov-ext-jdk15on")
     exclude("org.bouncycastle", "bcutil-jdk15on")
     exclude("org.bouncycastle", "bcpkix-jdk15on"),
-  "org.broadinstitute.dsde.workbench" %% "workbench-notifications" % "0.3-084d25b"
+  "org.broadinstitute.dsde.workbench" %% "workbench-notifications" % workbenchNotificationsV
     exclude("com.typesafe.akka", "akka-protobuf-v3_2.13")
     exclude("com.google.protobuf", "protobuf-java"),
   "com.typesafe.akka" %% "akka-http" % akkaHttpV,
@@ -46,17 +51,19 @@ libraryDependencies ++= Seq(
   "commons-io" % "commons-io" % "2.11.0",
   "commons-codec" % "commons-codec" % "1.17.1",
   "mysql" % "mysql-connector-java" % "8.0.28",
-  "org.liquibase" % "liquibase-core" % "4.7.1",
-  "org.hsqldb" % "hsqldb" % "2.6.1",
+  "org.liquibase" % "liquibase-core" % "4.30.0",
+  "org.hsqldb" % "hsqldb" % "2.7.4",
   "com.sendgrid" % "sendgrid-java" % "2.2.2",
-  "ch.qos.logback" % "logback-classic" % "1.4.14",
+  "ch.qos.logback" % "logback-classic" % "1.5.12",
   "org.broadinstitute.dsde.workbench" %% "sam-client" % "0.1-4cde1ff",
+  "com.azure" % "azure-identity" % "1.12.2",
+  "com.azure" % "azure-core-management" % "1.15.5",
 //---------- Test libraries -------------------//
   "org.broadinstitute.dsde.workbench" %%  "workbench-google" % workbenchGoogleV % Test classifier "tests",
   "com.typesafe.akka" %% "akka-testkit" % akkaV % Test,
   "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test,
   "org.scalatest" %% "scalatest" % scalaTestV % Test,
-  "org.mockito" %% "mockito-scala-scalatest" % "1.17.12" % Test,
+  "org.mockito" %% "mockito-scala-scalatest" % "1.17.37" % Test,
   "org.yaml" % "snakeyaml" % "1.33" % Test
 )
 

docker/build.sh

@@ -89,7 +89,7 @@ fi
 function make_jar()
 {
 	echo "building thurloe jar..."
-	docker run --rm -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:openjdk-17.0.2_1.7.2_2.13.10 /working/docker/install.sh /working
+	docker run --rm -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 /working/docker/install.sh /working
 }
 
 function docker_cmd()

docker/build_jar.sh

@@ -10,7 +10,7 @@ echo "building thurloe jar..."
 
 docker run --rm -v $PWD:/working \
 -v jar-cache:/root/.ivy \
--v jar-cache:/root/.ivy2 sbtscala/scala-sbt:openjdk-17.0.2_1.7.2_2.13.10 /working/docker/install.sh /working
+-v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 /working/docker/install.sh /working
 
 
 EXIT_CODE=$?

project/build.properties

@@ -1 +1 @@
-sbt.version=1.6.2
+sbt.version=1.10.5

project/plugins.sbt

@@ -1,13 +1,13 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.1")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.0")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.5")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.2.2")
 
-addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 
 addSbtPlugin(
-  "com.github.cb372" % "sbt-explicit-dependencies" % "0.2.16"
+  "com.github.cb372" % "sbt-explicit-dependencies" % "0.3.1"
 ) // Use `unusedCompileDependencies` to see unused dependencies
 
-addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.34")
+addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.13.0")
 
-addDependencyTreePlugin
+addDependencyTreePlugin

src/main/resources/application.conf

@@ -9,6 +9,10 @@ akka.http {
   }
 }
 
+auth {
+  googleClientId = ""
+}
+
 swagger {
   docsPath = "swagger/thurloe.yaml"
   uiVersion = "2.1.1"
@@ -61,3 +65,15 @@ liquibase {
   changelog = "org/broadinstitute/dsde/thurloe/liquibase/changelog.xml"
   initWithLiquibase = true
 }
+
+azureHosting {
+  # Set to true to enable Azure hosting, if not set or set to false, Gcp hosting will be used
+  enabled = ${?AZURE_HOSTING_ENABLED}
+  # AZURE or AZURE_GOV, default is AZURE
+  azureEnvironment = ${?AZURE_ENVIRONMENT}
+  tokenScope = ${?AZURE_TOKEN_SCOPE}
+}
+
+app {
+    enableNotifications = ${?ENABLE_NOTIFICATIONS}
+}

src/main/scala/thurloe/Main.scala

@@ -2,18 +2,19 @@ package thurloe
 
 import akka.actor.ActorSystem
 import akka.http.scaladsl.Http
-import com.typesafe.config.ConfigFactory
+import com.typesafe.config.{Config, ConfigFactory}
 import io.sentry.{Sentry, SentryOptions}
 import org.broadinstitute.dsde.workbench.google.{GoogleCredentialModes, HttpGooglePubSubDAO}
 import org.broadinstitute.dsde.workbench.model.WorkbenchEmail
 import org.broadinstitute.dsde.workbench.util.toScalaDuration
 import thurloe.dataaccess.{HttpSamDAO, HttpSendGridDAO}
+import thurloe.dataaccess.auth.CloudServiceAuthTokenProvider
 import thurloe.notification.NotificationMonitorSupervisor
 import thurloe.service.ThurloeServiceActor
 
 import java.io.File
 import scala.concurrent.ExecutionContext.Implicits.global
-import scala.jdk.CollectionConverters._
+import scala.jdk.CollectionConverters.CollectionHasAsScala
 
 object Main extends App {
   val sentryDsn = sys.env.get("SENTRY_DSN")
@@ -25,53 +26,71 @@ object Main extends App {
   }
 
   // We need an ActorSystem to host our application in
-  implicit val system = ActorSystem("thurloe")
+  implicit val system: ActorSystem = ActorSystem("thurloe")
 
   val config = ConfigFactory.load()
-  val gcsConfig = config.getConfig("gcs")
 
-  val pem =
-    GoogleCredentialModes.Pem(WorkbenchEmail(gcsConfig.getString("clientEmail")),
-                              new File(gcsConfig.getString("pathToPem")))
-  val pubSubDAO = new HttpGooglePubSubDAO(
-    gcsConfig.getString("appName"),
-    pem,
-    "thurloe",
-    gcsConfig.getString("serviceProject")
-  )
+  private val cloudAuthProvider = CloudServiceAuthTokenProvider.createProvider(config)
 
-  val samDao = new HttpSamDAO(config, pem)
-  private val httpSendGridDAO = new HttpSendGridDAO(samDao)
-  system.actorOf(
-    NotificationMonitorSupervisor.props(
-      toScalaDuration(gcsConfig.getDuration("notificationMonitor.pollInterval")),
-      toScalaDuration(gcsConfig.getDuration("notificationMonitor.pollIntervalJitter")),
-      pubSubDAO,
-      gcsConfig.getString("notificationMonitor.topicName"),
-      gcsConfig.getString("notificationMonitor.subscriptionName"),
-      gcsConfig.getInt("notificationMonitor.workerCount"),
-      httpSendGridDAO,
-      config
-        .getConfig("notification.templateIds")
-        .entrySet()
-        .asScala
-        .map(entry => entry.getKey -> entry.getValue.unwrapped().toString)
-        .toMap,
-      config.getString("notification.fireCloudPortalUrl"),
-      samDao
-    )
-  )
+  val samDao = new HttpSamDAO(config, cloudAuthProvider)
+
+  if (isNotificationsEnabled(config)) {
+    startNotificationsMonitor(config, samDao)
+  }
 
   val routes = new ThurloeServiceActor(samDao)
 
   for {
-    binding <- Http().newServerAt("0.0.0.0", 8000).bind(routes.route).recover {
-      case t: Throwable =>
-        system.log.error("FATAL - failure starting http server", t)
-        throw t
+    binding <- Http().newServerAt("0.0.0.0", 8000).bind(routes.route).recover { case t: Throwable =>
+      system.log.error("FATAL - failure starting http server", t)
+      throw t
     }
     _ = system.log.info("Thurloe now available for all your key/value pair and notification needs.")
     _ <- binding.whenTerminated
     _ <- system.terminate()
   } yield ()
+
+  private def startNotificationsMonitor(config: Config, httpSamDAO: HttpSamDAO) = {
+    val gcsConfig = config.getConfig("gcs")
+
+    val pem =
+      GoogleCredentialModes.Pem(WorkbenchEmail(gcsConfig.getString("clientEmail")),
+                                new File(gcsConfig.getString("pathToPem"))
+      )
+    val pubSubDAO = new HttpGooglePubSubDAO(
+      gcsConfig.getString("appName"),
+      pem,
+      "thurloe",
+      gcsConfig.getString("serviceProject")
+    )
+
+    val httpSendGridDAO = new HttpSendGridDAO(httpSamDAO)
+    system.actorOf(
+      NotificationMonitorSupervisor.props(
+        toScalaDuration(gcsConfig.getDuration("notificationMonitor.pollInterval")),
+        toScalaDuration(gcsConfig.getDuration("notificationMonitor.pollIntervalJitter")),
+        pubSubDAO,
+        gcsConfig.getString("notificationMonitor.topicName"),
+        gcsConfig.getString("notificationMonitor.subscriptionName"),
+        gcsConfig.getInt("notificationMonitor.workerCount"),
+        httpSendGridDAO,
+        config
+          .getConfig("notification.templateIds")
+          .entrySet()
+          .asScala
+          .map(entry => entry.getKey -> entry.getValue.unwrapped().toString)
+          .toMap,
+        config.getString("notification.fireCloudPortalUrl"),
+        samDao
+      )
+    )
+  }
+
+  private def isNotificationsEnabled(config: Config): Boolean =
+    if (config.hasPath("app.enableNotifications")) {
+      config.getBoolean("app.enableNotifications")
+    } else {
+      // notifications are enabled by default
+      true
+    }
 }

src/main/scala/thurloe/crypto/Encryption.scala

@@ -52,10 +52,12 @@ case object Aes256Cbc {
     }
 
   final def decrypt(encryptedBytes: EncryptedBytes, secretKey: SecretKey): Try[Array[Byte]] =
-    validateLength("Secret key", secretKey.key, keySize) validateAnotherLength ("Initialization vector", encryptedBytes.iv, blockSize) map {
-      _ =>
-        val cipher = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.iv)
-        cipher.doFinal(encryptedBytes.cipherText)
+    validateLength("Secret key", secretKey.key, keySize) validateAnotherLength ("Initialization vector",
+                                                                                encryptedBytes.iv,
+                                                                                blockSize
+    ) map { _ =>
+      val cipher = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.iv)
+      cipher.doFinal(encryptedBytes.cipherText)
     }
 }
 

src/main/scala/thurloe/dataaccess/HttpSamDAO.scala

@@ -6,21 +6,20 @@ import okhttp3.Protocol
 import org.broadinstitute.dsde.workbench.client.sam
 import org.broadinstitute.dsde.workbench.client.sam.ApiClient
 import org.broadinstitute.dsde.workbench.client.sam.api.AdminApi
-import org.broadinstitute.dsde.workbench.google.GoogleCredentialModes
+import thurloe.dataaccess.auth.CloudServiceAuthTokenProvider
 
 import scala.jdk.CollectionConverters._
 import scala.jdk.DurationConverters._
 
-class HttpSamDAO(config: Config, credentials: GoogleCredentialModes.Pem) extends SamDAO with LazyLogging {
+class HttpSamDAO(config: Config, cloudServiceAuthTokenProvider: CloudServiceAuthTokenProvider)
+    extends SamDAO
+    with LazyLogging {
 
-  val samConfig = config.getConfig("sam")
+  private val samConfig = config.getConfig("sam")
 
   private val samServiceURL = samConfig.getString("samBaseUrl")
   private val timeout = samConfig.getDuration("timeout").toScala
 
-  val USERINFO_EMAIL = "https://www.googleapis.com/auth/userinfo.email"
-  val USERINFO_PROFILE = "https://www.googleapis.com/auth/userinfo.profile"
-
   private def getApiClient = {
     val okHttpClient = new ApiClient().getHttpClient
 
@@ -30,29 +29,19 @@ class HttpSamDAO(config: Config, credentials: GoogleCredentialModes.Pem) extends
     val samApiClient = new ApiClient(okHttpClientBuilder.protocols(Seq(Protocol.HTTP_1_1).asJava).build())
     samApiClient.setBasePath(samServiceURL)
 
-    //Set credentials
-    val scopes = List(USERINFO_EMAIL, USERINFO_PROFILE)
-    val saPemCredentials = credentials.toGoogleCredential(scopes)
-    val expiresInSeconds = Option(saPemCredentials.getExpiresInSeconds).map(_.longValue()).getOrElse(0L)
-    val token = if (expiresInSeconds < 60 * 5) {
-      saPemCredentials.refreshToken()
-      saPemCredentials.getAccessToken
-    } else {
-      saPemCredentials.getAccessToken
-    }
+    val token: String = cloudServiceAuthTokenProvider.getAccessToken
     samApiClient.setAccessToken(token)
     samApiClient
   }
 
   protected def adminApi(samApiClient: ApiClient) = new AdminApi(samApiClient)
 
   override def getUserById(userId: String): List[sam.model.User] =
-    try {
+    try
       adminApi(getApiClient).adminGetUsersByQuery(userId, userId, userId, 5).asScala.toList
-    } catch {
+    catch {
       case e: Exception =>
         logger.warn(s"Sam user not found: $userId", e)
         List.empty
     }
-
 }