ID-932 Open Telemetry (#1264)

codegen_java/templates/libraries/okhttp-gson/build.sbt.mustache

@@ -38,7 +38,6 @@ lazy val root = (project in file(".")).
       "jakarta.annotation" % "jakarta.annotation-api" % "1.3.5" % "compile",
       "junit" % "junit" % "4.13.1" % "test",
       "com.novocode" % "junit-interface" % "0.10" % "test",
-      "javax.annotation" % "javax.annotation-api" % "1.3.2",
-      "io.opencensus" % "opencensus-api" % "0.31.0"
+      "javax.annotation" % "javax.annotation-api" % "1.3.2"
     )) ++ publishSettings:_*
   )

env/local.env

@@ -24,8 +24,8 @@ export GOOGLE_SUBDOMAIN_EMAIL="[email protected]"
 export INIT_WITH_LIQUIBASE="true"
 export LEONARDO_PET_SERVICE_ACCOUNT="[email protected]"
 export OIDC_AUTHORITY_ENDPOINT="https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0?p=b2c_1a_signup_signin_dev"
-export OPENCENSUS_SAMPLING_PROBABILITY=0
-export OPENCENSUS_STACKDRIVER_ENABLED="false"
+export GOOGLE_TRACE_SAMPLING_PROBABILITY=0
+export GOOGLE_TRACE_ENABLED="false"
 export POSTGRES_PASSWORD="sam-test"
 export POSTGRES_READ_URL="jdbc:postgresql://localhost:5432/testdb"
 export POSTGRES_WRITE_URL="jdbc:postgresql://localhost:5432/testdb"

env/test.env

@@ -19,8 +19,8 @@ export GOOGLE_SUBDOMAIN_EMAIL="[email protected]"
 export INIT_WITH_LIQUIBASE="true"
 export LEONARDO_PET_SERVICE_ACCOUNT="[email protected]"
 export OIDC_AUTHORITY_ENDPOINT="https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0?p=b2c_1a_signup_signin_dev"
-export OPENCENSUS_SAMPLING_PROBABILITY=0
-export OPENCENSUS_STACKDRIVER_ENABLED="false"
+export GOOGLE_TRACE_SAMPLING_PROBABILITY=0
+export GOOGLE_TRACE_ENABLED="false"
 export POSTGRES_PASSWORD="sam-test"
 export POSTGRES_READ_URL="jdbc:postgresql://localhost:5432/testdb"
 export POSTGRES_WRITE_URL="jdbc:postgresql://localhost:5432/testdb"

pact4s/src/test/resources/reference.conf

@@ -56,6 +56,12 @@ googleServices {
     keyId = "dockerhub-key"
     rotationPeriod = "180 days"
   }
+
+  traceExporter {
+    enabled = false
+    projectId = "not-actually-used"
+    samplingProbability = 0
+  }
 }
 
 db {

pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala

@@ -18,7 +18,6 @@ import org.broadinstitute.dsde.workbench.google2.mock.FakeGoogleStorageInterpret
 import org.broadinstitute.dsde.workbench.model._
 import org.broadinstitute.dsde.workbench.oauth2.OpenIDConnectConfiguration
 import org.broadinstitute.dsde.workbench.oauth2.mock.FakeOpenIDConnectConfiguration
-import org.broadinstitute.dsde.workbench.openTelemetry.{FakeOpenTelemetryMetricsInterpreter, OpenTelemetryMetrics, OpenTelemetryMetricsInterpreter}
 import org.broadinstitute.dsde.workbench.sam.api._
 import org.broadinstitute.dsde.workbench.sam.azure.{AzureService, MockCrlService}
 import org.broadinstitute.dsde.workbench.sam.config.AppConfig._
@@ -49,7 +48,6 @@ trait MockTestSupport {
 
   implicit val futureTimeout: Timeout = Timeout(Span(10, Seconds))
   implicit val eqWorkbenchException: Eq[WorkbenchException] = (x: WorkbenchException, y: WorkbenchException) => x.getMessage == y.getMessage
-  implicit val openTelemetry: FakeOpenTelemetryMetricsInterpreter.type = FakeOpenTelemetryMetricsInterpreter
 
   val samRequestContext: SamRequestContext = SamRequestContext()
 
@@ -163,8 +161,7 @@ object MockTestSupport extends MockTestSupport {
 
   def genSamRoutes(samDependencies: MockSamDependencies, uInfo: SamUser)(implicit
       system: ActorSystem,
-      materializer: Materializer,
-      openTelemetry: OpenTelemetryMetrics[IO]
+      materializer: Materializer
   ): MockSamRoutes = new MockSamRoutes(
     samDependencies.resourceService,
     samDependencies.userService,
@@ -228,7 +225,7 @@ object MockTestSupport extends MockTestSupport {
     override def asAdminServiceUser: Directive0 = Directive.Empty
   }
 
-  def genSamRoutesWithDefault(implicit system: ActorSystem, materializer: Materializer, openTelemetry: OpenTelemetryMetricsInterpreter[IO]): MockSamRoutes =
+  def genSamRoutesWithDefault(implicit system: ActorSystem, materializer: Materializer): MockSamRoutes =
     genSamRoutes(genSamDependencies(), Generator.genWorkbenchUserBoth.sample.get)
 
   /*

project/Dependencies.scala

@@ -19,7 +19,6 @@ object Dependencies {
   val workbenchGoogle2V = s"0.34-$workbenchLibV"
   val workbenchNotificationsV = s"0.6-$workbenchLibV"
   val workbenchOauth2V = s"0.5-$workbenchLibV"
-  val workbenchOpenTelemetryV = s"0.7-$workbenchLibV"
   val monocleVersion = "2.0.5"
   val crlVersion = "1.2.12-SNAPSHOT"
   val slf4jVersion = "2.0.6"
@@ -86,9 +85,6 @@ object Dependencies {
     "org.broadinstitute.dsde.workbench" %% "workbench-google" % workbenchGoogleV excludeAll (excludeWorkbenchModel, excludeWorkbenchUtil)
   val workbenchOauth2: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-oauth2" % workbenchOauth2V
   val workbenchOauth2Tests: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-oauth2" % workbenchOauth2V % "test" classifier "tests"
-  val workbenchOpenTelemetry: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-opentelemetry" % workbenchOpenTelemetryV
-  val workbenchOpenTelemetryTest: ModuleID =
-    "org.broadinstitute.dsde.workbench" %% "workbench-opentelemetry" % workbenchOpenTelemetryV % "test" classifier "tests"
   // the name of the auto-value package changed from auto-value to auto-value-annotations so old libraries are not evicted
   // leading to merge errors during sbt assembly. At this time the old version of auto-value is pulled in through the google2
   // workbench-libs dependency so exclude auto-value from there
@@ -115,13 +111,6 @@ object Dependencies {
   val scalikeCoreTest = "org.scalikejdbc" %% "scalikejdbc-test" % scalikejdbcVersion % "test"
   val postgres = "org.postgresql" % "postgresql" % postgresDriverVersion
 
-  val excludeScalaCllectionCompat = ExclusionRule(organization = "org.scala-lang.modules", name = "scala-collection-compat_2.12")
-  val opencensusScalaCode: ModuleID = "com.github.sebruck" %% "opencensus-scala-core" % "0.7.2" // excludeAll(excludIoGrpc, excludeCatsEffect )
-  val opencensusAkkaHttp: ModuleID =
-    "com.github.sebruck" %% "opencensus-scala-akka-http" % "0.7.2" excludeAll (excludeAkkaProtobufV3, excludeAkkaStream) // excludeAll(excludIoGrpc, excludeCatsEffect)
-  val opencensusStackDriverExporter: ModuleID =
-    "io.opencensus" % "opencensus-exporter-trace-stackdriver" % "0.31.1" // excludeAll(excludIoGrpc, excludeCatsEffect)
-  val opencensusLoggingExporter: ModuleID = "io.opencensus" % "opencensus-exporter-trace-logging" % "0.31.1" // excludeAll(excludIoGrpc, excludeCatsEffect)
   val slf4jApi: ModuleID = "org.slf4j" % "slf4j-api" % slf4jVersion
   val slf4jSimple: ModuleID = "org.slf4j" % "slf4j-simple" % slf4jVersion
 
@@ -131,11 +120,34 @@ object Dependencies {
   val pact4sCirce = "io.github.jbwheatley" %% "pact4s-circe" % pact4sV
   val circeCore = "io.circe" %% "circe-core" % "0.14.4"
 
-  val openCensusDependencies = Seq(
-    opencensusScalaCode,
-    opencensusAkkaHttp,
-    opencensusStackDriverExporter,
-    opencensusLoggingExporter
+  // OpenTelemetry
+  val openTelemetryVersion = "1.31.0"
+  val otelApi: ModuleID = "io.opentelemetry" % "opentelemetry-api" % openTelemetryVersion
+  val otelSdk: ModuleID = "io.opentelemetry" % "opentelemetry-sdk" % openTelemetryVersion
+  val otelSdkMetrics: ModuleID = "io.opentelemetry" % "opentelemetry-sdk-metrics" % openTelemetryVersion
+  val otelExporterLogging: ModuleID = "io.opentelemetry" % "opentelemetry-exporter-logging" % openTelemetryVersion
+  val otelSemconv: ModuleID = "io.opentelemetry.semconv" % "opentelemetry-semconv" % "1.21.0-alpha"
+  val otelAnnotation: ModuleID = "io.opentelemetry.instrumentation" % "opentelemetry-instrumentation-annotations" % openTelemetryVersion
+  val otelInstrumentationApi: ModuleID = "io.opentelemetry.instrumentation" % "opentelemetry-instrumentation-api" % openTelemetryVersion
+  val otelInstrumentationApiSemconv: ModuleID =
+    "io.opentelemetry.instrumentation" % "opentelemetry-instrumentation-api-semconv" % (openTelemetryVersion + "-alpha")
+  val otelPrometheusExporter: ModuleID = "io.opentelemetry" % "opentelemetry-exporter-prometheus" % (openTelemetryVersion + "-alpha")
+
+  // Google cloud open telemetry exporters
+  var gcpOpenTelemetryExporterVersion = "0.25.2"
+  var googleTraceExporter: ModuleID = "com.google.cloud.opentelemetry" % "exporter-trace" % gcpOpenTelemetryExporterVersion
+
+  val openTelemetryDependencies = Seq(
+    otelApi,
+    otelSdk,
+    otelSdkMetrics,
+    otelExporterLogging,
+    otelSemconv,
+    otelAnnotation,
+    otelInstrumentationApi,
+    otelInstrumentationApiSemconv,
+    otelPrometheusExporter,
+    googleTraceExporter
   )
 
   val pact4sDependencies = Seq(
@@ -190,8 +202,6 @@ object Dependencies {
     googleStorageLocal,
     workbenchOauth2,
     workbenchOauth2Tests,
-    workbenchOpenTelemetry,
-    workbenchOpenTelemetryTest,
     commonsCodec,
     liquibaseCore,
     circeYAML,
@@ -205,5 +215,5 @@ object Dependencies {
     azureManagedApplications,
     sentry,
     sentryLogback
-  ) ++ openCensusDependencies
+  ) ++ openTelemetryDependencies
 }

src/main/resources/sam.conf

@@ -24,18 +24,6 @@ liquibase {
   initWithLiquibase = ${?INIT_WITH_LIQUIBASE}
 }
 
-opencensus-scala {
-  trace {
-    sampling-probability = ${?OPENCENSUS_SAMPLING_PROBABILITY}
-    exporters {
-      stackdriver {
-        enabled = ${?OPENCENSUS_STACKDRIVER_ENABLED}
-        project-id = ${?GOOGLE_PROJECT}
-      }
-    }
-  }
-}
-
 termsOfService {
   isTosEnabled = ${?TOS_ENABLED}
   isGracePeriodEnabled = ${?TOS_GRACE_PERIOD_ENABLED}
@@ -121,6 +109,14 @@ googleServices {
     ${?ADMIN_SERVICE_ACCOUNT_3}
     ${?ADMIN_SERVICE_ACCOUNT_4}
   ]
+
+  traceExporter {
+    enabled = ${?OPENCENSUS_STACKDRIVER_ENABLED} # for backwards compatibility
+    enabled = ${?GOOGLE_TRACE_ENABLED}
+    projectId = ${?GOOGLE_PROJECT}
+    samplingProbability = ${?OPENCENSUS_SAMPLING_PROBABILITY} # for backwards compatibility
+    samplingProbability = ${?GOOGLE_TRACE_SAMPLING_PROBABILITY}
+  }
 }
 
 db {

src/main/scala/org/broadinstitute/dsde/workbench/sam/Boot.scala

@@ -5,8 +5,20 @@ import akka.http.scaladsl.Http
 import cats.data.NonEmptyList
 import cats.effect._
 import cats.implicits._
+import com.google.auth.oauth2.ServiceAccountCredentials
+import com.google.cloud.opentelemetry.trace.{TraceConfiguration, TraceExporter}
 import com.typesafe.scalalogging.LazyLogging
-import io.prometheus.client.CollectorRegistry
+import io.opentelemetry.api.OpenTelemetry
+import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator
+import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator
+import io.opentelemetry.context.propagation.{ContextPropagators, TextMapPropagator}
+import io.opentelemetry.exporter.prometheus.PrometheusHttpServer
+import io.opentelemetry.sdk.metrics.SdkMeterProvider
+import io.opentelemetry.sdk.{OpenTelemetrySdk, resources}
+import io.opentelemetry.sdk.trace.SdkTracerProvider
+import io.opentelemetry.sdk.trace.`export`.BatchSpanProcessor
+import io.opentelemetry.sdk.trace.samplers.Sampler
+import io.opentelemetry.semconv.ResourceAttributes
 import org.broadinstitute.dsde.workbench.dataaccess.PubSubNotificationDAO
 import org.broadinstitute.dsde.workbench.google.GoogleCredentialModes.{Json, Pem}
 import org.broadinstitute.dsde.workbench.google.{
@@ -22,7 +34,6 @@ import org.broadinstitute.dsde.workbench.google.{
 import org.broadinstitute.dsde.workbench.google2.{GoogleStorageInterpreter, GoogleStorageService}
 import org.broadinstitute.dsde.workbench.model.WorkbenchEmail
 import org.broadinstitute.dsde.workbench.oauth2.{ClientId, ClientSecret, OpenIDConnectConfiguration}
-import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics
 import org.broadinstitute.dsde.workbench.sam.api.{LivenessRoutes, SamRoutes, StandardSamUserDirectives}
 import org.broadinstitute.dsde.workbench.sam.azure.{AzureService, CrlService}
 import org.broadinstitute.dsde.workbench.sam.config.AppConfig.AdminConfig
@@ -37,7 +48,7 @@ import org.broadinstitute.dsde.workbench.util.DelegatePool
 import org.typelevel.log4cats.StructuredLogger
 import org.typelevel.log4cats.slf4j.Slf4jLogger
 
-import java.io.File
+import java.io.{File, FileInputStream}
 import java.nio.file.{Files, Paths}
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
@@ -56,15 +67,14 @@ object Boot extends IOApp with LazyLogging {
 
       // Shutdown all akka http connection pools/servers so we can re-bind to the ports
       Http().shutdownAllConnectionPools() *> system.terminate()
-      // Clean up prometheus registry so it will be fresh on reboot
-      CollectorRegistry.defaultRegistry.clear()
 
       IO.sleep(5 seconds) *> run(args)
     }
   }
 
   private def startup()(implicit system: ActorSystem): IO[Unit] = {
     val appConfig = AppConfig.load
+    instantiateOpenTelemetry(appConfig)
     val appDependencies = createAppDependencies(appConfig)
 
     appDependencies.use { dependencies => // this is where the resource is used
@@ -122,11 +132,6 @@ object Boot extends IOApp with LazyLogging {
         appConfig.samDatabaseConfig.samBackground
       )
 
-      // This is for sending custom metrics to stackdriver. all custom metrics starts with `OpenCensus/sam/`.
-      // Typing in `sam` in metrics explorer will show all sam custom metrics.
-      // As best practice, we should have all related metrics under same prefix separated by `/`
-      implicit0(openTelemetry: OpenTelemetryMetrics[IO]) <- OpenTelemetryMetrics.resource[IO]("sam", appConfig.prometheusConfig.endpointPort)
-
       cloudExtensionsInitializer <- cloudExtensionsInitializerResource(
         appConfig,
         foregroundDirectoryDAO,
@@ -154,8 +159,7 @@ object Boot extends IOApp with LazyLogging {
       azureManagedResourceGroupDAO,
       oauth2Config
     )(
-      actorSystem,
-      openTelemetry
+      actorSystem
     )
 
   private def cloudExtensionsInitializerResource(
@@ -353,14 +357,61 @@ object Boot extends IOApp with LazyLogging {
     )
   }
 
+  private def instantiateOpenTelemetry(appConfig: AppConfig): OpenTelemetry = {
+    val maybeVersion = Option(getClass.getPackage.getImplementationVersion)
+    val resourceBuilder =
+      resources.Resource.getDefault.toBuilder
+        .put(ResourceAttributes.SERVICE_NAME, "sam")
+    maybeVersion.foreach(version => resourceBuilder.put(ResourceAttributes.SERVICE_VERSION, version))
+    val resource = resourceBuilder.build
+
+    val maybeTracerProvider = appConfig.googleConfig.flatMap { googleConfig =>
+      if (googleConfig.googleServicesConfig.traceExporter.enabled) {
+        val traceProviderBuilder = SdkTracerProvider.builder
+        val googleTraceExporter = TraceExporter.createWithConfiguration(
+          TraceConfiguration
+            .builder()
+            .setProjectId(googleConfig.googleServicesConfig.traceExporter.projectId)
+            .setCredentials(
+              ServiceAccountCredentials.fromStream(
+                new FileInputStream(googleConfig.googleServicesConfig.serviceAccountCredentialJson.defaultServiceAccountJsonPath.asString)
+              )
+            )
+            .build()
+        )
+        traceProviderBuilder.addSpanProcessor(BatchSpanProcessor.builder(googleTraceExporter).build())
+        val probabilitySampler = Sampler.traceIdRatioBased(googleConfig.googleServicesConfig.traceExporter.samplingProbability)
+        val sdkTracerProvider = traceProviderBuilder
+          .setResource(resource)
+          .setSampler(Sampler.parentBased(probabilitySampler))
+          .build
+        Option(sdkTracerProvider)
+      } else {
+        None
+      }
+    }
+
+    val sdkMeterProvider =
+      SdkMeterProvider.builder
+        .registerMetricReader(PrometheusHttpServer.builder().setPort(appConfig.prometheusConfig.endpointPort).build())
+        .setResource(resource)
+        .build
+
+    val otelBuilder = OpenTelemetrySdk.builder
+    maybeTracerProvider.foreach(otelBuilder.setTracerProvider)
+    otelBuilder.setMeterProvider(sdkMeterProvider)
+    otelBuilder.setPropagators(ContextPropagators.create(TextMapPropagator.composite(W3CTraceContextPropagator.getInstance, W3CBaggagePropagator.getInstance)))
+    otelBuilder.buildAndRegisterGlobal
+  }
+
   private[sam] def createAppDependenciesWithSamRoutes(
       config: AppConfig,
       cloudExtensionsInitializer: CloudExtensionsInitializer,
       accessPolicyDAO: AccessPolicyDAO,
       directoryDAO: PostgresDirectoryDAO,
       azureManagedResourceGroupDAO: AzureManagedResourceGroupDAO,
       oauth2Config: OpenIDConnectConfiguration
-  )(implicit actorSystem: ActorSystem, openTelemetry: OpenTelemetryMetrics[IO]): AppDependencies = {
+  )(implicit actorSystem: ActorSystem): AppDependencies = {
     val resourceTypeMap = config.resourceTypes.map(rt => rt.name -> rt).toMap
     val policyEvaluatorService = PolicyEvaluatorService(config.emailDomain, resourceTypeMap, accessPolicyDAO, directoryDAO)
     val resourceService = new ResourceService(