Preparing for open source, first pass (#89)

* remove sbt scaffolding
* consolidate code
* cleanup code, eliminate closed-source sbt image
* remove jfrog references
* proto generate once
* exclude ChiefOfStateService from cov
* remove kafka broker setting
* remove namely from readme

Earthfile

@@ -1,4 +1,4 @@
-FROM registry.namely.land/namely/sbt:1.3.6-2.13.1
+FROM busybox:1.32
 
 all:
     # target running it all
@@ -7,46 +7,27 @@ all:
 
 code:
     # copy relevant files in, save as a base image
-    FROM registry.namely.land/namely/sbt:1.3.6-2.13.1
-    COPY -dir project sbt-dist .scalafmt.conf build.sbt .
-    COPY -dir api protos service .
-
+    FROM +sbt
+    # copy configurations
+    COPY -dir project .scalafmt.conf build.sbt .
+    # copy proto definitions
+    COPY -dir protos .
+    # copy code
+    COPY code/service/src ./code/service/src
     # clean and get dependencies
-    RUN \
-    --secret JFROG_USERNAME=+secrets/JFROG_USERNAME \
-    --secret JFROG_PASSWORD=+secrets/JFROG_PASSWORD \
-    sbt clean cleanFiles update
-
-    # RUN sbt clean cleanFiles
-    SAVE IMAGE
-
-test-local:
-    FROM +code
-    ARG CODECOV_TOKEN=""
-    ENV CODECOV_TOKEN=${CODECOV_TOKEN}
-    RUN sbt coverage test coverageAggregate
+    RUN sbt clean cleanFiles update protocGenerate
+    # save base image for use downstream
     SAVE IMAGE
 
-codecov:
-    FROM +test-local
-    ARG COMMIT_HASH=""
-    ARG BRANCH_NAME=""
-    ARG BUILD_NUMBER=""
-    RUN curl -s https://codecov.io/bash | bash -s - -B "${BRANCH_NAME}" -C "${COMMIT_HASH}" -b "${BUILD_NUMBER}"
-
-test-all:
-    BUILD +test-local
-    BUILD +codecov
-
 docker-prep:
     # package the jars/executables
     FROM +code
     ARG VERSION=dev
     # TODO: use a simpler linux packager
     # https://www.scala-sbt.org/sbt-native-packager/formats/debian.html
     RUN sbt docker:stage
-    RUN chmod -R u=rX,g=rX service/target/docker/stage
-    SAVE ARTIFACT service/target/docker/stage
+    RUN chmod -R u=rX,g=rX code/service/target/docker/stage
+    SAVE ARTIFACT code/service/target/docker/stage
 
 docker-build:
     # bundle into a slimmer, runnable container
@@ -72,8 +53,43 @@ docker-build:
     # build the image and push remotely (if all steps are successful)
     SAVE IMAGE --push registry.namely.land/namely/chief-of-state:${VERSION}
 
+
+test-local:
+    FROM +code
+    ARG CODECOV_TOKEN=""
+    ENV CODECOV_TOKEN=${CODECOV_TOKEN}
+    RUN sbt coverage test coverageAggregate
+    SAVE IMAGE
+
+codecov:
+    FROM +test-local
+    ARG COMMIT_HASH=""
+    ARG BRANCH_NAME=""
+    ARG BUILD_NUMBER=""
+    RUN curl -s https://codecov.io/bash | bash -s - -B "${BRANCH_NAME}" -C "${COMMIT_HASH}" -b "${BUILD_NUMBER}"
+
+test-all:
+    BUILD +test-local
+    BUILD +codecov
+
 it-test:
     # use the image from +docker-build for IT test if desired
     FROM docker:19.03.7-dind
     # DOCKER LOAD +docker-build img
     # RUN --with-docker docker run img ...
+
+sbt:
+    # TODO: move this to a central image
+    FROM openjdk:8u232-jdk-stretch
+
+    # Install sbt
+    ARG SBT_VERSION=1.3.6
+    RUN \
+        curl -L -o sbt.deb https://dl.bintray.com/sbt/debian/sbt-${SBT_VERSION}.deb && \
+        dpkg -i sbt.deb && \
+        rm sbt.deb && \
+        apt-get update && \
+        apt-get install sbt && \
+        sbt sbtVersion
+
+    SAVE IMAGE

Jenkinsfile

@@ -37,9 +37,6 @@ node(NODE_TAG) {
         earthRunner.addBuildArg("CODECOV_TOKEN", CODECOV_TOKEN)
         earthRunner.addBuildArg("BRANCH_NAME", buildInfo.branchName)
 
-        earthRunner.addSecret("JFROG_USERNAME")
-        earthRunner.addSecret("JFROG_PASSWORD")
-
         // add dynamic args
         if(buildInfo.shouldPush()) {
             // add --push if it should upload
@@ -49,13 +46,7 @@ node(NODE_TAG) {
             earthRunner.addArg("--no-output")
         }
 
-        // provide a context with the secrets we need as env vars
-        withCredentials([
-            string(credentialsId: 'data-jfrog-username', variable: 'JFROG_USERNAME'),
-            string(credentialsId: 'data-jfrog-password', variable: 'JFROG_PASSWORD')
-        ]) {
-            // run the earthly command that was built
-            sh earthRunner.getCommand("+all")
-        }
+         // run the earthly command that was built
+        sh earthRunner.getCommand("+all")
     }
 }

build.sbt

@@ -5,37 +5,35 @@ dockerImageCreationTask := (Docker / publishLocal in `chiefofstate`).value
 
 lazy val root = project
   .in(file("."))
-  .aggregate(api, protogen, `chiefofstate`)
+  .aggregate(protogen, `chiefofstate`)
   .settings(publishArtifact := false, skip in publish := true)
 
-lazy val api = project
-  .in(file("api"))
-  .enablePlugins(LagomApi)
-  .enablePlugins(LagomAkka)
-  .settings(name := "api")
-
 lazy val `chiefofstate` = project
-  .in(file("service"))
+  .in(file("code/service"))
   .enablePlugins(LagomScala)
   .enablePlugins(JavaAppPackaging, JavaAgent)
   .enablePlugins(PlayAkkaHttp2Support)
-  .enablePlugins(LagomImpl)
-  .enablePlugins(LagomAkka)
+  .enablePlugins(BuildSettings)
   .settings(name := "chiefofstate", javaAgents += Dependencies.Compile.KanelaAgent)
-  .dependsOn(protogen, api)
+  .dependsOn(protogen)
 
 lazy val protogen = project
-  .in(file(".protogen"))
+  .in(file("code/.protogen"))
   .enablePlugins(AkkaGrpcPlugin)
   .enablePlugins(ProtocRuntime)
   .settings(name := "protogen")
   .settings(
     inConfig(Compile)(
       Seq(
-        PB.protoSources ++= Seq(file("protos")),
-        PB.includePaths ++= Seq(file("protos")),
-        excludeFilter in PB.generate := new SimpleFileFilter(
-          (f: File) => f.getAbsolutePath.contains("google/protobuf/")
+        PB.protoSources := Seq(
+          // instruct scalapb to build all COS protos
+          file("protos/chief_of_state")
+        ),
+        PB.includePaths := Seq(
+          // includes the protobuf source for imports
+          file("protos"),
+          // includes external protobufs (like google dependencies)
+          baseDirectory.value / "target/protobuf_external"
         )
       )
     ),

service/src/main/resources/application.conf → code/service/src/main/resources/application.conf

@@ -1,6 +1,6 @@
 include "lagompb.conf"
 play {
-	modules.enabled += "com.namely.protobuf.chief_of_state.AkkaGrpcClientModule"
+	modules.enabled += "com.namely.protobuf.chief_of_state.v1.AkkaGrpcClientModule"
 	application {
 		loader = com.namely.chiefofstate.ApplicationLoader
 	}
@@ -63,15 +63,6 @@ lagom {
 		}
 	}
 
-	broker {
-		kafka {
-			brokers = "localhost:9092"
-			brokers = ${?COS_KAFKA_BROKER}
-			# disable kafka-native
-			service-name = ""
-		}
-	}
-
 	persistence {
 		jdbc {
 			create-tables.auto = false
@@ -81,19 +72,10 @@ lagom {
 }
 
 akka {
-	kafka {
-		producer {
-			kafka-clients {
-				bootstrap.servers = "localhost:9092"
-				bootstrap.servers = ${?COS_KAFKA_BROKER}
-				security.protocol = PLAINTEXT
-			}
-		}
-	}
 
 	grpc {
 		client {
-			"chief_of_state.WriteSideHandlerService" {
+			"chief_of_state.v1.WriteSideHandlerService" {
 				# Host to use if service-discovery-mechanism is set to static or grpc-dns
 				host = ${WRITE_SIDE_HANDLER_SERVICE_HOST}
 
@@ -133,7 +115,7 @@ akka {
 
 				connection-attempts = -1
 			}
-			"chief_of_state.ReadSideHandlerService" {
+			"chief_of_state.v1.ReadSideHandlerService" {
 				service-discovery {
 					mechanism = "static"
 					# Service name to use if a service-discovery.mechanism other than static or grpc-dns

service/src/main/scala/com/namely/chiefofstate/Application.scala → code/service/src/main/scala/com/namely/chiefofstate/Application.scala

@@ -12,7 +12,6 @@ import com.lightbend.lagom.scaladsl.server.{
   LagomServer
 }
 import com.namely.chiefofstate.config.{EncryptionSetting, HandlerSetting, ReadSideSetting, SendCommandSettings}
-import com.namely.chiefofstate.api.ChiefOfStateService
 import com.namely.protobuf.chiefofstate.v1.readside.ReadSideHandlerServiceClient
 import com.namely.protobuf.chiefofstate.v1.writeside.WriteSideHandlerServiceClient
 import com.softwaremill.macwire.wire
@@ -36,7 +35,7 @@ abstract class Application(context: LagomApplicationContext) extends BaseApplica
 
   // wiring up the grpc for the writeSide client
   lazy val writeSideHandlerServiceClient: WriteSideHandlerServiceClient = WriteSideHandlerServiceClient(
-    GrpcClientSettings.fromConfig("chief_of_state.WriteSideHandlerService")
+    GrpcClientSettings.fromConfig("chief_of_state.v1.WriteSideHandlerService")
   )
 
   // let us wire up the handler settings

service/src/main/scala/com/namely/chiefofstate/RestServiceImpl.scala → code/service/src/main/scala/com/namely/chiefofstate/RestServiceImpl.scala

@@ -5,11 +5,13 @@ import akka.actor.ActorSystem
 import akka.cluster.sharding.typed.scaladsl.ClusterSharding
 import com.lightbend.lagom.scaladsl.api.ServiceCall
 import com.lightbend.lagom.scaladsl.persistence.PersistentEntityRegistry
-import com.namely.chiefofstate.api.ChiefOfStateService
+import com.lightbend.lagom.scaladsl.api.{Descriptor, ServiceCall}
+import com.lightbend.lagom.scaladsl.api.Service.restCall
+import com.lightbend.lagom.scaladsl.api.transport.Method
 import com.google.protobuf.any.Any
 import io.superflat.lagompb.{AggregateRoot, BaseServiceImpl}
+import io.superflat.lagompb.BaseService
 import scalapb.{GeneratedMessage, GeneratedMessageCompanion}
-
 import scala.concurrent.{ExecutionContext, Future}
 
 class RestServiceImpl(
@@ -29,3 +31,10 @@ class RestServiceImpl(
   // TODO: Deprecate this!
   def aggregateStateCompanion: GeneratedMessageCompanion[_ <: scalapb.GeneratedMessage] = Any
 }
+
+trait ChiefOfStateService extends BaseService {
+
+  def handleCommand(): ServiceCall[NotUsed, String]
+
+  override val routes: Seq[Descriptor.Call[_, _]] = Seq(restCall(Method.GET, "/", handleCommand _))
+}

service/src/main/scala/com/namely/chiefofstate/Util.scala → code/service/src/main/scala/com/namely/chiefofstate/Util.scala

@@ -28,7 +28,8 @@ object Util {
   def toCosMetaData(metaData: MetaData): CosMetaData = {
     CosMetaData(
       entityId = metaData.entityId,
-      revisionNumber = metaData.revisionNumber,
+      // TODO: remove .toInt
+      revisionNumber = metaData.revisionNumber.toInt,
       revisionDate = metaData.revisionDate,
       data = metaData.data
     )

service/src/test/resources/application.conf → code/service/src/test/resources/application.conf

@@ -3,15 +3,15 @@ include "lagompb.conf"
 akka {
 	grpc {
 		client {
-			"chief_of_state.WriteSideHandlerService" {
+			"chief_of_state.v1.WriteSideHandlerService" {
 				# Host to use if service-discovery-mechanism is set to static or grpc-dns
 				host = ""
 
 				# port to use if service-discovery-mechism is static or service discovery does not return a port
 				port = ""
 			}
 
-			"chief_of_state.ReadSideHandlerService" {
+			"chief_of_state.v1.ReadSideHandlerService" {
 				# Host to use if service-discovery-mechanism is set to static or grpc-dns
 				host = ""
 

service/src/test/scala/com/namely/chiefofstate/AggregateCommandHandlerSpec.scala → code/service/src/test/scala/com/namely/chiefofstate/AggregateCommandHandlerSpec.scala

@@ -71,7 +71,7 @@ class AggregateCommandHandlerSpec extends BaseSpec with MockFactory {
       val cmd = Any.pack(GetStateRequest.defaultInstance)
       val priorState: Any = Any.pack(Account.defaultInstance.withAccountNumber(("123")))
       val priorEventMeta: LagompbMetaData = LagompbMetaData.defaultInstance
-        .withRevisionNumber(1L)
+        .withRevisionNumber(1)
 
       val cmdhandler: AggregateCommandHandler = new AggregateCommandHandler(null, null, testHandlerSetting)
       val result: Try[CommandHandlerResponse] = cmdhandler.handle(cmd, priorState, priorEventMeta)
@@ -89,7 +89,7 @@ class AggregateCommandHandlerSpec extends BaseSpec with MockFactory {
       val cmd = GetStateRequest.defaultInstance
       val priorState: Any = Any.pack(Account.defaultInstance)
       val priorEventMeta: LagompbMetaData = LagompbMetaData.defaultInstance
-        .withRevisionNumber(1L)
+        .withRevisionNumber(1)
 
       val cmdhandler: AggregateCommandHandler = new AggregateCommandHandler(null, null, testHandlerSetting)
       val result: Try[CommandHandlerResponse] = cmdhandler.handleTyped(cmd, priorState, priorEventMeta)
@@ -169,7 +169,7 @@ class AggregateCommandHandlerSpec extends BaseSpec with MockFactory {
 
       val currentState: Any = Any.pack(Account.defaultInstance)
       val currentMeta: LagompbMetaData = LagompbMetaData.defaultInstance
-        .withRevisionNumber(1L)
+        .withRevisionNumber(1)
 
       // let us create a mock instance of the handler service client
       val mockRequestBuilder = getMockRequestBuilder
@@ -248,7 +248,7 @@ class AggregateCommandHandlerSpec extends BaseSpec with MockFactory {
     "handle command successfully as expected with an event to persist" in {
       val priorState: Any = Any.pack(Account.defaultInstance)
       val priorEventMeta: LagompbMetaData = LagompbMetaData.defaultInstance
-        .withRevisionNumber(1L)
+        .withRevisionNumber(1)
 
       val innerCmd = Any.pack(OpenAccount.defaultInstance)
       val cmd = RemoteCommand()
@@ -333,7 +333,7 @@ class AggregateCommandHandlerSpec extends BaseSpec with MockFactory {
       result shouldBe CommandHandlerResponse()
         .withFailedResponse(
           FailedCommandHandlerResponse()
-            .withReason("received unknown event type chief_of_state.AccountOpened")
+            .withReason("received unknown event type chief_of_state.v1.AccountOpened")
             .withCause(FailureCause.VALIDATION_ERROR)
         )
 

service/src/test/scala/com/namely/chiefofstate/UtilSpec.scala → code/service/src/test/scala/com/namely/chiefofstate/UtilSpec.scala

@@ -10,7 +10,7 @@ class UtilSpec extends BaseSpec {
   "toCosMetaData" should {
     "return the right COS MetaData" in {
       val ts = Timestamp().withSeconds(3L).withNanos(2)
-      val revisionNumber = 2L
+      val revisionNumber = 2
       val data = Map("foo" -> "bar")
 
       val lagomMetaData = LagompbMetaData()

db/V1__schemas.sql

@@ -20,15 +20,6 @@ CREATE TABLE IF NOT EXISTS snapshot
     PRIMARY KEY (persistence_id, sequence_number)
 );
 
-CREATE TABLE IF NOT EXISTS read_side_offsets
-(
-    read_side_id     VARCHAR(255),
-    tag              VARCHAR(255),
-    sequence_offset  bigint,
-    time_uuid_offset char(36),
-    PRIMARY KEY (read_side_id, tag)
-);
-
 create table if not exists "AKKA_PROJECTION_OFFSET_STORE"
 (
     "PROJECTION_NAME" CHAR(255)                   NOT NULL,