Merge pull request #1 from andreweacott/feature/remove_shazam_bucket

Feature/remove shazam bucket

build.sbt

@@ -1,6 +1,6 @@
 name := "datapipeline-dsl"
 
-version := "0.1"
+version := "0.1.1-SNAPSHOT"
 
 scalaVersion := "2.12.5"
 

src/main/scala/datapipeline/compiler/AwsDataPipelineCompiler.scala

@@ -43,7 +43,8 @@ object AwsDataPipelineCompiler extends App {
 
   val pipelineBuilder: PipelineBuilder = reflectivelyLoadPipelineBuilder(className)
 
-  writePipelineJsonToFile(pipelineBuilder)
+  val filename = s"$CurrentWorkingDir${pipelineBuilder.name}.json"
+  writePipelineJsonToFile(pipelineBuilder, filename)
 
 }
 
@@ -106,9 +107,7 @@ object AwsDataPipelineCompilerHelpers {
     pipelineBuilder
   }
 
-  def writePipelineJsonToFile(pipelineBuilder: PipelineBuilder): Unit = {
-    val filename = s"$CurrentWorkingDir${pipelineBuilder.name}.json"
-
+  def writePipelineJsonToFile(pipelineBuilder: PipelineBuilder, filename: String): Unit = {
     println(s"Writing pipeline definition to: $filename")
 
     val os = new FileOutputStream(filename)

src/main/scala/datapipeline/dsl/Json.scala

@@ -217,7 +217,7 @@ object ResourceJsonRenderer {
           ("coreInstanceBidPrice" -> coreInstanceBidPrice.map(_.toString)) ~
           ("useOnDemandOnLastAttempt" -> useOnDemandOnLastAttempt.map(_.toString)) ~
           ("terminateAfter" -> terminateAfter.map(_.toString)) ~
-          ("emrLogUri" -> s"s3://szm-den-aws-data-pipeline/temp/$${environment}/$pipelineName/logs/emr") ~
+          ("emrLogUri" -> emrLogUri) ~
           ("keyPair" -> keyPair) ~
           ("region" -> region) ~
           ("applications" -> applications) ~

src/main/scala/datapipeline/dsl/Resource.scala

@@ -29,6 +29,7 @@ case class EmrCluster(name: String,
                       coreInstanceBidPrice: Option[BigDecimal] = None,
                       useOnDemandOnLastAttempt: Option[Boolean] = None,
                       terminateAfter: Option[FiniteDuration] = None,
+                      emrLogUri: Option[String] = None,
                       keyPair: Option[String] = None,
                       region: Option[String] = None,
                       applications: Seq[String] = Nil,

src/test/scala/datapipeline/dsl/AwsDataPipelineSpec.scala

@@ -260,6 +260,7 @@ class AwsDataPipelineSpec extends WordSpec with Matchers {
           terminateAfter = Some(5.hours),
           keyPair = Some("data-engineering"),
           region = Some("us-east-1"),
+          emrLogUri = Some("s3://s3-log-bucket"),
           applications = Seq("Spark"),
           configuration = Some(EmrConfiguration(
             name = "optimize spark",
@@ -291,7 +292,7 @@ class AwsDataPipelineSpec extends WordSpec with Matchers {
             |  "coreInstanceBidPrice": "0.66",
             |  "useOnDemandOnLastAttempt": "true",
             |  "terminateAfter": "5 hours",
-            |  "emrLogUri": "s3://szm-den-aws-data-pipeline/temp/${environment}/base-pipeline/logs/emr",
+            |  "emrLogUri": "s3://s3-log-bucket",
             |  "keyPair": "data-engineering",
             |  "region": "us-east-1",
             |  "applications": ["Spark"],