S3 maven artifacts support

3rdparty/jvm/com/amazonaws/BUILD

@@ -0,0 +1,85 @@
+load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "aws_java_sdk_core",
+    exports = [
+        "//external:jar/com/amazonaws/aws_java_sdk_core"
+    ],
+    runtime_deps = [
+        "//3rdparty/jvm/com/fasterxml/jackson/core:jackson_databind",
+        "//3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson_dataformat_cbor",
+        "//3rdparty/jvm/commons_logging:commons_logging",
+        "//3rdparty/jvm/joda_time:joda_time",
+        "//3rdparty/jvm/org/apache/httpcomponents:httpclient",
+        "//3rdparty/jvm/software/amazon/ion:ion_java"
+    ],
+    visibility = [
+        "//visibility:public"
+    ]
+)
+
+
+
+java_library(
+    name = "aws_java_sdk_kms",
+    exports = [
+        "//external:jar/com/amazonaws/aws_java_sdk_kms"
+    ],
+    runtime_deps = [
+        ":aws_java_sdk_core",
+        ":jmespath_java"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+
+
+java_library(
+    name = "aws_java_sdk_s3",
+    exports = [
+        "//external:jar/com/amazonaws/aws_java_sdk_s3"
+    ],
+    runtime_deps = [
+        ":aws_java_sdk_core",
+        ":aws_java_sdk_kms",
+        ":jmespath_java"
+    ],
+    visibility = [
+        "//visibility:public"
+    ]
+)
+
+
+
+java_library(
+    name = "aws_java_sdk_sts",
+    exports = [
+        "//external:jar/com/amazonaws/aws_java_sdk_sts"
+    ],
+    runtime_deps = [
+        ":aws_java_sdk_core",
+        ":jmespath_java"
+    ],
+    visibility = [
+        "//visibility:public"
+    ]
+)
+
+
+
+java_library(
+    name = "jmespath_java",
+    exports = [
+        "//external:jar/com/amazonaws/jmespath_java"
+    ],
+    runtime_deps = [
+        "//3rdparty/jvm/com/fasterxml/jackson/core:jackson_databind"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+

3rdparty/jvm/com/fasterxml/jackson/dataformat/BUILD

@@ -1,5 +1,20 @@
 load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
 load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "jackson_dataformat_cbor",
+    exports = [
+        "//external:jar/com/fasterxml/jackson/dataformat/jackson_dataformat_cbor"
+    ],
+    runtime_deps = [
+        "//3rdparty/jvm/com/fasterxml/jackson/core:jackson_core"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+
+
 java_library(
     name = "jackson_dataformat_yaml",
     exports = [

3rdparty/jvm/commons_logging/BUILD

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "commons_logging",
+    exports = [
+        "//external:jar/commons_logging/commons_logging"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+

3rdparty/jvm/joda_time/BUILD

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "joda_time",
+    exports = [
+        "//external:jar/joda_time/joda_time"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+

3rdparty/jvm/org/apache/httpcomponents/BUILD

@@ -7,6 +7,7 @@ java_library(
     ],
     runtime_deps = [
         "//3rdparty/jvm/commons_codec:commons_codec",
+        "//3rdparty/jvm/commons_logging:commons_logging",
         ":httpcore"
     ],
     visibility = [

3rdparty/jvm/org/apache/ivy/BUILD

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "ivy",
+    exports = [
+        "//external:jar/org/apache/ivy/ivy"
+    ],
+    visibility = [
+        "//visibility:public"
+    ]
+)
+
+

3rdparty/jvm/software/amazon/ion/BUILD

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import")
+load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library")
+java_library(
+    name = "ion_java",
+    exports = [
+        "//external:jar/software/amazon/ion/ion_java"
+    ],
+    visibility = [
+        "//3rdparty/jvm:__subpackages__"
+    ]
+)
+
+

README.md

@@ -283,6 +283,8 @@ responsible for building correctly, and correctly exporting any dependencies tha
 on the compile classpath.
 
 ## Code
-This code was originally forked from [pgr0ss/bazel-deps](https://github.com/pgr0ss/bazel-deps)
+This code was originally forked from [pgr0ss/bazel-deps](https://github.com/pgr0ss/bazel-deps).
 
 This code was inspired by the [aether examples](https://github.com/eclipse/aether-demo/blob/322fa556494335faaf3ad3b7dbe8f89aaaf6222d/aether-demo-snippets/src/main/java/org/eclipse/aether/examples/GetDependencyTree.java) for walking maven dependencies.
+
+S3 Handler was based on [frugalmechanic/fm-sbt-s3-resolver](https://github.com/frugalmechanic/fm-sbt-s3-resolver/).

dependencies.yaml

@@ -128,6 +128,22 @@ dependencies:
     paiges-core:
       lang: scala
       version: "0.2.1"
+
+  com.amazonaws:
+    aws-java-sdk-core:
+      lang: java
+      version: "1.11.507"
+    aws-java-sdk-s3:
+      lang: java
+      version: "1.11.507"
+    aws-java-sdk-sts:
+      lang: java
+      version: "1.11.507"
+
+  org.apache.ivy:
+    ivy:
+      lang: java
+      version: "2.4.0"
 
 replacements:
   org.scala-lang:

src/scala/com/github/johnynek/bazel_deps/BUILD

@@ -46,6 +46,36 @@ scala_library(
     exports = ["//3rdparty/jvm/org/apache/maven:maven_settings"],
     visibility = ["//visibility:public"])
 
+scala_library(
+    name = "s3_url_handler",
+    srcs = ["S3URLHandler.scala"],
+    deps = [
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_core",
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_s3",
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_sts",
+        "//3rdparty/jvm/org/apache/ivy:ivy",
+    ],
+    exports = ["//3rdparty/jvm/org/apache/maven:maven_settings"],
+    visibility = ["//visibility:public"])
+
+scala_library(
+    name = "s3_url_loader",
+    srcs = [
+        "S3Handler.scala", 
+        "S3URLConnection.scala"
+    ],
+    deps = [
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_core",
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_s3",
+        "//3rdparty/jvm/com/amazonaws:aws_java_sdk_sts",
+        "//3rdparty/jvm/org/apache/ivy:ivy",
+        ":s3_url_handler",
+    ],
+    exports = [
+        "//3rdparty/jvm/org/apache/maven:maven_settings",
+        ":s3_url_handler",
+    ],
+    visibility = ["//visibility:public"])
 
 scala_library(
     name = "coursier_resolver",
@@ -61,6 +91,7 @@ scala_library(
         ":depsmodel",
         ":graph",
         ":settings_loader",
+        ":s3_url_loader",
         ],
     visibility = ["//visibility:public"])
 

src/scala/com/github/johnynek/bazel_deps/CoursierResolver.scala

@@ -15,6 +15,15 @@ import scala.util.{Failure, Try}
 import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.concurrent.duration.Duration
 
+import java.net.{URL, URLStreamHandler, URLStreamHandlerFactory}
+
+private object S3URLStreamHandlerFactory extends URLStreamHandlerFactory {
+  def createURLStreamHandler(protocol: String): URLStreamHandler = protocol match {
+    case "s3" => new com.github.johnynek.bazel_deps.S3Handler
+    case _    => null
+  }
+}
+
 object CoursierResolver {
   // 12 concurrent downloads
   // most downloads are tiny sha downloads so try keep things alive
@@ -39,6 +48,8 @@ class CoursierResolver(servers: List[MavenServer], ec: ExecutionContext, runTime
   private[this] val repos = LocalRepositories.ivy2Local :: {
     val settings = SettingsLoader.settings
 
+    URL.setURLStreamHandlerFactory(S3URLStreamHandlerFactory)
+
     servers.map { case MavenServer(id, _, url) =>
       val authentication = Option(settings.getServer(id))
         .map(server => Authentication(server.getUsername, server.getPassword))

src/scala/com/github/johnynek/bazel_deps/S3Handler.scala

@@ -0,0 +1,7 @@
+package com.github.johnynek.bazel_deps
+
+import java.net.{URL, URLConnection, URLStreamHandler}
+
+final class S3Handler extends URLStreamHandler {
+    def openConnection(url: URL): URLConnection = new S3URLConnection(url)
+}

src/scala/com/github/johnynek/bazel_deps/S3URLConnection.scala

@@ -0,0 +1,102 @@
+package com.github.johnynek.bazel_deps
+
+import com.amazonaws.AmazonServiceException
+import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object}
+import com.github.johnynek.bazel_deps.S3URLHandler
+import java.io.InputStream
+import java.net.{HttpURLConnection, URL}
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter
+
+object S3URLConnection {
+  private val s3: S3URLHandler = new S3URLHandler()
+}
+
+/**
+ * Implements an HttpURLConnection for compatibility with Coursier (https://github.com/coursier/coursier)
+ */
+final class S3URLConnection(url: URL) extends HttpURLConnection(url) {
+  import S3URLConnection.s3
+
+  private trait S3Response extends AutoCloseable {
+    def meta: ObjectMetadata
+    def inputStream: Option[InputStream]
+  }
+
+  private case class HEADResponse(meta: ObjectMetadata) extends S3Response {
+    def close(): Unit = {}
+    def inputStream: Option[InputStream] = None
+  }
+
+  private case class GETResponse(obj: S3Object) extends S3Response {
+    def meta: ObjectMetadata = obj.getObjectMetadata
+    def inputStream: Option[InputStream] = Option(obj.getObjectContent())
+    def close(): Unit = obj.close()
+  }
+
+  private[this] var response: Option[S3Response] = None
+
+  def connect(): Unit = {
+    val (client, bucket, key) = s3.getClientBucketAndKey(url)
+
+    try {
+      response = getRequestMethod.toLowerCase match {
+        case "head" => Option(HEADResponse(client.getObjectMetadata(bucket, key)))
+        case "get" => Option(GETResponse(client.getObject(bucket, key)))
+        case "post" => ???
+        case "put" => ???
+        case _ => throw new IllegalArgumentException("Invalid request method: "+getRequestMethod)
+      }
+
+      responseCode = if (response.isEmpty) 404 else 200
+    } catch {
+      case ex: AmazonServiceException => responseCode = ex.getStatusCode
+    }
+
+    // Also set the responseMessage (an HttpURLConnection field) for better compatibility
+    responseMessage = statusMessageForCode(responseCode)
+    connected = true
+  }
+
+  def usingProxy(): Boolean = Option(s3.getProxyConfiguration.getProxyHost).exists{ _ != "" }
+
+  override def getInputStream: InputStream = {
+    if (!connected) connect()
+    response.flatMap{ _.inputStream }.orNull
+  }
+
+  override def getHeaderField(n: Int): String = {
+    // n == 0 means you want the HTTP Status Line
+    // This is called from HttpURLConnection.getResponseCode()
+    if (n == 0 && responseCode != -1) {
+      s"HTTP/1.0 $responseCode ${statusMessageForCode(responseCode)}"
+    } else {
+      super.getHeaderField(n)
+    }
+  }
+
+  override def getHeaderField(field: String): String = {
+    if (!connected) connect()
+
+    field.toLowerCase match {
+      case "content-type" => response.map{ _.meta.getContentType }.orNull
+      case "content-encoding" => response.map{ _.meta.getContentEncoding }.orNull
+      case "content-length" => response.map{ _.meta.getContentLength().toString }.orNull
+      case "last-modified" => response.map{ _.meta.getLastModified }.map{ _.toInstant.atOffset(ZoneOffset.UTC) }.map{ DateTimeFormatter.RFC_1123_DATE_TIME.format }.orNull
+      case _ => null // Should return null if no value for header
+    }
+  }
+
+  override def disconnect(): Unit = {
+    response.foreach{ _.close() }
+  }
+
+  private def statusMessageForCode(code: Int): String = {
+    // I'm not sure if we care about any codes besides 200 and 404
+    code match {
+      case 200 => "OK"
+      case 404 => "Not Found"
+      case _   => "DUMMY"
+    }
+  }
+}