-
Notifications
You must be signed in to change notification settings - Fork 121
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
S3 maven artifacts support #275
base: master
Are you sure you want to change the base?
Changes from all commits
bc5dbf4
2ee03c5
bcf58f5
881f665
792553a
f70c94d
f5043f4
1b4ac18
c7ca89b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import") | ||
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") | ||
java_library( | ||
name = "aws_java_sdk_core", | ||
exports = [ | ||
"//external:jar/com/amazonaws/aws_java_sdk_core" | ||
], | ||
runtime_deps = [ | ||
"//3rdparty/jvm/com/fasterxml/jackson/core:jackson_databind", | ||
"//3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson_dataformat_cbor", | ||
"//3rdparty/jvm/commons_logging:commons_logging", | ||
"//3rdparty/jvm/joda_time:joda_time", | ||
"//3rdparty/jvm/org/apache/httpcomponents:httpclient", | ||
"//3rdparty/jvm/software/amazon/ion:ion_java" | ||
], | ||
visibility = [ | ||
"//visibility:public" | ||
] | ||
) | ||
|
||
|
||
|
||
java_library( | ||
name = "aws_java_sdk_kms", | ||
exports = [ | ||
"//external:jar/com/amazonaws/aws_java_sdk_kms" | ||
], | ||
runtime_deps = [ | ||
":aws_java_sdk_core", | ||
":jmespath_java" | ||
], | ||
visibility = [ | ||
"//3rdparty/jvm:__subpackages__" | ||
] | ||
) | ||
|
||
|
||
|
||
java_library( | ||
name = "aws_java_sdk_s3", | ||
exports = [ | ||
"//external:jar/com/amazonaws/aws_java_sdk_s3" | ||
], | ||
runtime_deps = [ | ||
":aws_java_sdk_core", | ||
":aws_java_sdk_kms", | ||
":jmespath_java" | ||
], | ||
visibility = [ | ||
"//visibility:public" | ||
] | ||
) | ||
|
||
|
||
|
||
java_library( | ||
name = "aws_java_sdk_sts", | ||
exports = [ | ||
"//external:jar/com/amazonaws/aws_java_sdk_sts" | ||
], | ||
runtime_deps = [ | ||
":aws_java_sdk_core", | ||
":jmespath_java" | ||
], | ||
visibility = [ | ||
"//visibility:public" | ||
] | ||
) | ||
|
||
|
||
|
||
java_library( | ||
name = "jmespath_java", | ||
exports = [ | ||
"//external:jar/com/amazonaws/jmespath_java" | ||
], | ||
runtime_deps = [ | ||
"//3rdparty/jvm/com/fasterxml/jackson/core:jackson_databind" | ||
], | ||
visibility = [ | ||
"//3rdparty/jvm:__subpackages__" | ||
] | ||
) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import") | ||
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") | ||
java_library( | ||
name = "commons_logging", | ||
exports = [ | ||
"//external:jar/commons_logging/commons_logging" | ||
], | ||
visibility = [ | ||
"//3rdparty/jvm:__subpackages__" | ||
] | ||
) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import") | ||
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") | ||
java_library( | ||
name = "joda_time", | ||
exports = [ | ||
"//external:jar/joda_time/joda_time" | ||
], | ||
visibility = [ | ||
"//3rdparty/jvm:__subpackages__" | ||
] | ||
) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import") | ||
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") | ||
java_library( | ||
name = "ivy", | ||
exports = [ | ||
"//external:jar/org/apache/ivy/ivy" | ||
], | ||
visibility = [ | ||
"//visibility:public" | ||
] | ||
) | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
load("@io_bazel_rules_scala//scala:scala_import.bzl", "scala_import") | ||
load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") | ||
java_library( | ||
name = "ion_java", | ||
exports = [ | ||
"//external:jar/software/amazon/ion/ion_java" | ||
], | ||
visibility = [ | ||
"//3rdparty/jvm:__subpackages__" | ||
] | ||
) | ||
|
||
|
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
package com.github.johnynek.bazel_deps | ||
|
||
import java.net.{URL, URLConnection, URLStreamHandler} | ||
|
||
final class S3Handler extends URLStreamHandler { | ||
def openConnection(url: URL): URLConnection = new S3URLConnection(url) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
package com.github.johnynek.bazel_deps | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is this file mostly original code, or largely copied? If copied can you provide some git permalinks here to where it came from to help others. Also, is there any way we could use the compiled version of the original code rather than adding as much code here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Larged copied, I was doing this just to unblock me, so I didn't really think it through a lot. That being said, I'm really open to make any modifications to help others to use it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. My first thinking of bringing the code here was to avoid any reference to |
||
import com.amazonaws.AmazonServiceException | ||
import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object} | ||
import com.github.johnynek.bazel_deps.S3URLHandler | ||
import java.io.InputStream | ||
import java.net.{HttpURLConnection, URL} | ||
import java.time.ZoneOffset | ||
import java.time.format.DateTimeFormatter | ||
|
||
object S3URLConnection { | ||
private val s3: S3URLHandler = new S3URLHandler() | ||
} | ||
|
||
/** | ||
* Implements an HttpURLConnection for compatibility with Coursier (https://github.com/coursier/coursier) | ||
*/ | ||
final class S3URLConnection(url: URL) extends HttpURLConnection(url) { | ||
import S3URLConnection.s3 | ||
|
||
private trait S3Response extends AutoCloseable { | ||
def meta: ObjectMetadata | ||
def inputStream: Option[InputStream] | ||
} | ||
|
||
private case class HEADResponse(meta: ObjectMetadata) extends S3Response { | ||
def close(): Unit = {} | ||
def inputStream: Option[InputStream] = None | ||
} | ||
|
||
private case class GETResponse(obj: S3Object) extends S3Response { | ||
def meta: ObjectMetadata = obj.getObjectMetadata | ||
def inputStream: Option[InputStream] = Option(obj.getObjectContent()) | ||
def close(): Unit = obj.close() | ||
} | ||
|
||
private[this] var response: Option[S3Response] = None | ||
|
||
def connect(): Unit = { | ||
val (client, bucket, key) = s3.getClientBucketAndKey(url) | ||
|
||
try { | ||
response = getRequestMethod.toLowerCase match { | ||
case "head" => Option(HEADResponse(client.getObjectMetadata(bucket, key))) | ||
case "get" => Option(GETResponse(client.getObject(bucket, key))) | ||
case "post" => ??? | ||
case "put" => ??? | ||
case _ => throw new IllegalArgumentException("Invalid request method: "+getRequestMethod) | ||
} | ||
|
||
responseCode = if (response.isEmpty) 404 else 200 | ||
} catch { | ||
case ex: AmazonServiceException => responseCode = ex.getStatusCode | ||
} | ||
|
||
// Also set the responseMessage (an HttpURLConnection field) for better compatibility | ||
responseMessage = statusMessageForCode(responseCode) | ||
connected = true | ||
} | ||
|
||
def usingProxy(): Boolean = Option(s3.getProxyConfiguration.getProxyHost).exists{ _ != "" } | ||
|
||
override def getInputStream: InputStream = { | ||
if (!connected) connect() | ||
response.flatMap{ _.inputStream }.orNull | ||
} | ||
|
||
override def getHeaderField(n: Int): String = { | ||
// n == 0 means you want the HTTP Status Line | ||
// This is called from HttpURLConnection.getResponseCode() | ||
if (n == 0 && responseCode != -1) { | ||
s"HTTP/1.0 $responseCode ${statusMessageForCode(responseCode)}" | ||
} else { | ||
super.getHeaderField(n) | ||
} | ||
} | ||
|
||
override def getHeaderField(field: String): String = { | ||
if (!connected) connect() | ||
|
||
field.toLowerCase match { | ||
case "content-type" => response.map{ _.meta.getContentType }.orNull | ||
case "content-encoding" => response.map{ _.meta.getContentEncoding }.orNull | ||
case "content-length" => response.map{ _.meta.getContentLength().toString }.orNull | ||
case "last-modified" => response.map{ _.meta.getLastModified }.map{ _.toInstant.atOffset(ZoneOffset.UTC) }.map{ DateTimeFormatter.RFC_1123_DATE_TIME.format }.orNull | ||
case _ => null // Should return null if no value for header | ||
} | ||
} | ||
|
||
override def disconnect(): Unit = { | ||
response.foreach{ _.close() } | ||
} | ||
|
||
private def statusMessageForCode(code: Int): String = { | ||
// I'm not sure if we care about any codes besides 200 and 404 | ||
code match { | ||
case 200 => "OK" | ||
case 404 => "Not Found" | ||
case _ => "DUMMY" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd rather this be something like: |
||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm open to merging something like this, but I'd like to add some documentation on how this code is working and how users can interact with it. This is a nice link, but we need more here, I think.