Skip to content

Commit

Permalink
changed jumbo aws-java-sdk-bundle dependency to only required modules
Browse files Browse the repository at this point in the history
hadoop-aws depends on aws-java-sdk-bundle which is a really big jar most of which is not used
by hadoop-aws, so excluded that dependency and instead add only the individual modules required
by hadoop-aws (tested by building hadoop-aws from source with only those dependencies)
  • Loading branch information
sumwale committed Jun 26, 2021
1 parent 8c7c02e commit 1e636db
Show file tree
Hide file tree
Showing 4 changed files with 10 additions and 3 deletions.
3 changes: 2 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ allprojects {
junitVersion = '4.12'
mockitoVersion = '1.10.19'
hadoopVersion = '3.2.0'
awsSdkVersion = '1.11.375'
gcsHadoop2ConnectorVersion = 'hadoop2-2.0.0'
gcsHadoop3ConnectorVersion = 'hadoop3-2.0.0'
sparkAvroVersion = '4.0.0'
Expand Down Expand Up @@ -242,7 +243,7 @@ if (!hasJdbcConnectorProject) {
String sparkPython = 'python'
def checkResult = exec {
ignoreExitValue = true
commandLine 'sh', '-c', 'python --version | grep -Eq "( 3\\.[0-7])|( 2\\.)"'
commandLine 'sh', '-c', 'python --version 2>/dev/null | grep -Eq "( 3\\.[0-7])|( 2\\.)"'
}
if (checkResult.exitValue != 0) {
checkResult = exec {
Expand Down
6 changes: 6 additions & 0 deletions cluster/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,13 @@ dependencies {
exclude(group: 'com.google.protobuf', module: 'protobuf-java')
exclude(group: 'com.jcraft', module: 'jsch')
exclude(group: 'org.apache.directory.server', module: 'apacheds-kerberos-codec')
// exclude jumbo aws-java-sdk-bundle and only include individual modules used by hadoop-aws
exclude(group: 'com.amazonaws', module: 'aws-java-sdk-bundle')
}
compile group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: awsSdkVersion
compile group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: awsSdkVersion
compile group: 'com.amazonaws', name: 'aws-java-sdk-dynamodb', version: awsSdkVersion
compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: jacksonVersion

compile "io.snappydata:spark-metrics:${snappySparkMetricsLibVersion}"

Expand Down
2 changes: 1 addition & 1 deletion examples/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ if (!(new File(rootDir, 'cluster/build.gradle').exists())) {

testCompile "org.scalatest:scalatest_${scalaBinaryVersion}:${scalatestVersion}"
testCompile "commons-io:commons-io:2.6"
testCompile "io.snappydata:gemfire-core:1.6.3"
testCompile project(':snappy-store:gemfire-core')
testRuntime "org.pegdown:pegdown:${pegdownVersion}"
}

Expand Down
2 changes: 1 addition & 1 deletion spark
Submodule spark updated from 552203 to 1456d1

0 comments on commit 1e636db

Please sign in to comment.