diff --git a/geomesa-accumulo/geomesa-accumulo-dist/pom.xml b/geomesa-accumulo/geomesa-accumulo-dist/pom.xml index 98abf487713e..be59fc011420 100644 --- a/geomesa-accumulo/geomesa-accumulo-dist/pom.xml +++ b/geomesa-accumulo/geomesa-accumulo-dist/pom.xml @@ -64,21 +64,45 @@ org.locationtech.geomesa geomesa-accumulo-spark-runtime-accumulo20_${scala.binary.version} ${project.version} + + + * + * + + org.locationtech.geomesa geomesa-accumulo-spark-runtime-accumulo21_${scala.binary.version} ${project.version} + + + * + * + + org.locationtech.geomesa geomesa-spark-converter_${scala.binary.version} + + + * + * + + org.locationtech.geomesa geomesa-accumulo-jobs_${scala.binary.version} + + + * + * + + diff --git a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh index 218c44a3ad17..7c299e65a471 100755 --- a/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh +++ b/geomesa-accumulo/geomesa-accumulo-tools/conf-filtered/dependencies.sh @@ -30,25 +30,27 @@ function dependencies() { if [[ -n "$classpath" ]]; then accumulo_version="$(get_classpath_version accumulo-core "$classpath" "$accumulo_version")" hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" zk_version="$(get_classpath_version zookeeper "$classpath" "$zk_version")" fi + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" + fi + declare -a gavs=( "org.apache.accumulo:accumulo-core:${accumulo_version}:jar" "org.apache.accumulo:accumulo-server-base:${accumulo_version}:jar" "org.apache.accumulo:accumulo-start:${accumulo_version}:jar" "org.apache.accumulo:accumulo-hadoop-mapreduce:${accumulo_version}:jar" "org.apache.zookeeper:zookeeper:${zk_version}:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" "org.apache.commons:commons-configuration2:2.10.1:jar" - "org.apache.commons:commons-text:1.11.0:jar" - "org.apache.commons:commons-collections4:4.4:jar" - "org.apache.commons:commons-vfs2:2.9.0:jar" "commons-logging:commons-logging:1.3.3:jar" - "org.apache.hadoop:hadoop-auth:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs:${hadoop_version}:jar" - "com.fasterxml.woodstox:woodstox-core:5.3.0:jar" - "org.codehaus.woodstox:stax2-api:4.2.1:jar" + "org.apache.commons:commons-text:%%commons.text.version%%:jar" + "org.apache.commons:commons-vfs2:2.9.0:jar" "com.google.guava:guava:${guava_install_version}:jar" "io.netty:netty-codec:%%netty.version%%:jar" "io.netty:netty-handler:%%netty.version%%:jar" @@ -90,20 +92,9 @@ function dependencies() { ) fi - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - ) - else - gavs+=( - "commons-configuration:commons-configuration:1.6:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then + if ! version_ge "${hadoop_version}" 3.3.0; then gavs+=( - "commons-collections:commons-collections:3.2.2:jar" + "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" ) fi diff --git a/geomesa-fs/geomesa-fs-datastore/src/main/scala/org/locationtech/geomesa/fs/data/FileSystemDataStoreFactory.scala b/geomesa-fs/geomesa-fs-datastore/src/main/scala/org/locationtech/geomesa/fs/data/FileSystemDataStoreFactory.scala index 4e75bc70c36c..187491e9bff4 100644 --- a/geomesa-fs/geomesa-fs-datastore/src/main/scala/org/locationtech/geomesa/fs/data/FileSystemDataStoreFactory.scala +++ b/geomesa-fs/geomesa-fs-datastore/src/main/scala/org/locationtech/geomesa/fs/data/FileSystemDataStoreFactory.scala @@ -98,11 +98,12 @@ object FileSystemDataStoreFactory extends GeoMesaDataStoreInfo { FileSystemDataStoreParams.ConfigsParam ) + // lazy to avoid masking classpath errors with missing hadoop + private lazy val configuration = new Configuration() + override def canProcess(params: java.util.Map[String, _]): Boolean = FileSystemDataStoreParams.PathParam.exists(params) - private val configuration = new Configuration() - object FileSystemDataStoreParams extends NamespaceParams { val WriterFileTimeout: SystemProperty = SystemProperty("geomesa.fs.writer.partition.timeout", "60s") diff --git a/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh b/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh index 0616baeceecc..acf1cff34508 100755 --- a/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh +++ b/geomesa-fs/geomesa-fs-tools/conf-filtered/dependencies.sh @@ -12,9 +12,9 @@ # Update the versions as required to match the target environment. hadoop_install_version="%%hadoop.version.recommended%%" -aws_sdk_v1_install_version="1.12.735" # latest version as of 2024/06 -aws_sdk_v2_install_version="2.25.64" # latest version as of 2024/06 -aws_crt_install_version="0.29.18" +aws_sdk_v1_install_version="1.12.780" # latest version as of 2025/01 +aws_sdk_v2_install_version="2.29.44" # latest version as of 2025/01 +aws_crt_install_version="0.33.7" # this should match the parquet desired version snappy_install_version="1.1.1.6" @@ -31,55 +31,27 @@ function dependencies() { if [[ -n "$classpath" ]]; then hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" aws_sdk_v1_version="$(get_classpath_version aws-java-sdk-core "$classpath" "$aws_sdk_v1_version")" aws_sdk_v2_version="$(get_classpath_version aws-core "$classpath" "$aws_sdk_v2_version")" snappy_version="$(get_classpath_version snappy-java "$classpath" "$snappy_version")" fi + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" + fi + declare -a gavs=( - "org.apache.hadoop:hadoop-auth:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs-client:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" "org.apache.hadoop:hadoop-aws:${hadoop_version}:jar" "org.xerial.snappy:snappy-java:${snappy_version}:jar" - "com.fasterxml.woodstox:woodstox-core:5.3.0:jar" - "org.codehaus.woodstox:stax2-api:4.2.1:jar" - "org.apache.commons:commons-configuration2:2.8.0:jar" - "commons-configuration:commons-configuration:1.6:jar" - "commons-collections:commons-collections:3.2.2:jar" - "commons-lang:commons-lang:2.6:jar" - "commons-logging:commons-logging:1.1.3:jar" - "commons-cli:commons-cli:1.2:jar" - "commons-io:commons-io:2.5:jar" - "com.google.protobuf:protobuf-java:2.5.0:jar" - "org.apache.htrace:htrace-core:3.1.0-incubating:jar" - "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" - # these are the versions used by hadoop 2.8 and 3.1 - "org.apache.httpcomponents:httpclient:4.5.2:jar" - "org.apache.httpcomponents:httpcore:4.4.4:jar" - "commons-httpclient:commons-httpclient:3.1:jar" + "commons-logging:commons-logging:1.3.3:jar" + "org.apache.httpcomponents:httpclient:4.5.13:jar" + "org.apache.httpcomponents:httpcore:4.4.13:jar" ) - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - "com.google.guava:guava:27.0-jre:jar" - ) - else - gavs+=( - "com.google.guava:guava:11.0.2:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then - gavs+=( - "commons-collections:commons-collections:3.2.2:jar" - ) - fi - # aws sdk if version_ge "${hadoop_version}" 3.4.0; then gavs+=( @@ -105,6 +77,8 @@ function dependencies() { "software.amazon.awssdk:profiles:${aws_sdk_v2_version}:jar" "software.amazon.awssdk:protocol-core:${aws_sdk_v2_version}:jar" "software.amazon.awssdk:regions:${aws_sdk_v2_version}:jar" + "software.amazon.awssdk:retries:${aws_sdk_v2_version}:jar" + "software.amazon.awssdk:retries-spi:${aws_sdk_v2_version}:jar" "software.amazon.awssdk:s3:${aws_sdk_v2_version}:jar" "software.amazon.awssdk:s3-transfer-manager:${aws_sdk_v2_version}:jar" "software.amazon.awssdk:sdk-core:${aws_sdk_v2_version}:jar" @@ -119,6 +93,8 @@ function dependencies() { "com.amazonaws:aws-java-sdk-core:${aws_sdk_v1_version}:jar" "com.amazonaws:aws-java-sdk-s3:${aws_sdk_v1_version}:jar" "com.amazonaws:aws-java-sdk-dynamodb:${aws_sdk_v1_version}:jar" + "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" + "com.google.guava:guava:27.0-jre:jar" "joda-time:joda-time:2.8.1:jar" ) fi diff --git a/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh b/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh index 952edd94a8f7..29248632cba0 100755 --- a/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh +++ b/geomesa-gt/geomesa-gt-tools/conf-filtered/dependencies.sh @@ -7,13 +7,11 @@ # http://www.opensource.org/licenses/apache2.0.php. # -# This file lists the dependencies required for running the geomesa-fs command-line tools. +# This file lists the dependencies required for running the geomesa command-line tools. # Usually these dependencies will be provided by the environment (e.g. HADOOP_HOME). # Update the versions as required to match the target environment. hadoop_install_version="%%hadoop.version.recommended%%" -# required for hadoop - make sure it corresponds to the hadoop installed version -guava_install_version="%%geotools.guava.version%%" # gets the dependencies for this module # args: @@ -25,37 +23,22 @@ function dependencies() { if [[ -n "$classpath" ]]; then hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" + fi + + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" fi declare -a gavs=( - "org.apache.hadoop:hadoop-auth:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs-client:${hadoop_version}:jar" - "commons-configuration:commons-configuration:1.6:jar" - "commons-logging:commons-logging:1.1.3:jar" - # htrace 3 required for hadoop before 2.8 - # htrace 4 required for hadoop 2.8 and later - # since they have separate package names, should be safe to install both - "org.apache.htrace:htrace-core:3.1.0-incubating:jar" - "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" - "com.google.guava:guava:${guava_install_version}:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" ) - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - ) - else - gavs+=( - "commons-configuration:commons-configuration:1.6:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then + if ! version_ge "${hadoop_version}" 3.3.0; then gavs+=( - "commons-collections:commons-collections:3.2.2:jar" + "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" ) fi diff --git a/geomesa-hbase/geomesa-hbase-datastore/src/main/scala/org/locationtech/geomesa/hbase/data/HBaseConnectionPool.scala b/geomesa-hbase/geomesa-hbase-datastore/src/main/scala/org/locationtech/geomesa/hbase/data/HBaseConnectionPool.scala index 5910dc503c2f..91e0b9eabf49 100644 --- a/geomesa-hbase/geomesa-hbase-datastore/src/main/scala/org/locationtech/geomesa/hbase/data/HBaseConnectionPool.scala +++ b/geomesa-hbase/geomesa-hbase-datastore/src/main/scala/org/locationtech/geomesa/hbase/data/HBaseConnectionPool.scala @@ -17,6 +17,7 @@ import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants} import org.apache.hadoop.security.authentication.util.KerberosUtil import org.apache.hadoop.security.{SecurityUtil, UserGroupInformation} +import org.locationtech.geomesa.hbase.HBaseSystemProperties import org.locationtech.geomesa.hbase.data.HBaseDataStoreFactory.{HBaseGeoMesaKeyTab, HBaseGeoMesaPrincipal} import org.locationtech.geomesa.hbase.data.HBaseDataStoreParams.{ConfigPathsParam, ConfigsParam, ConnectionParam, ZookeeperParam} import org.locationtech.geomesa.hbase.utils.HBaseVersions @@ -38,7 +39,7 @@ object HBaseConnectionPool extends LazyLogging { // add common resources from system property - lazy to allow object initialization if there's an error private lazy val configuration = { val base = HBaseConfiguration.create() - HBaseDataStoreFactory.ConfigPathProperty.option.foreach(addResources(base, _)) + HBaseSystemProperties.ConfigPathProperty.option.foreach(addResources(base, _)) base } diff --git a/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh b/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh index 7b5bce61aac5..05e065703a18 100755 --- a/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh +++ b/geomesa-hbase/geomesa-hbase-tools/conf-filtered/dependencies.sh @@ -17,6 +17,7 @@ hadoop_install_version="%%hadoop.version%%" zookeeper_install_version="%%zookeeper.version.recommended%%" # required for hadoop - make sure it corresponds to the hadoop installed version guava_install_version="%%hbase.guava.version%%" +snappy_install_version="1.1.10.4" # gets the dependencies for this module # args: @@ -28,12 +29,20 @@ function dependencies() { local hbase_thirdparty_version="$hbase_thirdparty_install_version" local hadoop_version="$hadoop_install_version" local zk_version="$zookeeper_install_version" + local snappy_version="$snappy_install_version" if [[ -n "$classpath" ]]; then hbase_version="$(get_classpath_version hbase-client "$classpath" "$hbase_version")" hbase_thirdparty_version="$(get_classpath_version hbase-shaded-protobuf "$classpath" "$hbase_thirdparty_version")" hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" zk_version="$(get_classpath_version zookeeper "$classpath" "$zk_version")" + snappy_version="$(get_classpath_version snappy-java "$classpath" "$snappy_version")" + fi + + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" fi declare -a gavs=( @@ -41,23 +50,16 @@ function dependencies() { "org.apache.hbase:hbase-common:${hbase_version}:jar" "org.apache.hbase:hbase-hadoop-compat:${hbase_version}:jar" "org.apache.hbase:hbase-protocol:${hbase_version}:jar" - "com.google.protobuf:protobuf-java:2.5.0:jar" + "com.google.protobuf:protobuf-java:%%hbase.protobuf.version%%:jar" "org.apache.zookeeper:zookeeper:${zk_version}:jar" - "org.apache.hadoop:hadoop-auth:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs-client:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}:jar" - "com.fasterxml.woodstox:woodstox-core:5.3.0:jar" - "org.codehaus.woodstox:stax2-api:4.2.1:jar" - "commons-collections:commons-collections:3.2.2:jar" - "commons-configuration:commons-configuration:1.6:jar" - "commons-lang:commons-lang:2.6:jar" - "commons-logging:commons-logging:1.1.3:jar" - "commons-cli:commons-cli:1.2:jar" - "commons-io:commons-io:2.5:jar" - "org.apache.commons:commons-configuration2:2.8.0:jar" - "javax.servlet:servlet-api:2.4:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" + "org.xerial.snappy:snappy-java:${snappy_version}:jar" + "commons-logging:commons-logging:1.3.3:jar" + "org.apache.httpcomponents:httpclient:4.5.13:jar" + "org.apache.httpcomponents:httpcore:4.4.13:jar" + "commons-cli:commons-cli:1.5.0:jar" + "commons-io:commons-io:2.11.0:jar" "io.netty:netty-buffer:%%netty.version%%:jar" "io.netty:netty-codec:%%netty.version%%:jar" "io.netty:netty-common:%%netty.version%%:jar" @@ -68,25 +70,18 @@ function dependencies() { "io.netty:netty-transport-native-epoll:%%netty.version%%:jar:linux-x86_64" "io.netty:netty-transport-native-unix-common:%%netty.version%%:jar" "io.netty:netty:3.10.6.Final:jar" - # htrace 3 required for hadoop before 2.8 - # htrace 4 required for hadoop 2.8 and later - # since they have separate package names, should be safe to install both - "org.apache.htrace:htrace-core:3.1.0-incubating:jar" - "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" "io.dropwizard.metrics:metrics-core:3.2.6:jar" "com.google.guava:guava:${guava_install_version}:jar" ) - local hbase_maj_ver - hbase_maj_ver="$([[ "$hbase_version" =~ ([0-9][0-9]*)\. ]] && echo "${BASH_REMATCH[1]}")" - # additional dependencies that depend on the major version - if [[ "$hbase_maj_ver" -ge 2 ]]; then + if version_ge "${hbase_version}" 2.0.0; then gavs+=( "org.apache.hbase:hbase-mapreduce:${hbase_version}:jar" "org.apache.hbase:hbase-protocol-shaded:${hbase_version}:jar" "org.apache.hbase.thirdparty:hbase-shaded-miscellaneous:${hbase_thirdparty_version}:jar" "org.apache.hbase.thirdparty:hbase-shaded-netty:${hbase_thirdparty_version}:jar" + "org.apache.hbase.thirdparty:hbase-shaded-netty-tcnative:${hbase_thirdparty_version}:jar" "org.apache.hbase.thirdparty:hbase-shaded-protobuf:${hbase_thirdparty_version}:jar" "org.apache.hbase.thirdparty:hbase-unsafe:${hbase_thirdparty_version}:jar" "io.opentelemetry:opentelemetry-api:1.15.0:jar" @@ -99,20 +94,9 @@ function dependencies() { ) fi - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - ) - else - gavs+=( - "commons-configuration:commons-configuration:1.6:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then + if ! version_ge "${hadoop_version}" 3.3.0; then gavs+=( - "commons-collections:commons-collections:3.2.2:jar" + "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" ) fi diff --git a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh index eb4744333417..1eba86bcf5c5 100755 --- a/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh +++ b/geomesa-lambda/geomesa-lambda-tools/conf-filtered/dependencies.sh @@ -30,34 +30,31 @@ function dependencies() { local kafka_version="$kafka_install_version" if [[ -n "$classpath" ]]; then - accumulo_version="$(get_classpath_version accumulo-core "$classpath" $accumulo_version)" - hadoop_version="$(get_classpath_version hadoop-common "$classpath" $hadoop_version)" - zk_version="$(get_classpath_version zookeeper "$classpath" $zk_version)" + accumulo_version="$(get_classpath_version accumulo-core "$classpath" "$accumulo_version")" + hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" + zk_version="$(get_classpath_version zookeeper "$classpath" "$zk_version")" kafka_version="$(get_classpath_version kafka-clients "$classpath" $kafka_version)" fi + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" + fi + declare -a gavs=( "org.apache.accumulo:accumulo-core:${accumulo_version}:jar" "org.apache.accumulo:accumulo-server-base:${accumulo_version}:jar" "org.apache.accumulo:accumulo-start:${accumulo_version}:jar" "org.apache.accumulo:accumulo-hadoop-mapreduce:${accumulo_version}:jar" "org.apache.zookeeper:zookeeper:${zk_version}:jar" - "org.apache.commons:commons-configuration2:2.8.0:jar" - "org.apache.commons:commons-text:1.10.0:jar" - "org.apache.commons:commons-collections4:4.4:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" + "org.apache.commons:commons-configuration2:2.10.1:jar" + "commons-logging:commons-logging:1.3.3:jar" + "org.apache.commons:commons-text:%%commons.text.version%%:jar" "org.apache.commons:commons-vfs2:2.9.0:jar" - "commons-collections:commons-collections:3.2.2:jar" - "commons-logging:commons-logging:1.2:jar" - "org.apache.hadoop:hadoop-auth:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-hdfs:${hadoop_version}:jar" "org.apache.kafka:kafka-clients:${kafka_version}:jar" - "commons-logging:commons-logging:1.1.3:jar" - "org.apache.htrace:htrace-core:3.1.0-incubating:jar" - "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" - "com.yammer.metrics:metrics-core:2.2.0:jar" - "com.fasterxml.woodstox:woodstox-core:5.3.0:jar" - "org.codehaus.woodstox:stax2-api:4.2.1:jar" "com.google.guava:guava:${guava_install_version}:jar" "net.sf.jopt-simple:jopt-simple:5.0.4:jar" "io.netty:netty-codec:%%netty.version%%:jar" @@ -71,33 +68,38 @@ function dependencies() { # add accumulo 2.1 jars if needed if version_ge "${accumulo_version}" 2.1.0; then + local micrometer_version + local opentelemetry_version + + if version_ge "${accumulo_version}" 2.1.3; then + micrometer_version="1.12.2" + opentelemetry_version="1.34.1" + else + # these versions seem compatible even though they're not the exact versions shipped + micrometer_version="1.11.1" + opentelemetry_version="1.27.0" + gavs+=( + "io.opentelemetry:opentelemetry-semconv:${opentelemetry_version}-alpha:jar" + ) + fi + gavs+=( "org.apache.thrift:libthrift:%%thrift.version%%:jar" - "io.opentelemetry:opentelemetry-api:1.19.0:jar" - "io.opentelemetry:opentelemetry-context:1.19.0:jar" - "io.opentelemetry:opentelemetry-semconv:1.19.0-alpha:jar" - "io.micrometer:micrometer-core:1.9.6:jar" + "io.micrometer:micrometer-core:${micrometer_version}:jar" + "io.micrometer:micrometer-commons:${micrometer_version}:jar" + "io.opentelemetry:opentelemetry-api:${opentelemetry_version}:jar" + "io.opentelemetry:opentelemetry-context:${opentelemetry_version}:jar" ) else gavs+=( "org.apache.thrift:libthrift:0.12.0:jar" + "org.apache.htrace:htrace-core:3.1.0-incubating:jar" ) fi - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - ) - else - gavs+=( - "commons-configuration:commons-configuration:1.6:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then + if ! version_ge "${hadoop_version}" 3.3.0; then gavs+=( - "commons-collections:commons-collections:3.2.2:jar" + "org.apache.htrace:htrace-core4:4.1.0-incubating:jar" ) fi diff --git a/geomesa-tools/common-env/hadoop-env.sh b/geomesa-tools/common-env/hadoop-env.sh index 7885310d5a6f..068e294cd886 100644 --- a/geomesa-tools/common-env/hadoop-env.sh +++ b/geomesa-tools/common-env/hadoop-env.sh @@ -23,6 +23,8 @@ # Hadoop conf directory export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/etc/hadoop}" +export AWS_JAVA_V1_DISABLE_DEPRECATION_ANNOUNCEMENT=true + # Hadoop CDH configuration # Setting this variable to "1" will configure classpath settings for Hadoop # CDH. HADOOP_HOME and HADOOP_CONF_DIR are still used. diff --git a/geomesa-tools/conf-filtered/geomesa-env.sh b/geomesa-tools/conf-filtered/geomesa-env.sh index d495fcf96069..5109238a2323 100644 --- a/geomesa-tools/conf-filtered/geomesa-env.sh +++ b/geomesa-tools/conf-filtered/geomesa-env.sh @@ -338,17 +338,15 @@ function expand_classpath() { local classpath="$1" local expanded="" - for e in ${classpath//:/ }; do - if [[ $e = *\* && -d "${e%\*}" ]]; then - for f in ls $e; do - expanded+=":${e%\*}$f" - done + for entry in ${classpath//:/ }; do + if [[ $entry = *\* && -d "${entry%\*}" ]]; then + expanded+=":$(find "-L" "${entry%\*}" "-maxdepth" "1" "-type" "f" "-iname" "*.jar" "-not" "-iname" "*-sources.jar" "-not" "-iname" "*-tests.jar" | paste -sd: -)" else - expanded+=":$e" + expanded+=":$entry" fi done - echo "${expanded:1}" + fix_classpath_format "${expanded:1}" } # remove leading/trailing/double colons diff --git a/geomesa-tools/conf-filtered/parquet-dependencies.sh b/geomesa-tools/conf-filtered/parquet-dependencies.sh index 71cb58a4ec94..43a96a1adfdc 100755 --- a/geomesa-tools/conf-filtered/parquet-dependencies.sh +++ b/geomesa-tools/conf-filtered/parquet-dependencies.sh @@ -22,34 +22,19 @@ function dependencies() { if [[ -n "$classpath" ]]; then hadoop_version="$(get_classpath_version hadoop-common "$classpath" "$hadoop_version")" + hadoop_version="$(get_classpath_version hadoop-client-api "$classpath" "$hadoop_version")" + fi + + if [[ "$hadoop_version" == "3.2.3" ]]; then + echo >&2 "WARNING Updating Hadoop version from 3.2.3 to 3.2.4 due to invalid client-api Maven artifacts" + hadoop_version="3.2.4" fi declare -a gavs=( - "org.apache.hadoop:hadoop-common:${hadoop_version}:jar" - "com.fasterxml.woodstox:woodstox-core:5.3.0:jar" - "org.codehaus.woodstox:stax2-api:4.2.1:jar" - "org.apache.commons:commons-configuration2:2.8.0:jar" - "commons-collections:commons-collections:3.2.2:jar" - "commons-logging:commons-logging:1.2:jar" + "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" + "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" ) - # add hadoop 3+ jars if needed - if version_ge "${hadoop_version}" 3.0.0; then - gavs+=( - "org.apache.hadoop:hadoop-client-api:${hadoop_version}:jar" - "org.apache.hadoop:hadoop-client-runtime:${hadoop_version}:jar" - ) - else - gavs+=( - "commons-configuration:commons-configuration:1.6:jar" - ) - fi - if ! version_ge "${hadoop_version}" 3.4.0; then - gavs+=( - "commons-collections:commons-collections:3.2.2:jar" - ) - fi - echo "${gavs[@]}" | tr ' ' '\n' | sort | tr '\n' ' ' } diff --git a/geomesa-tools/src/main/scala/org/locationtech/geomesa/tools/utils/ParameterConverters.scala b/geomesa-tools/src/main/scala/org/locationtech/geomesa/tools/utils/ParameterConverters.scala index 113429e223d0..d29d7910c3ed 100644 --- a/geomesa-tools/src/main/scala/org/locationtech/geomesa/tools/utils/ParameterConverters.scala +++ b/geomesa-tools/src/main/scala/org/locationtech/geomesa/tools/utils/ParameterConverters.scala @@ -15,8 +15,8 @@ import org.geotools.filter.text.ecql.ECQL import org.locationtech.geomesa.convert.Modes.ErrorMode import org.locationtech.geomesa.tools.`export`.ExportFormat import org.locationtech.geomesa.utils.geotools.converters.FastConverter -import org.locationtech.geomesa.utils.text.DurationParsing import org.locationtech.geomesa.utils.text.Suffixes.Memory +import org.locationtech.geomesa.utils.text.{DurationParsing, TextTools} import java.util.Date import scala.concurrent.duration.Duration @@ -60,13 +60,9 @@ object ParameterConverters { class ExportFormatConverter(name: String) extends BaseConverter[ExportFormat](name) { override def convert(value: String): ExportFormat = { - try { - ExportFormat(value).getOrElse { - throw new ParameterException(s"Invalid format '$value'. Valid values are " + - ExportFormat.Formats.flatMap(f => f.extensions +: f.name).distinct.mkString("'", "', '", "'")) - } - } catch { - case NonFatal(e) => throw new ParameterException(getErrorString(value, s"format: $e")) + ExportFormat(value).getOrElse { + val valid = ExportFormat.Formats.flatMap(f => f.extensions :+ f.name).distinct + throw new ParameterException(getErrorString(value, s"an export format. Valid formats are: ${TextTools.wordList(valid)}")) } } } diff --git a/pom.xml b/pom.xml index 7b8f7cd21dd5..26144b33bce7 100644 --- a/pom.xml +++ b/pom.xml @@ -218,7 +218,7 @@ ${accumulo.version} ${hbase.version} - 4.1.5 + 4.1.9 3.11.14 ${cassandra.version} ${hadoop.version} @@ -1793,6 +1793,7 @@ commons-collections ${commons.collections.version} + org.apache.commons commons-collections4