From 6b3be2e742294ad34252e38fc7128c942c580462 Mon Sep 17 00:00:00 2001 From: Derek Moore Date: Mon, 19 Dec 2022 16:24:26 -0600 Subject: [PATCH 1/2] bump version to 0.3.0 Signed-off-by: Derek Moore --- README.md | 16 ++++++++-------- scripts/env.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index cfcc876..e8458db 100644 --- a/README.md +++ b/README.md @@ -118,8 +118,8 @@ If you would rather use a more automated deployment method, skip to the next sec 3. Upload the artifact: - group: io.pravega - artifact: flink-tools - - version: 0.2.0 - - file: flink-tools/build/libs/pravega-flink-tools-0.2.0.jar + - version: 0.3.0 + - file: flink-tools/build/libs/pravega-flink-tools-0.3.0.jar 4. Create Flink Cluster. - Name: stream-to-file @@ -130,7 +130,7 @@ If you would rather use a more automated deployment method, skip to the next sec 5. Create New App. - Name: stream-to-file - Main Application File Type: maven - - Main Application File: io.pravega:fliink-tools:0.2.0 + - Main Application File: io.pravega:flink-tools:0.3.0 - Main Class: io.pravega.flinktools.StreamToFileJob - Cluster Selectors: name: stream-to-file - Parallelism: 1 @@ -401,8 +401,8 @@ Below shows how to deploy this Flink job using the SDP UI. 2. Upload the artifact: - group: io.pravega - artifact: flink-tools - - version: 0.2.0 - - file: flink-tools/build/libs/pravega-flink-tools-0.2.0.jar + - version: 0.3.0 + - file: flink-tools/build/libs/pravega-flink-tools-0.3.0.jar 3. Create Flink Cluster. - Name: sample-data-generator-job @@ -416,7 +416,7 @@ Below shows how to deploy this Flink job using the SDP UI. 4. Create New App. - Name: sample-data-generator-job - Main Application File Type: maven - - Main Application File: io.pravega:fliink-tools:0.2.0 + - Main Application File: io.pravega:flink-tools:0.3.0 - Main Class: io.pravega.flinktools.SampleDataGeneratorJob - Cluster Selectors: name: sample-data-generator-job - Parallelism: 1 @@ -517,8 +517,8 @@ Follow these steps to release a new version of Flink Tools to Github Releases. git push --tags ```` -3. Monitor Travis CI at - https://travis-ci.com/github/pravega/flink-tools. +3. Monitor GitHub Actions CI at + https://github.com/pravega/flink-tools/actions. 4. When complete, the installation archive will be available at https://github.com/pravega/flink-tools/releases. diff --git a/scripts/env.sh b/scripts/env.sh index eae8630..a0628e9 100755 --- a/scripts/env.sh +++ b/scripts/env.sh @@ -16,7 +16,7 @@ APP_NAME=flink-tools APP_GROUP_ID=${APP_GROUP_ID:-io.pravega} APP_ARTIFACT_ID=${APP_ARTIFACT_ID:-flink-tools} # Change line below to bump the application version. -APP_VERSION=${APP_VERSION:-0.2.15} +APP_VERSION=${APP_VERSION:-0.3.0} GRADLE_OPTIONS="${GRADLE_OPTIONS:-"-Pversion=${APP_VERSION}"}" FLINK_IMAGE_TAG="1.15.2-2.12-1.3.1-20-f59a7da" NEW_IMAGE_TAG="${FLINK_IMAGE_TAG}-hadoop2.8.3" From 07c0a084663973bc70313f316b09270214864839 Mon Sep 17 00:00:00 2001 From: Derek Moore Date: Tue, 20 Dec 2022 09:22:03 -0600 Subject: [PATCH 2/2] upgrade parquet-tools to parquet-cli Signed-off-by: Derek Moore --- README.md | 6 +++--- scripts/{parquet-tools.sh => parquet-cli.sh} | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) rename scripts/{parquet-tools.sh => parquet-cli.sh} (68%) diff --git a/README.md b/README.md index e8458db..7d004c5 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ If the Parquet file is located on a standard Linux file system (including NFS), you can use a command similar to the following to view the content. ```shell script -scripts/parquet-tools.sh cat /tmp/sample1-parquet/2020-08-19--03/part-0-887 +scripts/parquet-cli.sh cat /tmp/sample1-parquet/2020-08-19--03/part-0-887 ``` If the Parquet file is located on an HDFS cluster in Kubernetes, @@ -276,8 +276,8 @@ you can use commands similar to the following to view the content. ```shell script scripts/hadoop-bash.sh root@hadoop-8c428aa0-76c0-4f42-8bea-2fc1e8300f78:~# -wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-tools/1.11.1/parquet-tools-1.11.1.jar -hadoop jar parquet-tools-1.11.1.jar cat hdfs://hadoop-hadoop-hdfs-nn.examples.svc.cluster.local:9000/tmp/sample1-parquet/2020-08-19--03/part-0-887 +wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-cli/1.12.2/parquet-cli-1.12.2-runtime.jar +hadoop jar parquet-cli-1.12.2-runtime.jar org.apache.parquet.cli.Main cat hdfs://hadoop-hadoop-hdfs-nn.examples.svc.cluster.local:9000/tmp/sample1-parquet/2020-08-19--03/part-0-887 ``` ## Stream-to-CSV-File: Continuously copying a Pravega stream to CSV files diff --git a/scripts/parquet-tools.sh b/scripts/parquet-cli.sh similarity index 68% rename from scripts/parquet-tools.sh rename to scripts/parquet-cli.sh index ebaffde..5dc26f2 100755 --- a/scripts/parquet-tools.sh +++ b/scripts/parquet-cli.sh @@ -7,6 +7,6 @@ # # http://www.apache.org/licenses/LICENSE-2.0 set -ex -JAR_FILE=/tmp/parquet-tools-1.11.1.jar -[ -f ${JAR_FILE} ] || wget -O ${JAR_FILE} https://repo1.maven.org/maven2/org/apache/parquet/parquet-tools/1.11.1/parquet-tools-1.11.1.jar -docker run --rm -it -v /tmp:/tmp -v /desdp:/desdp danisla/hadoop:2.9.0 hadoop jar /tmp/parquet-tools-1.11.1.jar $* +JAR_FILE=/tmp/parquet-cli-1.12.2-runtime.jar +[ -f ${JAR_FILE} ] || wget -O ${JAR_FILE} https://repo1.maven.org/maven2/org/apache/parquet/parquet-cli/1.12.2/parquet-cli-1.12.2-runtime.jar +docker run --rm -it -v /tmp:/tmp -v /desdp:/desdp danisla/hadoop:2.9.0 hadoop jar /tmp/parquet-cli-1.12.2-runtime.jar org.apache.parquet.cli.Main $*