Merge pull request #979 from exasol/develop

Changelist:

- * #944: Removed exaudfclient_bin_py3 (#975)
- * #967: Added ctpg script options parser (#968)
- * #972 and #973: Refactorings for Script Options parser (#976)
- * #977: Fixed Trivy update cache workflow (#978)

.github/workflows/check_bazel_tests.yml

@@ -37,6 +37,6 @@ jobs:
       working-directory: ./exaudfclient/
     - name: Script Options Parser Tests
       run: |
-        bazel test //base/script_options_parser/test/...
+        bazel test //base/script_options_parser/...
       working-directory: ./exaudfclient/
 

.github/workflows/update_trivy_cache.yaml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - name: Run install dependencies
         run: |
-          sudo apt update && sudo apt install -y awscli curl wget apt-transport-https gnupg lsb-release
+          sudo apt update && sudo apt install -y curl wget apt-transport-https gnupg lsb-release
           wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
           echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
           sudo apt-get update

doc/changes/changes_8.4.0.md

@@ -22,7 +22,14 @@ This release uses version 1.0.0 of the container tool.
 ## Refactorings
 
   - #950: Refactored existing script-options parser
+  - #944: Removed exaudfclient_bin_py3
+  - #967: Added ctpg script options parser
+  - #972: Refactor exception handling for Script Options parser
+  - #973: Support new lines and white spaces at the beginning in script options values
 
+## Bugs
+  - #977: Fixed Trivy update cache workflow
+
 ## Doc
 
  - #954: added release checklist

exaudfclient/.bazelrc

@@ -3,11 +3,11 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_
 build:benchmark --define benchmark=true
 build:r --define r=true
 build:java  --define java=true --action_env=JAVA_PREFIX
-build:python --define python=true  --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
-build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3
-build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3
-build:static-binary-py3 //:exaudfclient_py3_static
-build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3
+build:python --define python=true  --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
+build:fast-binary --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient
+build:slow-wrapper --define binary_type=slow_wrapper //:exaudfclient
+build:static-binary //:exaudfclient_static
+build:test-binaries --config=static-binary --config=slow-wrapper
 build:verbose --copt='-v' --subcommands --verbose_failures --announce_rc
 #TODO test linkopts="-flto"
 build:optimize --copt="-g0" --copt="-DNDEBUG" --copt=-fstack-protector-strong --copt=-fomit-frame-pointer --copt=-ffunction-sections --copt=-fdata-sections --copt="-O3" --copt="-U_FORTIFY_SOURCE" --copt="-flto" --copt="-fuse-linker-plugin"

exaudfclient/BUILD

@@ -89,17 +89,7 @@ cc_binary(
     name = "exaudfclient_bin",
     srcs = ["exaudfclient.cc", "//base:load_dynamic"],
     linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
-    deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_R_DEPS+
-            ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"],
-    defines = VM_ENABLED_DEFINES,
-    data = ["//base:libexaudflib_complete.so"]
-)
-
-cc_binary(
-    name = "exaudfclient_py3_bin",
-    srcs = ["exaudfclient.cc", "//base:load_dynamic"],
-    linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
-    deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+VM_R_DEPS+
+    deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+VM_R_DEPS+
             ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"],
     defines = VM_ENABLED_DEFINES,
     data = ["//base:libexaudflib_complete.so"]
@@ -119,10 +109,10 @@ cc_binary(
 ## as dependency as it is a binary for bazel.
 
 cc_binary(
-    name = "exaudfclient_py3_static_bin",
+    name = "exaudfclient_static_bin",
     srcs = ["exaudfclient.cc",  "//base:load_dynamic"],
     linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace
-    deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+VM_R_DEPS+
+    deps = ["//base/exaudflib:header", "//base/utils:utils"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+VM_R_DEPS+
             ["//base/exaudflib:exaudflib-deps", "//base/swig_factory:swig_factory"] + [ "@zmq//:zmq", "@protobuf//:protobuf"],
     defines = VM_ENABLED_DEFINES,
     data = ["//base:libexaudflib_complete.so"],
@@ -147,33 +137,33 @@ sh_library(
 )
 
 
-SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """
-FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """
-CREATE_BINARY_PY3_SCRIPT=select({
-        "//:fast_binary": FAST_BINARY_PY3,
-        "//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3,
-        "//conditions:default": FAST_BINARY_PY3
+SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location //base:exaudfclient.template.sh)" """
+FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """
+CREATE_BINARY_SCRIPT=select({
+        "//:fast_binary": FAST_BINARY,
+        "//:slow_wrapper": SLOW_WRAPPER_BINARY,
+        "//conditions:default": FAST_BINARY
     })
 genrule(
-    name = "exaudfclient_py3",
-    cmd = CREATE_BINARY_PY3_SCRIPT,
-    outs = ["exaudfclient_py3"],
-    srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
+    name = "exaudfclient",
+    cmd = CREATE_BINARY_SCRIPT,
+    outs = ["exaudfclient"],
+    srcs = [":exaudfclient_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
     output_to_bindir = True
 )
 
-SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """
-FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """
-CREATE_STATIC_BINARY_PY3_SCRIPT=select({
-        "//:fast_binary": FAST_BINARY_STATIC_PY3,
-        "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3,
-        "//conditions:default": FAST_BINARY_STATIC_PY3
+SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location //base:exaudfclient.template.sh)" """
+FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """
+CREATE_STATIC_BINARY_SCRIPT=select({
+        "//:fast_binary": FAST_BINARY_STATIC,
+        "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY,
+        "//conditions:default": FAST_BINARY_STATIC
     })
 
 genrule(
-    name = "exaudfclient_py3_static",
-    cmd = CREATE_STATIC_BINARY_PY3_SCRIPT,
-    outs = ["exaudfclient_py3_static"],
-    srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
+    name = "exaudfclient_static",
+    cmd = CREATE_STATIC_BINARY_SCRIPT,
+    outs = ["exaudfclient_static"],
+    srcs = [":exaudfclient_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"],
     output_to_bindir = True
 )

exaudfclient/exaudfclient.cc

@@ -26,7 +26,7 @@
 #include "base/streaming_container/streamingcontainer.h"
 #endif
 #include <functional>
-#include "base/debug_message.h"
+#include "base/utils/debug_message.h"
 #include <stdio.h>
 #include <stddef.h>
 #include <stdint.h>

flavors/standard-EXASOL-all/flavor_base/base_test_build_run/Dockerfile

@@ -27,12 +27,12 @@ RUN mkdir /exaudfclient /exaudf
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config java --config r --config test-binaries-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config java --config r --config test-binaries"]
 RUN cp -r -L bazel-bin/* /exaudf && rm -r /exaudf/external
 RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+RUN ./test_udfclient.sh /exaudf/exaudfclient_static
 
 WORKDIR /
 RUN mkdir /exasol_emulator

flavors/standard-EXASOL-all/flavor_base/build_run/Dockerfile

@@ -25,12 +25,12 @@ RUN mkdir /exaudfclient /exaudf
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config java --config optimize-r --config fast-binary-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config java --config optimize-r --config fast-binary"]
 RUN cp -r -L bazel-bin/* /exaudf && rm -r /exaudf/external
 
 
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+RUN ./test_udfclient.sh /exaudf/exaudfclient
 
 WORKDIR /exaudf/
 

flavors/standard-EXASOL-all/flavor_base/language_definition

@@ -1 +1 @@
-PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 JAVA=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=java#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 R=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=r#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
+PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient JAVA=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=java#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient R=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=r#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient

flavors/template-Exasol-all-java-17/flavor_base/base_test_build_run/Dockerfile

@@ -27,12 +27,12 @@ RUN mkdir /exaudfclient /exaudf
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config java --config test-binaries-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config java --config test-binaries"]
 RUN cp -r -L bazel-bin/* /exaudf && rm -r /exaudf/external
 RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs
 
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+RUN ./test_udfclient.sh /exaudf/exaudfclient
 
 WORKDIR /
 RUN mkdir /exasol_emulator

flavors/template-Exasol-all-java-17/flavor_base/build_run/Dockerfile

@@ -25,11 +25,11 @@ RUN mkdir /exaudfclient /exaudf /exaudf/src
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config java --config fast-binary-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config java --config fast-binary"]
 RUN cp -r -L bazel-bin/* /exaudf && rm -r /exaudf/external
 
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+RUN ./test_udfclient.sh /exaudf/exaudfclient
 
 WORKDIR /exaudf/
 

flavors/template-Exasol-all-java-17/flavor_base/language_definition

@@ -1 +1 @@
-JAVA=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=java#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
+JAVA=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=java#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient

flavors/template-Exasol-all-r-4/flavor_base/base_test_build_run/Dockerfile

@@ -27,11 +27,11 @@ RUN mkdir /exaudfclient /exaudf
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config r --config slow-wrapper-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config r --config slow-wrapper"]
 RUN cp -r -L bazel-bin/* /exaudf
 
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+RUN ./test_udfclient.sh /exaudf/exaudfclient
 
 WORKDIR /
 RUN mkdir /exasol_emulator

flavors/template-Exasol-all-r-4/flavor_base/build_run/Dockerfile

@@ -25,11 +25,11 @@ RUN mkdir /exaudfclient /exaudf /exaudf/src
 COPY /exaudfclient /exaudfclient
 
 WORKDIR /exaudfclient/
-RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config optimize-r --config fast-binary-py3"]
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config optimize-r --config fast-binary"]
 RUN cp -r -L bazel-bin/* /exaudf
 
 WORKDIR /exaudfclient/base
-RUN ./test_udfclient.sh /exaudf/exaudfclient_py3
+RUN ./test_udfclient.sh /exaudf/exaudfclient
 
 WORKDIR /exaudf/
 

flavors/template-Exasol-all-r-4/flavor_base/language_definition

@@ -1 +1 @@
-R=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=r#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3
+R=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=r#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient