Merge branch 'daphne-eu:main' into 629_Efficient_Processing_DenseMatr…

…ix_of_Strings

CMakeLists.txt

@@ -112,6 +112,9 @@ if(USE_PAPI)
 endif()
 ##########
 
+find_package(fmt)
+add_definitions(-DSPDLOG_FMT_EXTERNAL)
+
 option(USE_CUDA "Whether to activate compilation of CUDA features" OFF)
 include(CheckLanguage)
 check_language(CUDA)

build.sh

@@ -659,7 +659,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.1 Antlr4 (parser)
+    # Antlr4 (parser)
     #------------------------------------------------------------------------------
     antlrJarName="antlr-${antlrVersion}-complete.jar"
     antlrCppRuntimeDirName="antlr4-cpp-runtime-${antlrVersion}-source"
@@ -711,7 +711,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.2 catch2 (unit test framework)
+    # catch2 (unit test framework)
     #------------------------------------------------------------------------------
     # Download catch2 release zip (if necessary), and unpack the single header file
     # (if necessary).
@@ -736,7 +736,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.3 OpenBLAS (basic linear algebra subprograms)
+    # OpenBLAS (basic linear algebra subprograms)
     #------------------------------------------------------------------------------
     openBlasDirName="OpenBLAS-$openBlasVersion"
     openBlasZipName="${openBlasDirName}.zip"
@@ -762,7 +762,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.4 nlohmann/json (library for JSON parsing)
+    # nlohmann/json (library for JSON parsing)
     #------------------------------------------------------------------------------
     nlohmannjsonDirName=nlohmannjson
     nlohmannjsonSingleHeaderName=json.hpp
@@ -779,7 +779,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.5 abseil (compiled separately to apply a patch)
+    # abseil (compiled separately to apply a patch)
     #------------------------------------------------------------------------------
     abslPath=$sourcePrefix/abseil-cpp
     if [ $(arch) == 'armv64'  ] || [ $(arch) == 'aarch64' ]; then
@@ -807,7 +807,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.6 MPI (Default is MPI library is OpenMPI but cut can be any)
+    # MPI (Default is MPI library is OpenMPI but cut can be any)
     #------------------------------------------------------------------------------
     MPIZipName=openmpi-$openMPIVersion.tar.gz
     MPIInstDirName=$installPrefix
@@ -831,7 +831,7 @@ if [ $WITH_DEPS -gt 0 ]; then
         daphne_msg "No need to build OpenMPI again"
     fi
     #------------------------------------------------------------------------------
-    # #8.7 gRPC
+    # gRPC
     #------------------------------------------------------------------------------
     grpcDirName="grpc"
     grpcInstDir=$installPrefix
@@ -870,7 +870,7 @@ if [ $WITH_DEPS -gt 0 ]; then
         daphne_msg "No need to build GRPC again."
     fi
     #------------------------------------------------------------------------------
-    # #8.8 Arrow / Parquet
+    # Arrow / Parquet
     #------------------------------------------------------------------------------
     arrowDirName="apache-arrow-$arrowVersion"
     arrowArtifactFileName=$arrowDirName.tar.gz
@@ -906,28 +906,48 @@ if [ $WITH_DEPS -gt 0 ]; then
         daphne_msg "No need to build Arrow again."
     fi
     #------------------------------------------------------------------------------
-    # #8.9 spdlog
+    # fmt
+    #------------------------------------------------------------------------------
+    fmtDirName="fmt-$fmtVersion"
+    fmtArtifactFileName=$fmtDirName.zip
+    if ! is_dependency_downloaded "fmt_v${fmtVersion}"; then
+        rm -rf "${sourcePrefix:?}/${fmtDirName}"
+        wget "https://github.com/fmtlib/fmt/releases/download/${fmtVersion}/$fmtArtifactFileName" -qO  "$cacheDir/$fmtArtifactFileName"
+        unzip -q "$cacheDir/$fmtArtifactFileName" -d "$sourcePrefix"
+        dependency_download_success "fmt_v${fmtVersion}"
+    fi
+    if ! is_dependency_installed "fmt_v${fmtVersion}"; then
+        cmake -G Ninja -S "${sourcePrefix}/${fmtDirName}" -B "${buildPrefix}/${fmtDirName}" \
+            -DCMAKE_INSTALL_PREFIX="${installPrefix}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DFMT_MASTER_PROJECT=OFF
+        cmake --build "${buildPrefix}/${fmtDirName}" --target install/strip
+        dependency_install_success "fmt_v${fmtVersion}"
+    else
+        daphne_msg "No need to build fmt again."
+    fi
+    #------------------------------------------------------------------------------
+    # spdlog
     #------------------------------------------------------------------------------
     spdlogDirName="spdlog-$spdlogVersion"
     spdlogArtifactFileName=$spdlogDirName.tar.gz
     if ! is_dependency_downloaded "spdlog_v${spdlogVersion}"; then
         rm -rf "${sourcePrefix:?}/${spdlogDirName}"
-        wget "https://github.com/gabime/spdlog/archive/refs/tags/v$spdlogVersion.tar.gz" -qO \
+        # changed URL scheme due to  temporarily use tip of main branch (2024-10-03)
+#        wget "https://github.com/gabime/spdlog/archive/refs/tags/v$spdlogVersion.tar.gz" -qO \
+        wget https://github.com/gabime/spdlog/archive/$spdlogVersion.tar.gz -qO \
             "$cacheDir/$spdlogArtifactFileName"
         tar xzf "$cacheDir/$spdlogArtifactFileName" --directory="$sourcePrefix"
         dependency_download_success "spdlog_v${spdlogVersion}"
     fi
-
     if ! is_dependency_installed "spdlog_v${spdlogVersion}"; then
         cmake -G Ninja -S "${sourcePrefix}/${spdlogDirName}" -B "${buildPrefix}/${spdlogDirName}" \
-            -DCMAKE_INSTALL_PREFIX="${installPrefix}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+            -DSPDLOG_FMT_EXTERNAL=ON -DCMAKE_INSTALL_PREFIX="${installPrefix}" -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         cmake --build "${buildPrefix}/${spdlogDirName}" --target install/strip
         dependency_install_success "spdlog_v${spdlogVersion}"
     else
         daphne_msg "No need to build spdlog again."
     fi
     #------------------------------------------------------------------------------
-    # #8.10 Eigen
+    # Eigen
     #------------------------------------------------------------------------------
     eigenDirName="eigen-${eigenVersion}"
     if ! is_dependency_downloaded "eigen_v${eigenVersion}"; then
@@ -946,17 +966,17 @@ if [ $WITH_DEPS -gt 0 ]; then
       daphne_msg "No need to build eigen again."
     fi
     #------------------------------------------------------------------------------
-    # #8.11 HAWQ (libhdfs3)
+    # HAWQ (libhdfs3)
     #------------------------------------------------------------------------------
     hawqDirName="hawq-rel-v$hawqVersion"
-    hawqTarName="v${hawqVersion}.tar.gz"
+    hawqDlTarName="v${hawqVersion}.tar.gz"
+    hawqTarName="${hawqDirName}.tar.gz"
     hawqInstDirName=$installPrefix
 
     if [ $BUILD_HDFS == "-DUSE_HDFS=ON" ]; then
         if ! is_dependency_downloaded "hawq_v${hawqVersion}"; then
-              daphne_msg "Get HAWQ (libhdfs3) version ${hawqVersion}"
-              exit -1
-            wget "https://github.com/apache/hawq/archive/refs/tags/rel/${hawqTarName}" \
+            daphne_msg "Get HAWQ (libhdfs3) version ${hawqVersion}"
+            wget "https://github.com/apache/hawq/archive/refs/tags/rel/${hawqDlTarName}" \
                 -qO "${cacheDir}/${hawqTarName}"
             tar -xf "$cacheDir/$hawqTarName" -C "$sourcePrefix"
             daphne_msg "Applying 0005-libhdfs3-remove-gtest-dep.patch"
@@ -976,7 +996,7 @@ if [ $WITH_DEPS -gt 0 ]; then
     fi
 
     #------------------------------------------------------------------------------
-    # #8.12 Build MLIR
+    # Build MLIR
     #------------------------------------------------------------------------------
     # We rarely need to build MLIR/LLVM, only during the first build of the
     # prototype and after upgrades of the LLVM sub-module. To avoid unnecessary
@@ -1034,7 +1054,7 @@ if [ $WITH_DEPS -gt 0 ]; then
         daphne_msg "No need to build MLIR/LLVM again."
     fi
     #------------------------------------------------------------------------------
-    # 8.14 Liburing
+    # Liburing
     #------------------------------------------------------------------------------
     liburingDirName="liburing-$liburingVersion"
     liburingTarName="${liburingDirName}.tar.gz"
@@ -1063,9 +1083,8 @@ if [ $WITH_DEPS -gt 0 ]; then
             daphne_msg "No need to build liburing again."
         fi
     fi
-
     #------------------------------------------------------------------------------
-    # 8.14 Fetch bitstreams
+    # Fetch bitstreams
     #------------------------------------------------------------------------------
     if [[ $BUILD_FPGAOPENCL = *"ON"* ]]; then
         FPGAOPENCL_BISTREAM_DIR="$projectRoot/src/runtime/local/kernels/FPGAOPENCL/bitstreams"

software-package-versions.txt

@@ -28,7 +28,10 @@ nlohmannjsonVersion=3.10.5
 openBlasVersion=0.3.23
 openMPIVersion=4.1.5
 papiVersion=7.0.1
-spdlogVersion=1.11.0
+# temporarily (2024-10-03) use tip of default branch for spdlog due to compilation issue of latest
+# release 1.14.1 in combination with external fmt 11.0.2
+spdlogVersion=e593f6695c6065e6b345fe2862f04a519ed484e0
 ubuntuVersion=24.04
 hawqVersion=3.0.0.0
-liburingVersion=2.7
+liburingVersion=2.7
+fmtVersion=11.0.2

src/api/internal/daphne_internal.cpp

@@ -15,10 +15,11 @@
  */
 
 #include "runtime/local/datastructures/IAllocationDescriptor.h"
-#include <vector>
+
 #ifdef USE_MPI
 #include "runtime/distributed/worker/MPIWorker.h"
 #endif
+
 #include "compiler/execution/DaphneIrExecutor.h"
 #include <api/cli/DaphneUserConfig.h>
 #include <api/cli/StatusCode.h>
@@ -49,7 +50,6 @@
 
 #include <csetjmp>
 #include <csignal>
-#include <cstdlib>
 #include <cstring>
 #include <execinfo.h>
 
@@ -97,8 +97,7 @@ void handleSignals(int signal) {
 }
 
 void logErrorDaphneLibAware(DaphneLibResult *daphneLibRes, std::string msg) {
-    if (daphneLibRes != nullptr) // For DaphneLib (Python API), error message is
-                                 // handled later in script.py.
+    if (daphneLibRes != nullptr) // For DaphneLib (Python API), error message is handled later in script.py.
         daphneLibRes->error_message = msg;
     else
         spdlog::error(msg);
@@ -108,8 +107,7 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
     using clock = std::chrono::high_resolution_clock;
     clock::time_point tpBeg = clock::now();
 
-    // install signal handler to catch information from shared libraries (for
-    // exception handling)
+    // install signal handler to catch information from shared libraries (for exception handling)
     std::signal(SIGABRT, handleSignals);
     std::signal(SIGSEGV, handleSignals);
 
@@ -383,8 +381,7 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
     if (matmul_fixed_tile_sizes.size() > 0) {
         user_config.matmul_use_fixed_tile_sizes = true;
         user_config.matmul_fixed_tile_sizes = matmul_fixed_tile_sizes;
-        // Specifying a fixed tile size will be interpreted as wanting to use
-        // tiling.
+        // Specifying a fixed tile size will be interpreted as wanting to use tiling.
         user_config.matmul_tile = true;
     }
     user_config.use_mlir_hybrid_codegen = performHybridCodegen;
@@ -399,9 +396,8 @@ int startDAPHNE(int argc, const char **argv, DaphneLibResult *daphneLibRes, int
 
     // only overwrite with non-defaults
     if (numberOfThreads != 0) {
-        spdlog::trace("Overwriting config file supplied numberOfThreads={} "
-                      "with command line argument --num-threads={}",
-                      user_config.numberOfThreads, numberOfThreads);
+        spdlog::trace("Overwriting config file supplied numberOfThreads={} with command line argument --num-threads={}",
+                      user_config.numberOfThreads, static_cast<int>(numberOfThreads));
         user_config.numberOfThreads = numberOfThreads;
     }
 

src/parser/config/ConfigParser.cpp

@@ -143,7 +143,7 @@ void ConfigParser::readUserConfig(const std::string &filename, DaphneUserConfig
                     // not using spdlog::get() here as loggers are most likely
                     // not configured yet
                     spdlog::error(key2);
-                    spdlog::error(val2);
+                    spdlog::error(static_cast<std::string>(key2));
                 }
             }
         }

src/runtime/local/context/CUDAContext.h

@@ -19,6 +19,9 @@
 #include "runtime/local/context/DaphneContext.h"
 #include "runtime/local/kernels/CUDA/HostUtils.h"
 
+#include <fmt/core.h>
+#include <fmt/ranges.h>
+
 #include <iostream>
 #include <map>
 #include <memory>

src/runtime/local/datastructures/DenseMatrix.cpp

@@ -18,6 +18,7 @@
 #include <runtime/local/datastructures/AllocationDescriptorHost.h>
 #include <runtime/local/io/DaphneSerializer.h>
 
+#include <fmt/core.h>
 #include <spdlog/spdlog.h>
 
 #include <sstream>

src/runtime/local/kernels/CMakeLists.txt

@@ -73,7 +73,7 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER)
             PATH_SUFFIXES nvidia/current lib64 lib/x64 lib)
 
     target_link_libraries(CUDAKernels PUBLIC DataStructures LLVMSupport MLIRDaphne MLIRDaphneTransforms CUDA::cudart CUDA::cublasLt CUDA::cublas
-            CUDA::cusparse ${CUDA_cudnn_LIBRARY} CUDA::cusolver Util MLIRDaphneInference)
+            CUDA::cusparse ${CUDA_cudnn_LIBRARY} CUDA::cusolver Util MLIRDaphneInference fmt::fmt)
     set_target_properties(CUDAKernels PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/lib)
 endif()
 

src/runtime/local/kernels/CUDA/HostUtils.h

@@ -23,9 +23,17 @@
 #include <cusolverDn.h>
 #include <cusparse_v2.h>
 
+#include <fmt/format.h>
+
 #include <iostream>
 #include <memory>
 
+static auto format_as(cudaError_t e) { return fmt::underlying(e); }
+static auto format_as(cublasStatus_t e) { return fmt::underlying(e); }
+static auto format_as(cudnnStatus_t e) { return fmt::underlying(e); }
+static auto format_as(cusparseStatus_t e) { return fmt::underlying(e); }
+static auto format_as(cusolverStatus_t e) { return fmt::underlying(e); }
+
 #define CHECK_CUDART(call)                                                                                             \
     do {                                                                                                               \
         cudaError_t status = call;                                                                                     \

src/util/CMakeLists.txt

@@ -32,7 +32,7 @@ add_library(Util
 		StringRefCount.cpp
         )
 
-target_link_libraries(Util PRIVATE spdlog::spdlog)
+target_link_libraries(Util PRIVATE spdlog::spdlog fmt::fmt)
 
 # Make sure that certain .inc files have been generated by TableGen.
 add_dependencies(Util

src/util/DaphneLogger.cpp

@@ -16,6 +16,8 @@
 
 #include "DaphneLogger.h"
 
+#include <fmt/core.h>
+
 #include "spdlog/async.h"
 #include "spdlog/sinks/basic_file_sink.h"
 #include "spdlog/sinks/stdout_color_sinks.h"

src/util/DaphneLogger.h

@@ -16,6 +16,7 @@
 
 #pragma once
 
+#include <fmt/core.h>
 #include <spdlog/common.h>
 
 #include <api/cli/DaphneUserConfig.h>

src/util/Statistics.cpp

@@ -1,8 +1,24 @@
+/*
+ * Copyright 2024 The DAPHNE Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #include "Statistics.h"
 
+#include <fmt/core.h>
 #include <spdlog/spdlog.h>
 
-#include <iostream>
 #include <map>
 
 Statistics &Statistics::instance() {

src/util/Statistics.h

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2024 The DAPHNE Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #pragma once
 
 #include <util/KernelDispatchMapping.h>