Feature/columnar data serialization

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp

@@ -174,6 +174,8 @@ template <dataset_type_tag dataset_type_> class Dataset {
     // for columnar buffers, Data* data is empty and attributes is filled
     // for uniform buffers, indptr is empty
     struct Buffer {
+        using Data = Dataset::Data;
+
         Data* data{nullptr};
         std::vector<AttributeBuffer<Data>> attributes{};
         std::span<Indptr> indptr{};
@@ -222,15 +224,24 @@ template <dataset_type_tag dataset_type_> class Dataset {
     Buffer const& get_buffer(std::string_view component) const { return get_buffer(find_component(component, true)); }
     Buffer const& get_buffer(Idx i) const { return buffers_[i]; }
 
+    constexpr bool is_row_based(std::string_view component) const {
+        Idx const idx = find_component(component, false);
+        if (idx == invalid_index) {
+            return false;
+        }
+        return is_row_based(idx);
+    }
+    constexpr bool is_row_based(Idx const i) const { return is_row_based(buffers_[i]); }
+    constexpr bool is_row_based(Buffer const& buffer) const { return buffer.data != nullptr; }
     constexpr bool is_columnar(std::string_view component) const {
         Idx const idx = find_component(component, false);
         if (idx == invalid_index) {
             return false;
         }
         return is_columnar(idx);
     }
-    constexpr bool is_columnar(Idx const i) const { return is_columnar(buffers_[i]); }
-    constexpr bool is_columnar(Buffer const& buffer) const { return buffer.data == nullptr; }
+    constexpr bool is_columnar(Idx const i) const { return !is_row_based(i); }
+    constexpr bool is_columnar(Buffer const& buffer) const { return !is_row_based(buffer); }
 
     Idx find_component(std::string_view component, bool required = false) const {
         auto const found = std::ranges::find_if(dataset_info_.component_info, [component](ComponentInfo const& x) {
@@ -357,6 +368,8 @@ template <dataset_type_tag dataset_type_> class Dataset {
     Dataset get_individual_scenario(Idx scenario)
         requires(!is_indptr_mutable_v<dataset_type>)
     {
+        using AdvanceablePtr = std::conditional_t<is_data_mutable_v<dataset_type>, char*, char const*>;
+
         assert(0 <= scenario && scenario < batch_size());
 
         Dataset result{false, 1, dataset().name, meta_data()};
@@ -366,10 +379,17 @@ template <dataset_type_tag dataset_type_> class Dataset {
             Idx size = component_info.elements_per_scenario >= 0
                            ? component_info.elements_per_scenario
                            : buffer.indptr[scenario + 1] - buffer.indptr[scenario];
-            Data* data = component_info.elements_per_scenario >= 0
-                             ? component_info.component->advance_ptr(buffer.data, size * scenario)
-                             : component_info.component->advance_ptr(buffer.data, buffer.indptr[scenario]);
-            result.add_buffer(component_info.component->name, size, size, nullptr, data);
+            Idx offset = component_info.elements_per_scenario >= 0 ? size * scenario : buffer.indptr[scenario];
+            if (is_columnar(buffer)) {
+                result.add_buffer(component_info.component->name, size, size, nullptr, nullptr);
+                for (auto const& attribute_buffer : buffer.attributes) {
+                    result.add_attribute_buffer(component_info.component->name, attribute_buffer.meta_attribute->name,
+                                                static_cast<Data*>(static_cast<AdvanceablePtr>(attribute_buffer.data)));
+                }
+            } else {
+                Data* data = component_info.component->advance_ptr(buffer.data, offset);
+                result.add_buffer(component_info.component->name, size, size, nullptr, data);
+            }
         }
         return result;
     }

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/meta_data.hpp

@@ -63,15 +63,24 @@ template <class Functor, class... Args> decltype(auto) ctype_func_selector(CType
 }
 
 // set nan
-inline void set_nan(double& x) { x = nan; }
-inline void set_nan(IntS& x) { x = na_IntS; }
-inline void set_nan(ID& x) { x = na_IntID; }
+constexpr void set_nan(double& x) { x = nan; }
+constexpr void set_nan(IntS& x) { x = na_IntS; }
+constexpr void set_nan(ID& x) { x = na_IntID; }
 inline void set_nan(RealValue<asymmetric_t>& x) { x = RealValue<asymmetric_t>{nan}; }
 template <class Enum>
     requires std::same_as<std::underlying_type_t<Enum>, IntS>
-inline void set_nan(Enum& x) {
+constexpr void set_nan(Enum& x) {
     x = static_cast<Enum>(na_IntS);
 }
+template <typename T>
+    requires requires(T t) {
+                 { set_nan(t) };
+             }
+inline T const nan_value = [] {
+    T v{};
+    set_nan(v);
+    return v;
+}();
 
 using RawDataPtr = void*;            // raw mutable data ptr
 using RawDataConstPtr = void const*; // raw read-only data ptr

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: Contributors to the Power Grid Model project <[email protected]>
+//
+// SPDX-License-Identifier: MPL-2.0
+
+#pragma once
+
+#include "../dataset.hpp"
+
+#include <concepts>
+#include <ranges>
+#include <span>
+
+namespace power_grid_model::meta_data::detail {
+
+struct row_based_t {};
+struct columnar_t {};
+constexpr row_based_t row_based{};
+constexpr columnar_t columnar{};
+
+template <typename T>
+concept row_based_or_columnar_c = std::derived_from<T, row_based_t> || std::derived_from<T, columnar_t>;
+
+template <row_based_or_columnar_c T> constexpr bool is_row_based_v = std::derived_from<T, row_based_t>;
+template <row_based_or_columnar_c T> constexpr bool is_columnar_v = std::derived_from<T, columnar_t>;
+
+template <typename BufferType>
+    requires requires(BufferType const& b) {
+                 { b.attributes } -> std::convertible_to<std::vector<AttributeBuffer<typename BufferType::Data>>>;
+             }
+std::vector<AttributeBuffer<typename BufferType::Data>>
+reordered_attribute_buffers(BufferType& buffer, std::span<MetaAttribute const* const> attribute_order) {
+    using Data = typename BufferType::Data;
+    using AttributeBufferType = AttributeBuffer<Data>;
+
+    assert(buffer.data == nullptr);
+
+    std::vector<AttributeBufferType> result(attribute_order.size());
+    std::ranges::transform(
+        attribute_order, result.begin(), [&buffer](auto const* const attribute) -> AttributeBufferType {
+            if (auto it = std::ranges::find_if(buffer.attributes,
+                                               [&attribute](auto const& attribute_buffer) {
+                                                   return attribute_buffer.meta_attribute == attribute;
+                                               });
+                it != buffer.attributes.end()) {
+                return *it;
+            }
+            return AttributeBuffer<void>{};
+        });
+    return result;
+}
+
+} // namespace power_grid_model::meta_data::detail