diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp index cf9ca8cd8..de1dd420e 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp @@ -174,6 +174,8 @@ template class Dataset { // for columnar buffers, Data* data is empty and attributes is filled // for uniform buffers, indptr is empty struct Buffer { + using Data = Dataset::Data; + Data* data{nullptr}; std::vector> attributes{}; std::span indptr{}; @@ -222,6 +224,15 @@ template class Dataset { Buffer const& get_buffer(std::string_view component) const { return get_buffer(find_component(component, true)); } Buffer const& get_buffer(Idx i) const { return buffers_[i]; } + constexpr bool is_row_based(std::string_view component) const { + Idx const idx = find_component(component, false); + if (idx == invalid_index) { + return false; + } + return is_row_based(idx); + } + constexpr bool is_row_based(Idx const i) const { return is_row_based(buffers_[i]); } + constexpr bool is_row_based(Buffer const& buffer) const { return buffer.data != nullptr; } constexpr bool is_columnar(std::string_view component) const { Idx const idx = find_component(component, false); if (idx == invalid_index) { @@ -229,8 +240,8 @@ template class Dataset { } return is_columnar(idx); } - constexpr bool is_columnar(Idx const i) const { return is_columnar(buffers_[i]); } - constexpr bool is_columnar(Buffer const& buffer) const { return buffer.data == nullptr; } + constexpr bool is_columnar(Idx const i) const { return !is_row_based(i); } + constexpr bool is_columnar(Buffer const& buffer) const { return !is_row_based(buffer); } Idx find_component(std::string_view component, bool required = false) const { auto const found = std::ranges::find_if(dataset_info_.component_info, [component](ComponentInfo const& x) { @@ -357,6 +368,8 @@ template class Dataset { Dataset get_individual_scenario(Idx scenario) requires(!is_indptr_mutable_v) { + using AdvanceablePtr = std::conditional_t, char*, char const*>; + assert(0 <= scenario && scenario < batch_size()); Dataset result{false, 1, dataset().name, meta_data()}; @@ -366,10 +379,17 @@ template class Dataset { Idx size = component_info.elements_per_scenario >= 0 ? component_info.elements_per_scenario : buffer.indptr[scenario + 1] - buffer.indptr[scenario]; - Data* data = component_info.elements_per_scenario >= 0 - ? component_info.component->advance_ptr(buffer.data, size * scenario) - : component_info.component->advance_ptr(buffer.data, buffer.indptr[scenario]); - result.add_buffer(component_info.component->name, size, size, nullptr, data); + Idx offset = component_info.elements_per_scenario >= 0 ? size * scenario : buffer.indptr[scenario]; + if (is_columnar(buffer)) { + result.add_buffer(component_info.component->name, size, size, nullptr, nullptr); + for (auto const& attribute_buffer : buffer.attributes) { + result.add_attribute_buffer(component_info.component->name, attribute_buffer.meta_attribute->name, + static_cast(static_cast(attribute_buffer.data))); + } + } else { + Data* data = component_info.component->advance_ptr(buffer.data, offset); + result.add_buffer(component_info.component->name, size, size, nullptr, data); + } } return result; } diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/meta_data.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/meta_data.hpp index 1a2e7496e..1e7f0c62b 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/meta_data.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/meta_data.hpp @@ -63,15 +63,24 @@ template decltype(auto) ctype_func_selector(CType } // set nan -inline void set_nan(double& x) { x = nan; } -inline void set_nan(IntS& x) { x = na_IntS; } -inline void set_nan(ID& x) { x = na_IntID; } +constexpr void set_nan(double& x) { x = nan; } +constexpr void set_nan(IntS& x) { x = na_IntS; } +constexpr void set_nan(ID& x) { x = na_IntID; } inline void set_nan(RealValue& x) { x = RealValue{nan}; } template requires std::same_as, IntS> -inline void set_nan(Enum& x) { +constexpr void set_nan(Enum& x) { x = static_cast(na_IntS); } +template + requires requires(T t) { + { set_nan(t) }; + } +inline T const nan_value = [] { + T v{}; + set_nan(v); + return v; +}(); using RawDataPtr = void*; // raw mutable data ptr using RawDataConstPtr = void const*; // raw read-only data ptr diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp new file mode 100644 index 000000000..2a504b55f --- /dev/null +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Contributors to the Power Grid Model project +// +// SPDX-License-Identifier: MPL-2.0 + +#pragma once + +#include "../dataset.hpp" + +#include +#include +#include + +namespace power_grid_model::meta_data::detail { + +struct row_based_t {}; +struct columnar_t {}; +constexpr row_based_t row_based{}; +constexpr columnar_t columnar{}; + +template +concept row_based_or_columnar_c = std::derived_from || std::derived_from; + +template constexpr bool is_row_based_v = std::derived_from; +template constexpr bool is_columnar_v = std::derived_from; + +template + requires requires(BufferType const& b) { + { b.attributes } -> std::convertible_to>>; + } +std::vector> +reordered_attribute_buffers(BufferType& buffer, std::span attribute_order) { + using Data = typename BufferType::Data; + using AttributeBufferType = AttributeBuffer; + + assert(buffer.data == nullptr); + + std::vector result(attribute_order.size()); + std::ranges::transform( + attribute_order, result.begin(), [&buffer](auto const* const attribute) -> AttributeBufferType { + if (auto it = std::ranges::find_if(buffer.attributes, + [&attribute](auto const& attribute_buffer) { + return attribute_buffer.meta_attribute == attribute; + }); + it != buffer.attributes.end()) { + return *it; + } + return AttributeBuffer{}; + }); + return result; +} + +} // namespace power_grid_model::meta_data::detail diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp index f465536d0..b63870ea9 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp @@ -4,8 +4,11 @@ #pragma once +#include "common.hpp" + #include "../../common/common.hpp" #include "../../common/exception.hpp" +#include "../../common/typing.hpp" #include "../dataset.hpp" #include "../meta_data.hpp" @@ -364,6 +367,18 @@ class Deserializer { using DataByteMeta = std::vector>; using AttributeByteMeta = std::vector>>; + using Buffer = WritableDataset::Buffer; + struct BufferView { + Buffer const* buffer{nullptr}; + Idx idx{0}; + std::span const> reordered_attribute_buffers; + }; + + using row_based_t = detail::row_based_t; + using columnar_t = detail::columnar_t; + static constexpr auto row_based = detail::row_based; + static constexpr auto columnar = detail::columnar; + public: // not copyable Deserializer(Deserializer const&) = delete; @@ -433,6 +448,7 @@ class Deserializer { std::string version_; bool is_batch_{}; std::map, std::less<>> attributes_; + // offset of the msgpack bytes, the number of elements, // for the actual data, per component (outer), per batch (inner) // if a component has no element for a certain scenario, that offset and size will be zero. @@ -685,11 +701,31 @@ class Deserializer { } void parse_component(Idx component_idx) { + if (dataset_handler_.is_row_based(component_idx)) { + parse_component(row_based, component_idx); + } else { + parse_component(columnar, component_idx); + } + } + + template + void parse_component(row_or_column_t row_or_column_tag, Idx component_idx) { auto const& buffer = dataset_handler_.get_buffer(component_idx); + + assert(dataset_handler_.is_row_based(buffer) == detail::is_row_based_v); + assert(dataset_handler_.is_columnar(buffer) == detail::is_columnar_v); + assert(is_row_based(buffer) == detail::is_row_based_v); + assert(is_columnar(buffer) == detail::is_columnar_v); + auto const& info = dataset_handler_.get_component_info(component_idx); auto const& msg_data = msg_data_offsets_[component_idx]; + Idx const batch_size = dataset_handler_.batch_size(); component_key_ = info.component->name; + + // set nan + set_nan(row_or_column_tag, buffer, info); + // handle indptr if (info.elements_per_scenario < 0) { // first always zero @@ -699,37 +735,45 @@ class Deserializer { msg_data.cbegin(), msg_data.cend(), buffer.indptr.begin() + 1, std::plus{}, [](auto const& x) { return x.size; }, Idx{}); } - // set nan - info.component->set_nan(buffer.data, 0, info.total_elements); + // attributes - auto const attributes = [&]() -> std::span { - auto const found = attributes_.find(info.component); - if (found == attributes_.cend()) { - return {}; + std::span const attributes = [this, + &info]() -> std::span { + if (auto const it = attributes_.find(info.component); it != attributes_.cend()) { + return it->second; } - return found->second; + return {}; }(); + auto const reordered_attribute_buffers = detail::is_columnar_v + ? detail::reordered_attribute_buffers(buffer, attributes) + : std::vector>{}; + + BufferView const buffer_view{ + .buffer = &buffer, .idx = 0, .reordered_attribute_buffers = reordered_attribute_buffers}; + // all scenarios for (scenario_number_ = 0; scenario_number_ != batch_size; ++scenario_number_) { - Idx const scenario_offset = info.elements_per_scenario < 0 ? buffer.indptr[scenario_number_] + Idx const scenario_offset = info.elements_per_scenario < 0 ? buffer_view.buffer->indptr[scenario_number_] : scenario_number_ * info.elements_per_scenario; #ifndef NDEBUG if (info.elements_per_scenario < 0) { - assert(buffer.indptr[scenario_number_ + 1] - buffer.indptr[scenario_number_] == + assert(buffer_view.buffer->indptr[scenario_number_ + 1] - + buffer_view.buffer->indptr[scenario_number_] == msg_data[scenario_number_].size); } else { assert(info.elements_per_scenario == msg_data[scenario_number_].size); } #endif - void* scenario_pointer = info.component->advance_ptr(buffer.data, scenario_offset); - parse_scenario(*info.component, scenario_pointer, msg_data[scenario_number_], attributes); + BufferView const scenario = advance(buffer_view, scenario_offset); + parse_scenario(row_or_column_tag, *info.component, scenario, msg_data[scenario_number_], attributes); } scenario_number_ = -1; component_key_ = ""; } - void parse_scenario(MetaComponent const& component, void* scenario_pointer, ComponentByteMeta const& msg_data, + void parse_scenario(detail::row_based_or_columnar_c auto row_or_column_tag, MetaComponent const& component, + BufferView const& buffer_view, ComponentByteMeta const& msg_data, std::span attributes) { // skip for empty scenario if (msg_data.size == 0) { @@ -739,49 +783,95 @@ class Deserializer { offset_ = msg_data.offset; parse_map_array(); for (element_number_ = 0; element_number_ != msg_data.size; ++element_number_) { - void* element_pointer = component.advance_ptr(scenario_pointer, element_number_); + BufferView const element_buffer = advance(buffer_view, element_number_); // check the element is map or array auto const element_visitor = parse_map_array(); if (element_visitor.is_map) { - parse_map_element(element_pointer, element_visitor.size, component); + parse_map_element(row_or_column_tag, element_buffer, element_visitor.size, component); } else { - parse_array_element(element_pointer, element_visitor.size, attributes); + parse_array_element(row_or_column_tag, element_buffer, element_visitor.size, component, attributes); } } element_number_ = -1; offset_ = 0; } - void parse_map_element(void* element_pointer, Idx map_size, MetaComponent const& component) { + void parse_map_element(row_based_t tag, BufferView const& buffer_view, Idx map_size, + MetaComponent const& component) { + while (map_size-- != 0) { + attribute_key_ = parse_string(); + Idx const component_attribute_idx = component.find_attribute(attribute_key_); + if (component_attribute_idx >= 0) { + parse_attribute(tag, buffer_view, component, component.attributes[component_attribute_idx]); + } else { + attribute_key_ = {}; // allow unknown key for additional user info + parse_skip(); + } + } + attribute_key_ = ""; + } + + void parse_map_element(columnar_t /*tag*/, BufferView const& buffer_view, Idx map_size, + MetaComponent const& /*component*/) { while (map_size-- != 0) { attribute_key_ = parse_string(); - Idx const found_idx = component.find_attribute(attribute_key_); - if (found_idx < 0) { - attribute_key_ = {}; - // allow unknown key for additional user info + if (auto it = std::ranges::find_if(buffer_view.buffer->attributes, + [this](auto const& attribute_buffer) { + assert(attribute_buffer.meta_attribute != nullptr); + return attribute_buffer.meta_attribute->name == attribute_key_; + }); + it != buffer_view.buffer->attributes.end()) { + parse_attribute(*it, buffer_view.idx); + } else { + attribute_key_ = {}; // allow unknown key for additional user info parse_skip(); - continue; } - parse_attribute(element_pointer, component.attributes[found_idx]); } attribute_key_ = ""; } - void parse_array_element(void* element_pointer, Idx array_size, std::span attributes) { + template + void parse_array_element(row_or_column_t row_or_column_tag, BufferView const& buffer_view, Idx array_size, + MetaComponent const& component, std::span attributes) { if (array_size != static_cast(attributes.size())) { throw SerializationError{ "An element of a list should have same length as the list of predefined attributes!\n"}; } + for (attribute_number_ = 0; attribute_number_ != array_size; ++attribute_number_) { - parse_attribute(element_pointer, *attributes[attribute_number_]); + if constexpr (detail::is_row_based_v) { + parse_attribute(row_or_column_tag, buffer_view, component, *attributes[attribute_number_]); + } else { + static_assert(detail::is_columnar_v); + if (auto const& attribute_buffer = buffer_view.reordered_attribute_buffers[attribute_number_]; + attribute_buffer.data != nullptr) { + parse_attribute(attribute_buffer, buffer_view.idx); + } else { + parse_skip(); + } + } } attribute_number_ = -1; } - void parse_attribute(void* element_pointer, MetaAttribute const& attribute) { - // call relevant parser - ctype_func_selector(attribute.ctype, [element_pointer, &attribute, this] { - ValueVisitor visitor{{}, attribute.get_attribute(element_pointer)}; + void parse_attribute(row_based_t /*tag*/, BufferView const& buffer_view, MetaComponent const& component, + MetaAttribute const& attribute) { // call relevant parser + assert(is_row_based(buffer_view)); + + ctype_func_selector(attribute.ctype, [&buffer_view, &component, &attribute, this] { + ValueVisitor visitor{ + {}, attribute.get_attribute(component.advance_ptr(buffer_view.buffer->data, buffer_view.idx))}; + msgpack::parse(data_, size_, offset_, visitor); + }); + } + + // parse a single attribute into an attribute buffer (a single column in a columnar buffer) + void parse_attribute(AttributeBuffer const& buffer, Idx idx) { + assert(buffer.data != nullptr); + assert(buffer.meta_attribute != nullptr); + + ctype_func_selector(buffer.meta_attribute->ctype, [&buffer, &idx, this] { + ValueVisitor visitor{{}, *(reinterpret_cast(buffer.data) + idx)}; msgpack::parse(data_, size_, offset_, visitor); }); } @@ -816,6 +906,38 @@ class Deserializer { } } + static void set_nan(row_based_t /*tag*/, Buffer const& buffer, ComponentInfo const& info) { + assert(is_row_based(buffer)); + info.component->set_nan(buffer.data, 0, info.total_elements); + } + static void set_nan(columnar_t /*tag*/, Buffer const& buffer, ComponentInfo const& info) { + assert(is_columnar(buffer)); + for (auto const& attribute_buffer : buffer.attributes) { + if (attribute_buffer.meta_attribute != nullptr) { + ctype_func_selector(attribute_buffer.meta_attribute->ctype, [&attribute_buffer, &info] { + std::ranges::fill(std::span{reinterpret_cast(attribute_buffer.data), + narrow_cast(info.total_elements)}, + nan_value); + }); + } + } + } + + static constexpr BufferView advance(BufferView buffer_view, Idx offset) { + buffer_view.idx += offset; + return buffer_view; + } + static constexpr bool is_row_based(BufferView const& buffer_view) { + assert(buffer_view.buffer != nullptr); + return is_row_based(*buffer_view.buffer); + } + static constexpr bool is_row_based(WritableDataset::Buffer const& buffer) { return buffer.data != nullptr; } + static constexpr bool is_columnar(BufferView const& buffer_view) { + assert(buffer_view.buffer != nullptr); + return is_columnar(*buffer_view.buffer); + } + static constexpr bool is_columnar(WritableDataset::Buffer const& buffer) { return buffer.data == nullptr; } + [[noreturn]] void handle_error(std::exception const& e) { std::stringstream ss; ss << e.what(); diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/serializer.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/serializer.hpp index f7c97544a..81b00977c 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/serializer.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/serializer.hpp @@ -4,6 +4,8 @@ #pragma once +#include "common.hpp" + #include "../../common/common.hpp" #include "../../common/exception.hpp" #include "../dataset.hpp" @@ -199,9 +201,15 @@ struct JsonConverter : msgpack::null_visitor { class Serializer { using RawElementPtr = void const*; + struct BufferView { + ConstDataset::Buffer const* buffer{nullptr}; + Idx idx{0}; + std::span const> reordered_attribute_buffers; + }; + struct ComponentBuffer { MetaComponent const* component; - RawElementPtr data; + BufferView buffer_view; Idx size; }; @@ -209,6 +217,11 @@ class Serializer { std::vector component_buffers; }; + using row_based_t = detail::row_based_t; + using columnar_t = detail::columnar_t; + static constexpr auto row_based = detail::row_based; + static constexpr auto columnar = detail::columnar; + public: static constexpr std::string_view version = "1.0"; // top dict: version, type, is_batch, attributes, data @@ -284,6 +297,7 @@ class Serializer { msgpack::packer packer_; bool use_compact_list_{}; std::map> attributes_; + std::map>> reordered_attribute_buffers_; // json Idx json_indent_{-1}; @@ -306,13 +320,13 @@ class Serializer { ComponentInfo const& info = dataset_handler_.get_component_info(component); ConstDataset::Buffer const& buffer = dataset_handler_.get_buffer(component); component_buffer.component = info.component; + + component_buffer.buffer_view.buffer = &buffer; if (info.elements_per_scenario < 0) { - component_buffer.data = - component_buffer.component->advance_ptr(buffer.data, buffer.indptr[begin_scenario]); + component_buffer.buffer_view.idx = buffer.indptr[begin_scenario]; component_buffer.size = buffer.indptr[end_scenario] - buffer.indptr[begin_scenario]; } else { - component_buffer.data = - component_buffer.component->advance_ptr(buffer.data, info.elements_per_scenario * begin_scenario); + component_buffer.buffer_view.idx = info.elements_per_scenario * begin_scenario; component_buffer.size = info.elements_per_scenario * (end_scenario - begin_scenario); } // only store the view if it is non-empty @@ -325,16 +339,34 @@ class Serializer { void check_attributes() { attributes_ = {}; - for (auto const& buffer : component_buffers_) { + for (auto const& component_buffer : component_buffers_) { std::vector attributes; - for (auto const& attribute : buffer.component->attributes) { + std::vector> reordered_attribute_buffers; + for (auto const& attribute : component_buffer.component->attributes) { // if not all the values of an attribute are nan // add this attribute to the list - if (!attribute.check_all_nan(buffer.data, buffer.size)) { - attributes.push_back(&attribute); + assert(is_row_based(component_buffer) || is_columnar(component_buffer)); + if (is_row_based(component_buffer)) { + if (!attribute.check_all_nan( + component_buffer.component->advance_ptr(component_buffer.buffer_view.buffer->data, + component_buffer.buffer_view.idx), + component_buffer.size)) { + attributes.push_back(&attribute); + } + } else if (auto it = std::ranges::find_if(component_buffer.buffer_view.buffer->attributes, + [&attribute](auto const& attribute_buffer) { + return attribute_buffer.meta_attribute == &attribute; + }); + it != component_buffer.buffer_view.buffer->attributes.end()) { + if (!check_all_nan(*it, 0, component_buffer.size)) { + attributes.push_back(&attribute); + reordered_attribute_buffers.push_back(*it); + } } } - attributes_[buffer.component] = attributes; + + attributes_[component_buffer.component] = std::move(attributes); + reordered_attribute_buffers_[component_buffer.component] = std::move(reordered_attribute_buffers); } } @@ -356,30 +388,6 @@ class Serializer { return json_buffer_; } - static void json_convert_inf(nlohmann::json& json_document) { - switch (json_document.type()) { - case nlohmann::json::value_t::object: - [[fallthrough]]; - case nlohmann::json::value_t::array: - for (auto& value : json_document) { - json_convert_inf(value); - } - break; - case nlohmann::json::value_t::number_float: - json_inf_to_string(json_document); - break; - default: - break; - } - } - - static void json_inf_to_string(nlohmann::json& value) { - double const v = value.get(); - if (std::isinf(v)) { - value = v > 0.0 ? "inf" : "-inf"; - } - } - void serialize(bool use_compact_list) { msgpack_buffer_.clear(); use_compact_list_ = use_compact_list; @@ -431,6 +439,24 @@ class Serializer { } void pack_component(ComponentBuffer const& component_buffer) { + assert(component_buffer.buffer_view.buffer != nullptr); + if (dataset_handler_.is_row_based(*component_buffer.buffer_view.buffer)) { + pack_component(row_based, component_buffer); + } else { + pack_component(columnar, component_buffer); + } + } + + template + void pack_component(row_or_column_t row_or_column_tag, ComponentBuffer const& component_buffer) { + assert(component_buffer.buffer_view.buffer != nullptr); + assert(is_row_based(component_buffer) == detail::is_row_based_v); + assert(is_columnar(component_buffer) == detail::is_columnar_v); + assert(dataset_handler_.is_row_based(*component_buffer.buffer_view.buffer) == + detail::is_row_based_v); + assert(dataset_handler_.is_columnar(*component_buffer.buffer_view.buffer) == + detail::is_columnar_v); + packer_.pack(component_buffer.component); pack_array(component_buffer.size); bool const use_compact_list = use_compact_list_; @@ -442,37 +468,92 @@ class Serializer { assert(found != attributes_.cend()); return found->second; }(); + auto const reordered_attribute_buffers = [&]() -> std::span const> { + if (detail::is_row_based_v || !use_compact_list) { + return {}; + } + auto const found = reordered_attribute_buffers_.find(component_buffer.component); + assert(found != reordered_attribute_buffers_.cend()); + return found->second; + }(); + + BufferView const buffer_view{.buffer = component_buffer.buffer_view.buffer, + .idx = component_buffer.buffer_view.idx, + .reordered_attribute_buffers = reordered_attribute_buffers}; + for (Idx element = 0; element != component_buffer.size; ++element) { - RawElementPtr element_ptr = component_buffer.component->advance_ptr(component_buffer.data, element); + BufferView const element_buffer = advance(buffer_view, element); if (use_compact_list) { - pack_element_in_list(element_ptr, attributes); + pack_element_in_list(row_or_column_tag, element_buffer, *component_buffer.component, attributes); } else { - pack_element_in_dict(element_ptr, component_buffer); + pack_element_in_dict(row_or_column_tag, element_buffer, component_buffer); } } } - void pack_element_in_list(RawElementPtr element_ptr, std::span attributes) { + void pack_element_in_list(row_based_t tag, BufferView const& element_buffer, MetaComponent const& component, + std::span attributes) { + assert(is_row_based(element_buffer)); + pack_array(attributes.size()); for (auto const* const attribute : attributes) { - if (check_nan(element_ptr, *attribute)) { + if (check_nan(tag, element_buffer, component, *attribute)) { + packer_.pack_nil(); + } else { + pack_attribute(tag, element_buffer, component, *attribute); + } + } + } + + void pack_element_in_list(columnar_t /*tag*/, BufferView const& element_buffer, MetaComponent const& /*component*/, + std::span attributes) { + assert(is_columnar(element_buffer)); + assert(element_buffer.reordered_attribute_buffers.size() == attributes.size()); + + (void)attributes; // suppress unused variable in release mode + + pack_array(element_buffer.reordered_attribute_buffers.size()); + for (auto const& attribute_buffer : element_buffer.reordered_attribute_buffers) { + if (check_nan(attribute_buffer, element_buffer.idx)) { packer_.pack_nil(); } else { - pack_attribute(element_ptr, *attribute); + pack_attribute(attribute_buffer, element_buffer.idx); } } } - void pack_element_in_dict(RawElementPtr element_ptr, ComponentBuffer const& component_buffer) { + void pack_element_in_dict(row_based_t tag, BufferView const& element_buffer, + ComponentBuffer const& component_buffer) { + assert(is_row_based(element_buffer)); + uint32_t valid_attributes_count = 0; for (auto const& attribute : component_buffer.component->attributes) { - valid_attributes_count += static_cast(!check_nan(element_ptr, attribute)); + valid_attributes_count += + static_cast(!check_nan(tag, element_buffer, *component_buffer.component, attribute)); } pack_map(valid_attributes_count); for (auto const& attribute : component_buffer.component->attributes) { - if (!check_nan(element_ptr, attribute)) { + if (!check_nan(tag, element_buffer, *component_buffer.component, attribute)) { packer_.pack(attribute.name); - pack_attribute(element_ptr, attribute); + pack_attribute(tag, element_buffer, *component_buffer.component, attribute); + } + } + } + + void pack_element_in_dict(columnar_t /*tag*/, BufferView const& element_buffer, + ComponentBuffer const& /*component_buffer*/) { + assert(is_columnar(element_buffer)); + assert(element_buffer.reordered_attribute_buffers.empty()); + + uint32_t valid_attributes_count = 0; + for (auto const& attribute_buffer : element_buffer.buffer->attributes) { + valid_attributes_count += static_cast(!check_nan(attribute_buffer, element_buffer.idx)); + } + pack_map(valid_attributes_count); + for (auto const& attribute_buffer : element_buffer.buffer->attributes) { + if (!check_nan(attribute_buffer, element_buffer.idx)) { + packer_.pack(attribute_buffer.meta_attribute->name); + pack_attribute(attribute_buffer, element_buffer.idx); } } } @@ -495,17 +576,62 @@ class Serializer { packer_.pack_map(static_cast(count)); } - static bool check_nan(RawElementPtr element_ptr, MetaAttribute const& attribute) { + static bool check_nan(row_based_t /*tag*/, BufferView const& element_buffer, MetaComponent const& component, + MetaAttribute const& attribute) { + assert(is_row_based(element_buffer)); + + RawElementPtr element_ptr = component.advance_ptr(element_buffer.buffer->data, element_buffer.idx); return ctype_func_selector(attribute.ctype, [element_ptr, &attribute] { return is_nan(attribute.get_attribute(element_ptr)); }); } - void pack_attribute(RawElementPtr element_ptr, MetaAttribute const& attribute) { + static bool check_nan(AttributeBuffer const& attribute_buffer, Idx idx) { + return check_all_nan(attribute_buffer, idx, 1); + } + + static bool check_all_nan(AttributeBuffer const& attribute_buffer, Idx idx, Idx size) { + return ctype_func_selector(attribute_buffer.meta_attribute->ctype, [&] { + return std::ranges::all_of( + std::span{reinterpret_cast(attribute_buffer.data) + idx, static_cast(size)}, + [](auto const& x) { return is_nan(x); }); + }); + } + + void pack_attribute(row_based_t /*tag*/, BufferView const& element_buffer, MetaComponent const& component, + MetaAttribute const& attribute) { + RawElementPtr element_ptr = component.advance_ptr(element_buffer.buffer->data, element_buffer.idx); ctype_func_selector(attribute.ctype, [this, element_ptr, &attribute] { packer_.pack(attribute.get_attribute(element_ptr)); }); } + void pack_attribute(AttributeBuffer const& attribute_buffer, Idx idx) { + return ctype_func_selector(attribute_buffer.meta_attribute->ctype, [&] { + packer_.pack(*(reinterpret_cast(attribute_buffer.data) + idx)); + }); + } + + static constexpr BufferView advance(BufferView buffer_view, Idx offset) { + buffer_view.idx += offset; + return buffer_view; + } + + static constexpr bool is_row_based(ComponentBuffer const& component_buffer) { + return is_row_based(component_buffer.buffer_view); + } + static constexpr bool is_row_based(BufferView const& buffer_view) { + assert(buffer_view.buffer != nullptr); + return is_row_based(*buffer_view.buffer); + } + static constexpr bool is_row_based(ConstDataset::Buffer const& buffer) { return buffer.data != nullptr; } + static constexpr bool is_columnar(ComponentBuffer const& component_buffer) { + return is_columnar(component_buffer.buffer_view); + } + static constexpr bool is_columnar(BufferView const& buffer_view) { + assert(buffer_view.buffer != nullptr); + return is_columnar(*buffer_view.buffer); + } + static constexpr bool is_columnar(ConstDataset::Buffer const& buffer) { return buffer.data == nullptr; } }; } // namespace power_grid_model::meta_data diff --git a/tests/cpp_unit_tests/test_dataset.cpp b/tests/cpp_unit_tests/test_dataset.cpp index 45763ce1c..94caafb61 100644 --- a/tests/cpp_unit_tests/test_dataset.cpp +++ b/tests/cpp_unit_tests/test_dataset.cpp @@ -967,41 +967,88 @@ TEST_CASE_TEMPLATE("Test dataset (common)", DatasetType, ConstDataset, MutableDa auto dataset = create_dataset(true, batch_size, dataset_type); - auto a_buffer = std::vector(a_elements_per_scenario * batch_size); - auto b_buffer = std::vector(3); - auto b_indptr = std::vector{0, 0, narrow_cast(b_buffer.size())}; - - add_homogeneous_buffer(dataset, A::name, a_elements_per_scenario, static_cast(a_buffer.data())); - add_inhomogeneous_buffer(dataset, B::name, b_buffer.size(), b_indptr.data(), - static_cast(b_buffer.data())); - - for (auto scenario = 0; scenario < batch_size; ++scenario) { - auto const scenario_dataset = dataset.get_individual_scenario(scenario); - - CHECK(&scenario_dataset.meta_data() == &dataset.meta_data()); - CHECK(!scenario_dataset.empty()); - CHECK(scenario_dataset.is_batch() == false); - CHECK(scenario_dataset.batch_size() == 1); - CHECK(scenario_dataset.n_components() == dataset.n_components()); - - CHECK(scenario_dataset.get_component_info(A::name).component == &dataset_type.get_component(A::name)); - CHECK(scenario_dataset.get_component_info(A::name).elements_per_scenario == a_elements_per_scenario); - CHECK(scenario_dataset.get_component_info(A::name).total_elements == a_elements_per_scenario); - - CHECK(scenario_dataset.get_component_info(B::name).component == &dataset_type.get_component(B::name)); - CHECK(scenario_dataset.get_component_info(B::name).elements_per_scenario == - dataset.template get_buffer_span(scenario).size()); - CHECK(scenario_dataset.get_component_info(B::name).total_elements == - scenario_dataset.get_component_info(B::name).elements_per_scenario); - - auto const scenario_span_a = scenario_dataset.template get_buffer_span(); - auto const scenario_span_b = scenario_dataset.template get_buffer_span(); - auto const dataset_span_a = dataset.template get_buffer_span(scenario); - auto const dataset_span_b = dataset.template get_buffer_span(scenario); - CHECK(scenario_span_a.data() == dataset_span_a.data()); - CHECK(scenario_span_a.size() == dataset_span_a.size()); - CHECK(scenario_span_b.data() == dataset_span_b.data()); - CHECK(scenario_span_b.size() == dataset_span_b.size()); + auto const check_get_individual_scenario = [&] { + for (auto scenario = 0; scenario < batch_size; ++scenario) { + CAPTURE(scenario); + auto const scenario_dataset = dataset.get_individual_scenario(scenario); + + CHECK(&scenario_dataset.meta_data() == &dataset.meta_data()); + CHECK(!scenario_dataset.empty()); + CHECK(scenario_dataset.is_batch() == false); + CHECK(scenario_dataset.batch_size() == 1); + CHECK(scenario_dataset.n_components() == dataset.n_components()); + + CHECK(scenario_dataset.get_component_info(A::name).component == + &dataset_type.get_component(A::name)); + CHECK(scenario_dataset.get_component_info(A::name).elements_per_scenario == + a_elements_per_scenario); + CHECK(scenario_dataset.get_component_info(A::name).total_elements == a_elements_per_scenario); + + CHECK(scenario_dataset.get_component_info(B::name).component == + &dataset_type.get_component(B::name)); + auto const expected_size = + dataset.is_row_based(dataset.get_buffer(B::name)) + ? dataset.template get_buffer_span(scenario).size() + : dataset.template get_columnar_buffer_span(scenario).size(); + CHECK(scenario_dataset.get_component_info(B::name).elements_per_scenario == expected_size); + CHECK(scenario_dataset.get_component_info(B::name).total_elements == + scenario_dataset.get_component_info(B::name).elements_per_scenario); + + if (dataset.is_row_based(dataset.get_buffer(A::name))) { + auto const scenario_span_a = scenario_dataset.template get_buffer_span(); + auto const dataset_span_a = dataset.template get_buffer_span(scenario); + CHECK(scenario_span_a.data() == dataset_span_a.data()); + CHECK(scenario_span_a.size() == dataset_span_a.size()); + } else { + auto const scenario_span_a = + scenario_dataset.template get_columnar_buffer_span(); + auto const dataset_span_a = + dataset.template get_columnar_buffer_span(scenario); + REQUIRE(scenario_span_a.size() == dataset_span_a.size()); + for (Idx idx = 0; idx < scenario_span_a.size(); ++idx) { + auto const scenario_element = scenario_span_a[idx].get(); + auto const& dataset_element = dataset_span_a[idx].get(); + CHECK(scenario_element.id == dataset_element.id); + CHECK(scenario_element.a1 == dataset_element.a1); + } + } + if (dataset.is_row_based(dataset.get_buffer(B::name))) { + auto const scenario_span_b = scenario_dataset.template get_buffer_span(); + auto const dataset_span_b = dataset.template get_buffer_span(scenario); + CHECK(scenario_span_b.data() == dataset_span_b.data()); + CHECK(scenario_span_b.size() == dataset_span_b.size()); + } else { + auto const scenario_span_b = + scenario_dataset.template get_columnar_buffer_span(); + auto const dataset_span_b = + dataset.template get_columnar_buffer_span(scenario); + CHECK(scenario_span_b.begin() == dataset_span_b.begin()); + CHECK(scenario_span_b.size() == dataset_span_b.size()); + } + } + }; + + SUBCASE("row-based") { + auto a_buffer = std::vector(a_elements_per_scenario * batch_size); + auto b_buffer = std::vector(3); + auto b_indptr = std::vector{0, 0, narrow_cast(b_buffer.size())}; + add_homogeneous_buffer(dataset, A::name, a_elements_per_scenario, static_cast(a_buffer.data())); + add_inhomogeneous_buffer(dataset, B::name, b_buffer.size(), b_indptr.data(), + static_cast(b_buffer.data())); + + check_get_individual_scenario(); + } + SUBCASE("columnar") { + auto a_id_buffer = std::vector(a_elements_per_scenario * batch_size); + auto a_a1_buffer = std::vector(a_elements_per_scenario * batch_size); + auto b_indptr = std::vector{0, 0, 3}; + + add_homogeneous_buffer(dataset, A::name, a_elements_per_scenario, nullptr); + add_attribute_buffer(dataset, A::name, "id", static_cast(a_id_buffer.data())); + add_attribute_buffer(dataset, A::name, "a1", static_cast(a_a1_buffer.data())); + add_inhomogeneous_buffer(dataset, B::name, b_indptr.back(), b_indptr.data(), nullptr); + + check_get_individual_scenario(); } } } diff --git a/tests/cpp_unit_tests/test_deserializer.cpp b/tests/cpp_unit_tests/test_deserializer.cpp index 78321d1be..1eb4d7650 100644 --- a/tests/cpp_unit_tests/test_deserializer.cpp +++ b/tests/cpp_unit_tests/test_deserializer.cpp @@ -143,6 +143,11 @@ constexpr std::string_view json_single = R"( "-inf", "+inf" ] + ], + "asym_load": [ + { + "id": 100 + } ] } } @@ -269,7 +274,7 @@ TEST_CASE("Deserializer") { CHECK(deserializer.get_dataset_info().dataset().name == "input"s); CHECK(!deserializer.get_dataset_info().is_batch()); CHECK(deserializer.get_dataset_info().batch_size() == 1); - CHECK(deserializer.get_dataset_info().n_components() == 4); + CHECK(deserializer.get_dataset_info().n_components() == 5); } SUBCASE("Check buffer") { @@ -284,7 +289,7 @@ TEST_CASE("Deserializer") { CHECK(info.get_component_info("sym_load").total_elements == 4); } - SUBCASE("Check parse") { + SUBCASE("Check parse row-based") { std::vector node(3); std::vector line(2); std::vector source(3); @@ -334,6 +339,82 @@ TEST_CASE("Deserializer") { CHECK(sym_load[3].p_specified == -std::numeric_limits::infinity()); CHECK(sym_load[3].q_specified == std::numeric_limits::infinity()); } + + SUBCASE("Check parse columnar") { + std::vector node_id(3); + std::vector node_u_rated(3); + std::vector line_id(2); + std::vector line_r1(2); + std::vector line_r0(2); + std::vector line_x1(2); + std::vector line_x0(2); + std::vector source_id(3); + std::vector source_u_ref(3); + std::vector source_sk(3); + std::vector source_rx_ratio(3); + std::vector sym_load_id(4); + std::vector> sym_load_p_specified(4); + std::vector> sym_load_q_specified(4); + + auto& info = deserializer.get_dataset_info(); + info.set_buffer("node", nullptr, nullptr); + info.add_attribute_buffer("node", "id", node_id.data()); + info.add_attribute_buffer("node", "u_rated", node_u_rated.data()); + info.set_buffer("line", nullptr, nullptr); + info.add_attribute_buffer("line", "id", line_id.data()); + info.add_attribute_buffer("line", "r1", line_r1.data()); + info.add_attribute_buffer("line", "r0", line_r0.data()); + info.add_attribute_buffer("line", "x1", line_x1.data()); + info.add_attribute_buffer("line", "x0", line_x0.data()); + info.set_buffer("source", nullptr, nullptr); + info.add_attribute_buffer("source", "id", source_id.data()); + info.add_attribute_buffer("source", "u_ref", source_u_ref.data()); + info.add_attribute_buffer("source", "sk", source_sk.data()); + info.add_attribute_buffer("source", "rx_ratio", source_rx_ratio.data()); + info.set_buffer("sym_load", nullptr, nullptr); + info.add_attribute_buffer("sym_load", "id", sym_load_id.data()); + info.add_attribute_buffer("sym_load", "p_specified", sym_load_p_specified.data()); + info.add_attribute_buffer("sym_load", "q_specified", sym_load_q_specified.data()); + + deserializer.parse(); + // check node + CHECK(node_id[0] == 1); + CHECK(node_u_rated[0] == doctest::Approx(10.5e3)); + CHECK(node_id[1] == 2); + CHECK(node_u_rated[1] == doctest::Approx(10.5e3)); + CHECK(node_id[2] == 3); + CHECK(node_u_rated[2] == doctest::Approx(10.5e3)); + // check line + CHECK(line_id[0] == 4); + CHECK(line_r1[0] == doctest::Approx(0.11)); + CHECK(is_nan(line_r0[0])); + CHECK(line_id[1] == 5); + CHECK(line_x1[1] == doctest::Approx(0.16)); + CHECK(is_nan(line_x0[1])); + // check source + CHECK(source_id[0] == 15); + CHECK(source_u_ref[0] == doctest::Approx(1.03)); + CHECK(source_sk[0] == doctest::Approx(1e20)); + CHECK(is_nan(source_rx_ratio[0])); + CHECK(source_id[1] == 16); + CHECK(source_u_ref[1] == doctest::Approx(1.04)); + CHECK(is_nan(source_sk[1])); + CHECK(is_nan(source_rx_ratio[1])); + CHECK(source_id[2] == 17); + CHECK(source_u_ref[2] == doctest::Approx(1.03)); + CHECK(source_sk[2] == doctest::Approx(1e10)); + CHECK(source_rx_ratio[2] == doctest::Approx(0.2)); + // check sym_load + CHECK(sym_load_id[0] == 7); + CHECK(sym_load_p_specified[0] == doctest::Approx(1.01e6)); + CHECK(sym_load_id[1] == 8); + CHECK(sym_load_q_specified[1] == doctest::Approx(0.22e6)); + CHECK(sym_load_id[2] == 36); + CHECK(sym_load_p_specified[2] == std::numeric_limits::infinity()); + CHECK(sym_load_id[3] == 37); + CHECK(sym_load_p_specified[3] == -std::numeric_limits::infinity()); + CHECK(sym_load_q_specified[3] == std::numeric_limits::infinity()); + } } SUBCASE("Batch dataset") { @@ -354,7 +435,7 @@ TEST_CASE("Deserializer") { CHECK(info.get_component_info("asym_load").total_elements == 4); } - SUBCASE("Check parse") { + SUBCASE("Check parse row-based") { std::vector sym_load(4); std::vector asym_load(4); IdxVector sym_load_indptr(deserializer.get_dataset_info().batch_size() + 1); @@ -403,6 +484,71 @@ TEST_CASE("Deserializer") { CHECK(asym_load[3].q_specified(1) == doctest::Approx(80.0)); CHECK(asym_load[3].q_specified(2) == std::numeric_limits::infinity()); } + + SUBCASE("Check parse columnar") { + std::vector sym_load_id(4); + std::vector sym_load_status(4); + std::vector> sym_load_p_specified(4); + std::vector> sym_load_q_specified(4); + std::vector asym_load_id(4); + std::vector asym_load_status(4); + std::vector> asym_load_p_specified(4); + std::vector> asym_load_q_specified(4); + IdxVector sym_load_indptr(deserializer.get_dataset_info().batch_size() + 1); + + auto& info = deserializer.get_dataset_info(); + info.set_buffer("sym_load", sym_load_indptr.data(), nullptr); + info.add_attribute_buffer("sym_load", "id", sym_load_id.data()); + info.add_attribute_buffer("sym_load", "status", sym_load_status.data()); + info.add_attribute_buffer("sym_load", "p_specified", sym_load_p_specified.data()); + info.add_attribute_buffer("sym_load", "q_specified", sym_load_q_specified.data()); + info.set_buffer("asym_load", nullptr, nullptr); + info.add_attribute_buffer("asym_load", "id", asym_load_id.data()); + info.add_attribute_buffer("asym_load", "status", asym_load_status.data()); + info.add_attribute_buffer("asym_load", "p_specified", asym_load_p_specified.data()); + info.add_attribute_buffer("asym_load", "q_specified", asym_load_q_specified.data()); + + deserializer.parse(); + + // sym_load + CHECK(sym_load_indptr == IdxVector{0, 1, 1, 3, 4}); + CHECK(sym_load_id[0] == 7); + CHECK(sym_load_p_specified[0] == doctest::Approx(20.0)); + CHECK(sym_load_status[0] == na_IntS); + CHECK(sym_load_id[1] == 7); + CHECK(is_nan(sym_load_p_specified[1])); + CHECK(sym_load_q_specified[1] == doctest::Approx(10.0)); + CHECK(sym_load_status[1] == na_IntS); + CHECK(sym_load_id[2] == 8); + CHECK(is_nan(sym_load_p_specified[2])); + CHECK(is_nan(sym_load_q_specified[2])); + CHECK(sym_load_status[2] == 0); + CHECK(sym_load_id[3] == 37); + CHECK(sym_load_p_specified[3] == -std::numeric_limits::infinity()); + CHECK(sym_load_q_specified[3] == std::numeric_limits::infinity()); + + // asym_load + CHECK(asym_load_id[0] == 9); + CHECK(asym_load_p_specified[0](0) == doctest::Approx(100.0)); + CHECK(is_nan(asym_load_p_specified[0](1))); + CHECK(asym_load_p_specified[0](2) == doctest::Approx(200)); + CHECK(is_nan(asym_load_q_specified[0])); + CHECK(asym_load_id[1] == 9); + CHECK(is_nan(asym_load_p_specified[1])); + CHECK(is_nan(asym_load_q_specified[1])); + CHECK(asym_load_id[2] == 9); + CHECK(is_nan(asym_load_p_specified[2])); + CHECK(asym_load_q_specified[2](0) == doctest::Approx(70.0)); + CHECK(asym_load_q_specified[2](1) == doctest::Approx(80.0)); + CHECK(asym_load_q_specified[2](2) == doctest::Approx(90.0)); + CHECK(asym_load_id[3] == 31); + CHECK(asym_load_p_specified[3](0) == -std::numeric_limits::infinity()); + CHECK(asym_load_p_specified[3](1) == doctest::Approx(75.0)); + CHECK(asym_load_p_specified[3](2) == -std::numeric_limits::infinity()); + CHECK(asym_load_q_specified[3](0) == std::numeric_limits::infinity()); + CHECK(asym_load_q_specified[3](1) == doctest::Approx(80.0)); + CHECK(asym_load_q_specified[3](2) == std::numeric_limits::infinity()); + } } } diff --git a/tests/cpp_unit_tests/test_serializer.cpp b/tests/cpp_unit_tests/test_serializer.cpp index 81c3520b9..f333b018d 100644 --- a/tests/cpp_unit_tests/test_serializer.cpp +++ b/tests/cpp_unit_tests/test_serializer.cpp @@ -168,6 +168,13 @@ TEST_CASE("Serializer") { sym_load_gen[2].p_specified = std::numeric_limits::infinity(); sym_load_gen[3].p_specified = -std::numeric_limits::infinity(); + std::vector sym_load_gen_id(sym_load_gen.size()); + RealValueVector sym_load_gen_p_specified(sym_load_gen.size()); + + std::ranges::transform(sym_load_gen, sym_load_gen_id.begin(), [](auto const& value) { return value.id; }); + std::ranges::transform(sym_load_gen, sym_load_gen_p_specified.begin(), + [](auto const& value) { return value.p_specified; }); + std::vector asym_load_gen(5); meta_data_gen::meta_data.get_dataset("update").get_component("asym_load").set_nan(asym_load_gen.data(), 0, 5); asym_load_gen[0].id = 5; @@ -179,12 +186,20 @@ TEST_CASE("Serializer") { asym_load_gen[1].p_specified = {15.0, nan, 16.0}; asym_load_gen[2].p_specified = {std::numeric_limits::infinity(), 11.0, 17.0}; asym_load_gen[3].p_specified = {10.0, -std::numeric_limits::infinity(), 19.0}; - // nan for asym_load_gen[2].p_specified + // nan for asym_load_gen[4].p_specified + + std::vector asym_load_gen_id(asym_load_gen.size()); + RealValueVector asym_load_gen_p_specified(asym_load_gen.size()); + + std::ranges::transform(asym_load_gen, asym_load_gen_id.begin(), [](auto const& value) { return value.id; }); + std::ranges::transform(asym_load_gen, asym_load_gen_p_specified.begin(), + [](auto const& value) { return value.p_specified; }); - SUBCASE("Single dataset") { + SUBCASE("Single row-based dataset") { ConstDataset handler{false, 1, "update", meta_data_gen::meta_data}; handler.add_buffer("sym_load", 4, 4, nullptr, sym_load_gen.data()); handler.add_buffer("asym_load", 5, 5, nullptr, asym_load_gen.data()); + Serializer serializer{handler, SerializationFormat::json}; CHECK(serializer.get_string(false, -1) == single_dataset_dict); @@ -193,12 +208,51 @@ TEST_CASE("Serializer") { CHECK(serializer.get_string(true, 2) == single_dataset_list_indent); } - SUBCASE("Batch dataset") { + SUBCASE("Single columnar dataset") { + ConstDataset handler{false, 1, "update", meta_data_gen::meta_data}; + handler.add_buffer("sym_load", 4, 4, nullptr, nullptr); + handler.add_attribute_buffer("sym_load", "id", sym_load_gen_id.data()); + handler.add_attribute_buffer("sym_load", "p_specified", sym_load_gen_p_specified.data()); + handler.add_buffer("asym_load", 5, 5, nullptr, nullptr); + handler.add_attribute_buffer("asym_load", "id", asym_load_gen_id.data()); + handler.add_attribute_buffer("asym_load", "p_specified", asym_load_gen_p_specified.data()); + + Serializer serializer{handler, SerializationFormat::json}; + + CHECK(serializer.get_string(false, -1) == single_dataset_dict); + CHECK(serializer.get_string(true, -1) == single_dataset_list); + CHECK(serializer.get_string(false, 2) == single_dataset_dict_indent); + CHECK(serializer.get_string(true, 2) == single_dataset_list_indent); + } + + SUBCASE("Batch row-based dataset") { ConstDataset handler{true, 2, "update", meta_data_gen::meta_data}; std::array const indptr_gen{0, 0, 1}; handler.add_buffer("sym_load", 2, 4, nullptr, sym_load_gen.data()); handler.add_buffer("asym_load", 2, 4, nullptr, asym_load_gen.data()); handler.add_buffer("asym_gen", -1, 1, indptr_gen.data(), asym_load_gen.data() + 4); + + Serializer serializer{handler, SerializationFormat::json}; + + CHECK(serializer.get_string(false, -1) == batch_dataset_dict); + CHECK(serializer.get_string(true, -1) == batch_dataset_list); + CHECK(serializer.get_string(false, 2) == batch_dataset_dict_indent); + CHECK(serializer.get_string(true, 2) == batch_dataset_list_indent); + } + + SUBCASE("Batch columnar dataset") { + ConstDataset handler{true, 2, "update", meta_data_gen::meta_data}; + std::array const indptr_gen{0, 0, 1}; + handler.add_buffer("sym_load", 2, 4, nullptr, nullptr); + handler.add_attribute_buffer("sym_load", "id", sym_load_gen_id.data()); + handler.add_attribute_buffer("sym_load", "p_specified", sym_load_gen_p_specified.data()); + handler.add_buffer("asym_load", 2, 4, nullptr, nullptr); + handler.add_attribute_buffer("asym_load", "id", asym_load_gen_id.data()); + handler.add_attribute_buffer("asym_load", "p_specified", asym_load_gen_p_specified.data()); + handler.add_buffer("asym_gen", -1, 1, indptr_gen.data(), nullptr); + handler.add_attribute_buffer("asym_gen", "id", asym_load_gen_id.data() + 4); + handler.add_attribute_buffer("asym_gen", "p_specified", asym_load_gen_p_specified.data() + 4); + Serializer serializer{handler, SerializationFormat::json}; CHECK(serializer.get_string(false, -1) == batch_dataset_dict);