State serialization improvements (#570)

* Re-working plugin state serialization

* Remove dead comments

* Better backrds compatibility

* Apply clang-format

* Serialization work

* Apply clang-format

* More byte serialization work

* Serialization for non-parameter state

* Binary serialization mostly working

* Apply clang-format

* Mossing arena frame

* Tweaking plugin state arena usage

* Small fixes

* Apply clang-format

* Using ChunkList for parameter deserialization

* Updating non-param state

* Fixes

* More fixes

* Adding a couple of tests

* Apply clang-format

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

examples/StatefulPlugin/PluginEditor.cpp

@@ -28,7 +28,7 @@ PluginEditor::PluginEditor (StatefulPlugin& plug) : juce::AudioProcessorEditor (
     const auto setSizeFromState = [this]
     {
         const auto& stateSize = plugin.getState().nonParams.editorBounds.get();
-        setSize (stateSize.x, stateSize.y);
+        setSize (stateSize.first, stateSize.second);
     };
     setSizeFromState();
 
@@ -65,5 +65,5 @@ void PluginEditor::resized()
     undoButton.setBounds (bounds.removeFromLeft (80));
     redoButton.setBounds (bounds.removeFromLeft (80));
 
-    plugin.getState().nonParams.editorBounds = getLocalBounds().getBottomRight();
+    plugin.getState().nonParams.editorBounds = std::make_pair (getLocalBounds().getRight(), getLocalBounds().getBottom());
 }

examples/StatefulPlugin/StatefulPlugin.h

@@ -34,7 +34,7 @@ struct PluginNonParameterState : chowdsp::NonParamState
         addStateValues ({ &editorBounds });
     }
 
-    chowdsp::StateValue<juce::Point<int>> editorBounds { "editor_bounds", { 300, 500 } };
+    chowdsp::StateValue<std::pair<int, int>> editorBounds { "editor_bounds", { 300, 500 } };
 };
 
 using State = chowdsp::PluginStateImpl<PluginParameterState, PluginNonParameterState>;

modules/common/chowdsp_data_structures/Allocators/chowdsp_ArenaAllocator.h

@@ -103,6 +103,13 @@ class ArenaAllocator
         return reinterpret_cast<T*> (raw_data.data() + offset_bytes);
     }
 
+    /** Returns a pointer to the internal buffer with a given offset in bytes */
+    template <typename T, typename IntType>
+    const T* data (IntType offset_bytes) const noexcept
+    {
+        return reinterpret_cast<const T*> (raw_data.data() + offset_bytes);
+    }
+
     /**
      * Creates a "frame" for the allocator.
      * Once the frame goes out of scope, the allocator will be reset

modules/common/chowdsp_data_structures/Allocators/chowdsp_ChainedArenaAllocator.h

@@ -114,6 +114,12 @@ class ChainedArenaAllocator
         return get_current_arena().template data<T> (offset_bytes);
     }
 
+    /** Returns the default size for an individual arena */
+    [[nodiscard]] size_t get_default_arena_size() const noexcept
+    {
+        return arena_size_bytes;
+    }
+
     /** Returns the arena currently being used */
     ArenaAllocatorView& get_current_arena()
     {
@@ -133,6 +139,11 @@ class ChainedArenaAllocator
         return arena_list.count;
     }
 
+    [[nodiscard]] auto* get_extra_alloc_list() const noexcept
+    {
+        return extra_alloc_list;
+    }
+
     /**
      * Returns the total number of bytes currently being used
      * by this allocator.

modules/common/chowdsp_json/JSONUtils/chowdsp_JSONAdaptors.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#if JUCE_MODULE_AVAILABLE_chowdsp_version
+#include <chowdsp_version/chowdsp_version.h>
+#endif
+
+#ifndef DOXYGEN
+namespace nlohmann
+{
+/** Adapter so that nlohmann::json can serialize juce::String */
+template <>
+struct adl_serializer<juce::String>
+{
+    static void to_json (json& j, const juce::String& s)
+    {
+        j = s.toUTF8();
+    }
+
+    static void from_json (const json& j, juce::String& s)
+    {
+        s = j.get<std::string>();
+    }
+};
+
+#if JUCE_MODULE_AVAILABLE_chowdsp_version
+/** Adapter so that nlohmann::json can serialize chowdsp::Version */
+template <>
+struct adl_serializer<::chowdsp::Version>
+{
+    static void to_json (json& j, const ::chowdsp::Version& version)
+    {
+        j = version.getVersionHint();
+    }
+
+    static void from_json (const json& j, ::chowdsp::Version& version)
+    {
+        if (! j.is_number_integer())
+        {
+            version = {};
+            return;
+        }
+
+        const auto versionHint = j.get<int>();
+        const auto major = versionHint / 10000;
+        const auto minor = (versionHint % 10000) / 100;
+        const auto patch = versionHint % 100;
+        version = { major, minor, patch };
+    }
+};
+#endif
+} // namespace nlohmann
+#endif

modules/common/chowdsp_json/chowdsp_json.h

@@ -32,6 +32,5 @@ namespace chowdsp
 using json = nlohmann::json;
 } // namespace chowdsp
 
-#include "JSONUtils/chowdsp_StringAdapter.h"
-
+#include "JSONUtils/chowdsp_JSONAdaptors.h"
 #include "JSONUtils/chowdsp_JSONUtils.h"

modules/common/chowdsp_serialization/Serialization/chowdsp_ByteSerializer.cpp

@@ -0,0 +1,78 @@
+namespace chowdsp
+{
+nonstd::span<const std::byte> get_bytes_for_deserialization (nonstd::span<const std::byte>& bytes)
+{
+    using namespace bytes_detail;
+
+    size_type bytes_count;
+    std::memcpy (&bytes_count, bytes.data(), sizeof_s);
+    jassert (sizeof_s + bytes_count <= bytes.size());
+
+    const auto serial_bytes = bytes.subspan (sizeof_s, bytes_count);
+    bytes = bytes.subspan (sizeof_s + bytes_count);
+
+    return serial_bytes;
+}
+
+size_t get_serial_num_bytes (ChainedArenaAllocator& arena,
+                             const ChainedArenaAllocator::Frame* frame)
+{
+    size_t num_bytes = 0;
+    auto* start_arena = frame == nullptr ? arena.get_arenas().head : frame->arena_at_start;
+    const auto add_bytes_count = [&num_bytes, start_arena, frame] (const ArenaAllocatorView& arena_node)
+    {
+        size_t bytes_offset = 0;
+        if (start_arena == &arena_node && frame != nullptr)
+            bytes_offset = frame->arena_frame.bytes_used_at_start;
+        num_bytes += arena_node.get_bytes_used() - bytes_offset;
+    };
+
+    for (auto* arena_node = start_arena; arena_node != &arena.get_current_arena(); arena_node = arena_node->next)
+        add_bytes_count (*arena_node);
+    add_bytes_count (arena.get_current_arena());
+
+    return num_bytes;
+}
+
+void dump_serialized_bytes (nonstd::span<std::byte> serial,
+                            ChainedArenaAllocator& arena,
+                            const ChainedArenaAllocator::Frame* frame)
+{
+    const auto num_bytes = serial.size();
+    jassert (num_bytes == get_serial_num_bytes (arena, frame));
+
+    auto* start_arena = frame == nullptr ? arena.get_arenas().head : frame->arena_at_start;
+    size_t bytes_counter = 0;
+    const auto copy_bytes = [num_bytes, start_arena, frame, &serial, &bytes_counter] (const ArenaAllocatorView& arena_node)
+    {
+        size_t bytes_offset = 0;
+        if (start_arena == &arena_node && frame != nullptr)
+            bytes_offset = frame->arena_frame.bytes_used_at_start;
+
+        const auto bytes_to_copy = std::min (arena_node.get_bytes_used() - bytes_offset,
+                                             num_bytes - bytes_counter);
+        std::memcpy (serial.data() + bytes_counter, arena_node.data<std::byte> (bytes_offset), bytes_to_copy);
+
+        bytes_counter += bytes_to_copy;
+    };
+    for (auto* arena_node = start_arena; arena_node != &arena.get_current_arena(); arena_node = arena_node->next)
+        copy_bytes (*arena_node);
+    copy_bytes (arena.get_current_arena());
+}
+
+void dump_serialized_bytes (juce::MemoryBlock& data,
+                            ChainedArenaAllocator& arena,
+                            const ChainedArenaAllocator::Frame* frame)
+{
+    const auto initial_size = data.getSize();
+    const auto num_bytes = get_serial_num_bytes (arena, frame);
+    data.setSize (initial_size + num_bytes);
+    dump_serialized_bytes ({ static_cast<std::byte*> (data.getData()) + initial_size, num_bytes }, arena, frame);
+}
+
+std::string_view deserialize_string (nonstd::span<const std::byte>& bytes)
+{
+    const auto serial_bytes = get_bytes_for_deserialization (bytes);
+    return { reinterpret_cast<const char*> (serial_bytes.data()), serial_bytes.size() };
+}
+} // namespace chowdsp

modules/common/chowdsp_serialization/Serialization/chowdsp_ByteSerializer.h

@@ -0,0 +1,132 @@
+#pragma once
+
+namespace chowdsp
+{
+#ifndef DOXYGEN
+namespace bytes_detail
+{
+    using size_type = uint16_t;
+    static constexpr auto sizeof_s = sizeof (size_type);
+} // namespace bytes_detail
+#endif
+
+template <typename ArenaType>
+static std::byte* get_bytes_for_serialization (size_t bytes_count, ArenaType& arena)
+{
+    using namespace bytes_detail;
+    jassert (bytes_count <= std::numeric_limits<size_type>::max());
+
+    if constexpr (std::is_same_v<ArenaType, ChainedArenaAllocator>)
+        jassert (sizeof_s + bytes_count <= arena.get_default_arena_size());
+
+    auto* bytes = static_cast<std::byte*> (arena.allocate_bytes (sizeof_s + bytes_count, 1));
+    jassert (bytes != nullptr);
+
+    const auto bytes_count_cast = static_cast<size_type> (bytes_count);
+    std::memcpy (bytes, &bytes_count_cast, sizeof_s);
+
+    return bytes + sizeof_s; // NOLINT
+}
+
+nonstd::span<const std::byte> get_bytes_for_deserialization (nonstd::span<const std::byte>& bytes);
+
+template <typename TDest, typename TSource>
+void serialize_direct (TDest* ptr, const TSource& source)
+{
+    const auto source_cast = static_cast<TDest> (source);
+    std::memcpy (ptr, &source_cast, sizeof (TDest));
+}
+
+template <typename T, typename ArenaType>
+static size_t serialize_object (const T& object, ArenaType& arena)
+{
+    auto* bytes = get_bytes_for_serialization (sizeof (T), arena);
+    std::memcpy (bytes, &object, sizeof (T)); // NOLINT
+    return bytes_detail::sizeof_s + sizeof (T);
+}
+
+template <typename T, typename ArenaType>
+static size_t serialize_span (nonstd::span<const T> data, ArenaType& arena)
+{
+    const auto num_bytes = sizeof (T) * data.size();
+    auto* bytes = get_bytes_for_serialization (num_bytes, arena);
+    std::memcpy (bytes, data.data(), num_bytes); // NOLINT
+    return bytes_detail::sizeof_s + num_bytes;
+}
+
+template <typename ArenaType>
+static size_t serialize_string (std::string_view str, ArenaType& arena)
+{
+    const auto num_bytes = sizeof (char) * str.size();
+    auto* bytes = get_bytes_for_serialization (num_bytes, arena);
+    std::memcpy (bytes, str.data(), num_bytes); // NOLINT
+    return bytes_detail::sizeof_s + num_bytes;
+}
+
+template <typename MemoryResourceType>
+static nonstd::span<const std::byte> dump_serialized_bytes (const ArenaAllocator<MemoryResourceType>& arena,
+                                                            const typename ArenaAllocator<MemoryResourceType>::Frame* frame = nullptr)
+{
+    const auto bytes_offset = frame == nullptr ? 0 : frame->bytes_used_at_start;
+    const auto bytes_count = arena.get_bytes_used() - bytes_offset;
+    return { arena.template data<std::byte> (bytes_offset), bytes_count };
+}
+
+size_t get_serial_num_bytes (ChainedArenaAllocator& arena,
+                             const ChainedArenaAllocator::Frame* frame = nullptr);
+
+void dump_serialized_bytes (nonstd::span<std::byte> serial,
+                            ChainedArenaAllocator& arena,
+                            const ChainedArenaAllocator::Frame* frame = nullptr);
+
+void dump_serialized_bytes (juce::MemoryBlock& data,
+                            ChainedArenaAllocator& arena,
+                            const ChainedArenaAllocator::Frame* frame = nullptr);
+
+template <typename T>
+T deserialize_direct (nonstd::span<const std::byte>& bytes)
+{
+    T x;
+    std::memcpy (&x, bytes.data(), sizeof (T));
+    bytes = bytes.subspan (sizeof (T));
+    return x;
+}
+
+template <typename T>
+static T deserialize_object (nonstd::span<const std::byte>& bytes)
+{
+    const auto serial_bytes = get_bytes_for_deserialization (bytes);
+    jassert (serial_bytes.size() == sizeof (T));
+
+    JUCE_BEGIN_IGNORE_WARNINGS_GCC_LIKE ("-Wclass-memaccess")
+    T object;
+    std::memcpy (&object, serial_bytes.data(), serial_bytes.size());
+    return object;
+    JUCE_END_IGNORE_WARNINGS_GCC_LIKE
+}
+
+template <typename T>
+static size_t get_span_length (const nonstd::span<const std::byte>& bytes)
+{
+    using namespace bytes_detail;
+
+    size_type sizeof_span_bytes;
+    std::memcpy (&sizeof_span_bytes, bytes.data(), sizeof_s);
+
+    jassert (sizeof_span_bytes % sizeof (T) == 0);
+    return static_cast<size_t> (sizeof_span_bytes) / sizeof (T);
+}
+
+template <typename T>
+static void deserialize_span (nonstd::span<T> span, nonstd::span<const std::byte>& bytes)
+{
+    const auto serial_bytes = get_bytes_for_deserialization (bytes);
+
+    jassert (serial_bytes.size() % sizeof (T) == 0);
+    jassert (serial_bytes.size() / sizeof (T) == span.size());
+
+    std::memcpy (span.data(), serial_bytes.data(), serial_bytes.size());
+}
+
+std::string_view deserialize_string (nonstd::span<const std::byte>& bytes);
+} // namespace chowdsp

modules/common/chowdsp_serialization/chowdsp_serialization.cpp

@@ -0,0 +1,3 @@
+#include "chowdsp_serialization.h"
+
+#include "Serialization/chowdsp_ByteSerializer.cpp"

modules/common/chowdsp_serialization/chowdsp_serialization.h

@@ -8,7 +8,7 @@ BEGIN_JUCE_MODULE_DECLARATION
    version:       2.3.0
    name:          ChowDSP Serialization Utilities
    description:   Utility methods for serializing data structures into XML, JSON, or some other format
-   dependencies:  juce_core, chowdsp_core, chowdsp_json, chowdsp_reflection
+   dependencies:  juce_core, chowdsp_core, chowdsp_json, chowdsp_reflection, chowdsp_data_structures
 
    website:       https://ccrma.stanford.edu/~jatin/chowdsp
    license:       BSD 3-Clause
@@ -25,8 +25,10 @@ BEGIN_JUCE_MODULE_DECLARATION
 #include <chowdsp_core/chowdsp_core.h>
 #include <chowdsp_json/chowdsp_json.h>
 #include <chowdsp_reflection/chowdsp_reflection.h>
+#include <chowdsp_data_structures/chowdsp_data_structures.h>
 
 #include "Serialization/chowdsp_BaseSerializer.h"
 #include "Serialization/chowdsp_Serialization.h"
 #include "Serialization/chowdsp_JSONSerializer.h"
 #include "Serialization/chowdsp_XMLSerializer.h"
+#include "Serialization/chowdsp_ByteSerializer.h"

modules/plugin/chowdsp_parameters/ParamUtils/chowdsp_ParameterTypes.cpp

@@ -33,7 +33,7 @@ FloatParameter::FloatParameter (const ParameterID& parameterID,
                 { return valueToTextFunction (v); })
             .withValueFromStringFunction (std::move (textToValueFunction))),
 #endif
-      unsnappedDefault (valueRange.convertTo0to1 (defaultFloatValue)),
+      defaultValueInRange (defaultFloatValue),
       normalisableRange (valueRange)
 {
 }