Merge pull request #20 from WolvenKit/fix/wwise-bnk-v123

Use vlq instead of u4 to for event action count

CMakeLists.txt

@@ -31,6 +31,7 @@ set(SOURCE
     src/wwtools/bnk.cpp
     src/kaitai/kaitaistream.cpp
     src/kaitai/structs/bnk.cpp
+    src/kaitai/structs/vlq.cpp
     src/kaitai/structs/w3sc.cpp
     src/kaitai/structs/wem.cpp
   )

include/kaitai/exceptions.h

@@ -19,6 +19,41 @@
 
 namespace kaitai {
 
+/**
+ * Common ancestor for all errors related to `bytes_to_str` operation. Also used
+ * to signal misc non-specific `bytes_to_str` failures.
+ */
+class bytes_to_str_error: public std::runtime_error {
+public:
+    bytes_to_str_error(const std::string what):
+        std::runtime_error(std::string("bytes_to_str error: ") + what) {}
+
+    virtual ~bytes_to_str_error() KS_NOEXCEPT {};
+};
+
+/**
+ * Exception to signal that `bytes_to_str` operation was requested to use some encoding
+ * that is not available in given runtime environment.
+ */
+class unknown_encoding: public bytes_to_str_error {
+public:
+    unknown_encoding(const std::string enc_name):
+        bytes_to_str_error(std::string("unknown encoding: `") + enc_name + std::string("`")) {}
+
+    virtual ~unknown_encoding() KS_NOEXCEPT {};
+};
+
+/**
+ * Exception to signal that `bytes_to_str` operation failed to decode given byte sequence.
+ */
+class illegal_seq_in_encoding: public bytes_to_str_error {
+public:
+    illegal_seq_in_encoding(const std::string what):
+        bytes_to_str_error("illegal sequence: " + what) {}
+
+    virtual ~illegal_seq_in_encoding() KS_NOEXCEPT {};
+};
+
 /**
  * Common ancestor for all error originating from Kaitai Struct usage.
  * Stores KSY source path, pointing to an element supposedly guilty of
@@ -80,7 +115,7 @@ class validation_failed_error: public kstruct_error {
 template<typename T>
 class validation_not_equal_error: public validation_failed_error {
 public:
-    validation_not_equal_error<T>(const T& expected, const T& actual, kstream* io, const std::string src_path):
+    validation_not_equal_error(const T& expected, const T& actual, kstream* io, const std::string src_path):
         validation_failed_error("not equal", io, src_path),
         m_expected(expected),
         m_actual(actual)
@@ -89,7 +124,7 @@ class validation_not_equal_error: public validation_failed_error {
 
     // "not equal, expected #{expected.inspect}, but got #{actual.inspect}"
 
-    virtual ~validation_not_equal_error<T>() KS_NOEXCEPT {};
+    virtual ~validation_not_equal_error() KS_NOEXCEPT {};
 
 protected:
     const T& m_expected;
@@ -103,7 +138,7 @@ class validation_not_equal_error: public validation_failed_error {
 template<typename T>
 class validation_less_than_error: public validation_failed_error {
 public:
-    validation_less_than_error<T>(const T& min, const T& actual, kstream* io, const std::string src_path):
+    validation_less_than_error(const T& min, const T& actual, kstream* io, const std::string src_path):
         validation_failed_error("not in range", io, src_path),
         m_min(min),
         m_actual(actual)
@@ -112,7 +147,7 @@ class validation_less_than_error: public validation_failed_error {
 
     // "not in range, min #{min.inspect}, but got #{actual.inspect}"
 
-    virtual ~validation_less_than_error<T>() KS_NOEXCEPT {};
+    virtual ~validation_less_than_error() KS_NOEXCEPT {};
 
 protected:
     const T& m_min;
@@ -126,7 +161,7 @@ class validation_less_than_error: public validation_failed_error {
 template<typename T>
 class validation_greater_than_error: public validation_failed_error {
 public:
-    validation_greater_than_error<T>(const T& max, const T& actual, kstream* io, const std::string src_path):
+    validation_greater_than_error(const T& max, const T& actual, kstream* io, const std::string src_path):
         validation_failed_error("not in range", io, src_path),
         m_max(max),
         m_actual(actual)
@@ -135,7 +170,7 @@ class validation_greater_than_error: public validation_failed_error {
 
     // "not in range, max #{max.inspect}, but got #{actual.inspect}"
 
-    virtual ~validation_greater_than_error<T>() KS_NOEXCEPT {};
+    virtual ~validation_greater_than_error() KS_NOEXCEPT {};
 
 protected:
     const T& m_max;
@@ -149,15 +184,15 @@ class validation_greater_than_error: public validation_failed_error {
 template<typename T>
 class validation_not_any_of_error: public validation_failed_error {
 public:
-    validation_not_any_of_error<T>(const T& actual, kstream* io, const std::string src_path):
+    validation_not_any_of_error(const T& actual, kstream* io, const std::string src_path):
         validation_failed_error("not any of the list", io, src_path),
         m_actual(actual)
     {
     }
 
     // "not any of the list, got #{actual.inspect}"
 
-    virtual ~validation_not_any_of_error<T>() KS_NOEXCEPT {};
+    virtual ~validation_not_any_of_error() KS_NOEXCEPT {};
 
 protected:
     const T& m_actual;
@@ -170,15 +205,15 @@ class validation_not_any_of_error: public validation_failed_error {
 template<typename T>
 class validation_expr_error: public validation_failed_error {
 public:
-    validation_expr_error<T>(const T& actual, kstream* io, const std::string src_path):
+    validation_expr_error(const T& actual, kstream* io, const std::string src_path):
         validation_failed_error("not matching the expression", io, src_path),
         m_actual(actual)
     {
     }
 
     // "not matching the expression, got #{actual.inspect}"
 
-    virtual ~validation_expr_error<T>() KS_NOEXCEPT {};
+    virtual ~validation_expr_error() KS_NOEXCEPT {};
 
 protected:
     const T& m_actual;

include/kaitai/kaitaistream.h

@@ -2,13 +2,15 @@
 #define KAITAI_STREAM_H
 
 // Kaitai Struct runtime API version: x.y.z = 'xxxyyyzzz' decimal
-#define KAITAI_STRUCT_VERSION 9000L
+#define KAITAI_STRUCT_VERSION 11000L
 
 #include <istream>
 #include <sstream>
 #include <stdint.h>
 #include <sys/types.h>
 #include <limits>
+#include <stdexcept>
+#include <errno.h>
 
 namespace kaitai {
 
@@ -165,7 +167,7 @@ class kstream {
 
     static std::string bytes_strip_right(std::string src, char pad_byte);
     static std::string bytes_terminate(std::string src, char term, bool include);
-    static std::string bytes_to_str(std::string src, std::string src_enc);
+    static std::string bytes_to_str(const std::string src, const char *src_enc);
 
     //@}
 
@@ -226,31 +228,84 @@ class kstream {
      * since C++11) in older C++ implementations.
      */
     template<typename I>
-    static std::string to_string(I val) {
-        // check for c++11 support
-        // https://stackoverflow.com/a/27913885
-        #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900)
-        // check if we don't have something too large like GCC's `__int128_t`
-        static_assert(
-            std::is_integral<I>::value && 
-            std::numeric_limits<I>::max() >= 0 && 
+// check for C++11 support - https://stackoverflow.com/a/40512515
+#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900)
+    // https://stackoverflow.com/a/27913885
+    typename std::enable_if<
+            std::is_integral<I>::value &&
+            // check if we don't have something too large like GCC's `__int128_t`
+            std::numeric_limits<I>::max() >= 0 &&
             std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(),
-            "to_string() only works at best with unsigned 64-bit integers"
-        );
-        #endif
+            std::string
+    >::type
+#else
+    std::string
+#endif
+    static to_string(I val) {
         // in theory, `digits10 + 3` would be enough (minus sign + leading digit
         // + null terminator), but let's add a little more to be safe
         char buf[std::numeric_limits<I>::digits10 + 5];
         if (val < 0) {
             buf[0] = '-';
-            // get absolute value without undefined behavior (https://stackoverflow.com/a/12231604)
-            unsigned_to_decimal(-static_cast<uint64_t>(val), &buf[1]);
+
+            // NB: `val` is negative and we need to get its absolute value (i.e. minus `val`). However, since
+            // `int64_t` uses two's complement representation, its range is `[-2**63, 2**63 - 1] =
+            // [-0x8000_0000_0000_0000, 0x7fff_ffff_ffff_ffff]` (both ends inclusive) and thus the naive
+            // `-val` operation will overflow for `val = std::numeric_limits<int64_t>::min() =
+            // -0x8000_0000_0000_0000` (because the result of `-val` is mathematically
+            // `-(-0x8000_0000_0000_0000) = 0x8000_0000_0000_0000`, but the `int64_t` type can represent at
+            // most `0x7fff_ffff_ffff_ffff`). And signed integer overflow is undefined behavior in C++.
+            //
+            // To avoid undefined behavior for `val = -0x8000_0000_0000_0000 = -2**63`, we do the following
+            // steps for all negative `val`s:
+            //
+            // 1. Convert the signed (and negative) `val` to an unsigned `uint64_t` type. This is a
+            //    well-defined operation in C++: the resulting `uint64_t` value will be `val mod 2**64` (`mod`
+            //    is modulo). The maximum `val` we can have here is `-1` (because `val < 0`), a theoretical
+            //    minimum we are able to support would be `-2**64 + 1 = -0xffff_ffff_ffff_ffff` (even though
+            //    in practice the widest standard type is `int64_t` with the minimum of `-2**63`):
+            //
+            //    * `static_cast<uint64_t>(-1) = -1 mod 2**64 = 2**64 + (-1) = 0xffff_ffff_ffff_ffff = 2**64 - 1`
+            //    * `static_cast<uint64_t>(-2**64 + 1) = (-2**64 + 1) mod 2**64 = 2**64 + (-2**64 + 1) = 1`
+            //
+            // 2. Subtract `static_cast<uint64_t>(val)` from `2**64 - 1 = 0xffff_ffff_ffff_ffff`. Since
+            //    `static_cast<uint64_t>(val)` is in range `[1, 2**64 - 1]` (see step 1), the result of this
+            //    subtraction will be mathematically in range `[0, (2**64 - 1) - 1] = [0, 2**64 - 2]`. So the
+            //    mathematical result cannot be negative, hence this unsigned integer subtraction can never
+            //    wrap around (which wouldn't be a good thing to rely upon because it confuses programmers and
+            //    code analysis tools).
+            //
+            // 3. Since we did mathematically `(2**64 - 1) - (2**64 + val) = -val - 1` so far (and we wanted
+            //    to do `-val`), we add `1` to correct that. From step 2 we know that the result of `-val - 1`
+            //    is in range `[0, 2**64 - 2]`, so adding `1` will not wrap (at most we could get `2**64 - 1 =
+            //    0xffff_ffff_ffff_ffff`, which is still in the valid range of `uint64_t`).
+
+            unsigned_to_decimal((std::numeric_limits<uint64_t>::max() - static_cast<uint64_t>(val)) + 1, &buf[1]);
         } else {
             unsigned_to_decimal(val, buf);
         }
         return std::string(buf);
     }
 
+    /**
+     * Converts string `str` to an integer value. Throws an exception if the
+     * string is not a valid integer.
+     *
+     * This one is supposed to mirror `std::stoll()` (which is available only
+     * since C++11) in older C++ implementations.
+     *
+     * Major difference between standard `std::stoll()` and `string_to_int()`
+     * is that this one does not perform any partial conversions and always
+     * throws `std::invalid_argument` if the string is not a valid integer.
+     *
+     * @param str String to convert
+     * @param base Base of the integer (default: 10)
+     * @throws std::invalid_argument if the string is not a valid integer
+     * @throws std::out_of_range if the integer is out of range
+     * @return Integer value of the string
+     */
+    static int64_t string_to_int(const std::string& str, int base = 10);
+
     /**
      * Reverses given string `val`, so that the first character becomes the
      * last and the last one becomes the first. This should be used to avoid
@@ -285,6 +340,32 @@ class kstream {
 
     static void unsigned_to_decimal(uint64_t number, char *buffer);
 
+#ifdef KS_STR_ENCODING_WIN32API
+    enum {
+        KAITAI_CP_UNSUPPORTED = -1,
+        KAITAI_CP_UTF16LE = -2,
+        KAITAI_CP_UTF16BE = -3,
+    };
+
+    /**
+     * Converts string name of the encoding into a Windows codepage number. We extend standard Windows codepage list
+     * with a few special meanings (see KAITAI_CP_* enum), reserving negative values of integer for that.
+     * @param src_enc string name of the encoding; this should match canonical name of the encoding as per discussion
+     *     in https://github.com/kaitai-io/kaitai_struct/issues/116
+     * @return Windows codepage number or member of KAITAI_CP_* enum.
+     * @ref https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+     */
+    static int encoding_to_win_codepage(const char *src_enc);
+
+    /**
+     * Converts bytes packed in std::string into a UTF-8 string, based on given source encoding indicated by `codepage`.
+     * @param src bytes to be converted
+     * @param codepage Windows codepage number or member of KAITAI_CP_* enum.
+     * @return UTF-8 string
+     */
+    static std::string bytes_to_str(const std::string src, int codepage);
+#endif
+
     static const int ZLIB_BUF_SIZE = 128 * 1024;
 };
 

include/kaitai/structs/bnk.h

@@ -5,11 +5,13 @@
 
 #include "kaitai/kaitaistruct.h"
 #include <stdint.h>
+#include "vlq.h"
 #include <vector>
 
-#if KAITAI_STRUCT_VERSION < 9000L
-#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required"
+#if KAITAI_STRUCT_VERSION < 11000L
+#error "Incompatible Kaitai Struct C++/STL API: version 0.11 or later is required"
 #endif
+class vlq_t;
 
 class bnk_t : public kaitai::kstruct {
 
@@ -294,18 +296,14 @@ class bnk_t : public kaitai::kstruct {
     private:
         uint32_t m_version;
         uint32_t m_id;
-        std::string m_blank1;
-        std::string m_blank2;
         std::string m_random;
         uint32_t m_length;
         bnk_t* m__root;
         bnk_t::section_t* m__parent;
 
     public:
-        [[nodiscard]] uint32_t version() const;
+        uint32_t version() const { return m_version; }
         uint32_t id() const { return m_id; }
-        std::string blank1() const { return m_blank1; }
-        std::string blank2() const { return m_blank2; }
         std::string random() const { return m_random; }
         uint32_t length() const { return m_length; }
         bnk_t* _root() const { return m__root; }
@@ -326,13 +324,41 @@ class bnk_t : public kaitai::kstruct {
         ~event_t();
 
     private:
-        uint32_t m_event_action_count;
+        bool f_version;
+        uint32_t m_version;
+
+    public:
+        uint32_t version();
+
+    private:
+        bool f_event_action_count_value;
+        int32_t m_event_action_count_value;
+
+    public:
+        int32_t event_action_count_value();
+
+    private:
+        vlq_t* m_event_action_count_new;
+        bool n_event_action_count_new;
+
+    public:
+        bool _is_null_event_action_count_new() { event_action_count_new(); return n_event_action_count_new; };
+
+    private:
+        uint32_t m_event_action_count_old;
+        bool n_event_action_count_old;
+
+    public:
+        bool _is_null_event_action_count_old() { event_action_count_old(); return n_event_action_count_old; };
+
+    private:
         std::vector<uint32_t>* m_event_actions;
         bnk_t* m__root;
         bnk_t::hirc_obj_t* m__parent;
 
     public:
-        uint32_t event_action_count() const { return m_event_action_count; }
+        vlq_t* event_action_count_new() const { return m_event_action_count_new; }
+        uint32_t event_action_count_old() const { return m_event_action_count_old; }
         std::vector<uint32_t>* event_actions() const { return m_event_actions; }
         bnk_t* _root() const { return m__root; }
         bnk_t::hirc_obj_t* _parent() const { return m__parent; }
@@ -1165,4 +1191,4 @@ class bnk_t : public kaitai::kstruct {
     kaitai::kstruct* _parent() const { return m__parent; }
 };
 
-#endif  // BNK_H_
+#endif  // BNK_H_