From ae72ad1beb8969eac954252116bf2c9d13e6e24f Mon Sep 17 00:00:00 2001 From: Abheek Dhawan <67982792+abheekda1@users.noreply.github.com> Date: Wed, 17 Jan 2024 10:25:00 -0600 Subject: [PATCH 1/8] fix: use vlq instead of u4 to for event action count --- ksy/bnk.ksy | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ksy/bnk.ksy b/ksy/bnk.ksy index 6c79e6d..f220c67 100644 --- a/ksy/bnk.ksy +++ b/ksy/bnk.ksy @@ -2,6 +2,8 @@ meta: id: bnk file-extension: bnk endian: le + imports: + - vlq # https://formats.kaitai.io/vlq_base128_le/ seq: - id: data @@ -47,12 +49,14 @@ types: type: u4 - id: id type: u4 - - id: blank1 - contents: [00, 00, 00, 00] - - id: blank2 - contents: [00, 00, 00, 00] + #- id: blank1 + # contents: [00, 00, 00, 00] + #- id: blank2 + # contents: [00, 00, 00, 00] + #- id: random + # size: length - 16 - id: random - size: length - 16 + size: length - 8 # BKHD END # DIDX BEGIN @@ -252,11 +256,11 @@ types: event: seq: - id: event_action_count - type: u4 + type: vlq - id: event_actions type: u4 repeat: expr - repeat-expr: event_action_count + repeat-expr: event_action_count.value audio_bus: seq: - id: parent_audio_bus_id @@ -482,8 +486,10 @@ types: seq: - id: data size: size +# HIRC END enums: +# HIRC ENUM BEGIN object_type: 1: settings 2: sound_effect_or_voice @@ -550,4 +556,4 @@ enums: 3: random_continuous 4: sequence_step_new_path 5: random_step_new_path -# HIRC END +# HIRC ENUM END From d3c841f5887e28ba8442eda74606515c31c1361b Mon Sep 17 00:00:00 2001 From: Abheek Dhawan <67982792+abheekda1@users.noreply.github.com> Date: Wed, 17 Jan 2024 10:27:39 -0600 Subject: [PATCH 2/8] feat: add vlq kaitai spec --- ksy/vlq.ksy | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 ksy/vlq.ksy diff --git a/ksy/vlq.ksy b/ksy/vlq.ksy new file mode 100644 index 0000000..a077000 --- /dev/null +++ b/ksy/vlq.ksy @@ -0,0 +1,65 @@ +meta: + id: vlq_base128_le + title: Variable length quantity, unsigned/signed integer, base128, little-endian + license: CC0-1.0 + ks-version: 0.7 +doc: | + A variable-length unsigned/signed integer using base128 encoding. 1-byte groups + consist of 1-bit flag of continuation and 7-bit value chunk, and are ordered + "least significant group first", i.e. in "little-endian" manner. + + This particular encoding is specified and used in: + + * DWARF debug file format, where it's dubbed "unsigned LEB128" or "ULEB128". + http://dwarfstd.org/doc/dwarf-2.0.0.pdf - page 139 + * Google Protocol Buffers, where it's called "Base 128 Varints". + https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints + * Apache Lucene, where it's called "VInt" + https://lucene.apache.org/core/3_5_0/fileformats.html#VInt + * Apache Avro uses this as a basis for integer encoding, adding ZigZag on + top of it for signed ints + https://avro.apache.org/docs/current/spec.html#binary_encode_primitive + + More information on this encoding is available at https://en.wikipedia.org/wiki/LEB128 + + This particular implementation supports serialized values to up 8 bytes long. +-webide-representation: '{value:dec}' +seq: + - id: groups + type: group + repeat: until + repeat-until: not _.has_next +types: + group: + -webide-representation: '{value}' + doc: | + One byte group, clearly divided into 7-bit "value" chunk and 1-bit "continuation" flag. + seq: + - id: b + type: u1 + instances: + has_next: + value: (b & 0b1000_0000) != 0 + doc: If true, then we have more bytes to read + value: + value: b & 0b0111_1111 + doc: The 7-bit (base128) numeric value chunk of this group +instances: + len: + value: groups.size + value: + value: >- + groups[0].value + + (len >= 2 ? (groups[1].value << 7) : 0) + + (len >= 3 ? (groups[2].value << 14) : 0) + + (len >= 4 ? (groups[3].value << 21) : 0) + + (len >= 5 ? (groups[4].value << 28) : 0) + + (len >= 6 ? (groups[5].value << 35) : 0) + + (len >= 7 ? (groups[6].value << 42) : 0) + + (len >= 8 ? (groups[7].value << 49) : 0) + doc: Resulting unsigned value as normal integer + sign_bit: + value: '1 << (7 * len - 1)' + value_signed: + value: '(value ^ sign_bit) - sign_bit' + doc-ref: https://graphics.stanford.edu/~seander/bithacks.html#VariableSignExtend From 31b0be388efca37d471bc85817bc6a56de830547 Mon Sep 17 00:00:00 2001 From: Abheek Dhawan <67982792+abheekda1@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:43:28 -0600 Subject: [PATCH 3/8] fix: allow for both new and old versions of wwise events --- ksy/bnk.ksy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ksy/bnk.ksy b/ksy/bnk.ksy index f220c67..626e586 100644 --- a/ksy/bnk.ksy +++ b/ksy/bnk.ksy @@ -255,12 +255,22 @@ types: type: random_bytes(_parent.length - 4 - 1 - 1 - 4 - 1 - 1 - (1 * parameter_count) - (4 * parameter_count) - 1 - (8 * (type == action_type::set_state ? 1 : 0)) - (8 * (type == action_type::set_switch ? 1 : 0))) event: seq: - - id: event_action_count + - id: event_action_count_new type: vlq + if: version >= 123 + - id: event_action_count_old + type: u4 + if: version < 123 - id: event_actions type: u4 repeat: expr - repeat-expr: event_action_count.value + repeat-expr: event_action_count_value + instances: + version: + value: _root.data[0].section_data.as.version + event_action_count_value: + value: 'version >= 123 ? event_action_count_new.as.value : event_action_count_old.as' + audio_bus: seq: - id: parent_audio_bus_id From 9d919ef463f0edc69062be158ff9f4ed9c915740 Mon Sep 17 00:00:00 2001 From: Abheek Dhawan Date: Wed, 17 Jan 2024 13:36:35 -0600 Subject: [PATCH 4/8] feat: update kaitai runtime and bnk cpp header/source --- include/kaitai/exceptions.h | 55 +++++++-- include/kaitai/kaitaistream.h | 111 +++++++++++++++--- include/kaitai/structs/bnk.h | 46 ++++++-- include/kaitai/structs/vlq.h | 147 +++++++++++++++++++++++ ksy/bnk.ksy | 8 +- src/kaitai/kaitaistream.cpp | 215 +++++++++++++++++++++++++++++++--- src/kaitai/structs/bnk.cpp | 168 +++++++++++++------------- src/kaitai/structs/vlq.cpp | 119 +++++++++++++++++++ 8 files changed, 735 insertions(+), 134 deletions(-) create mode 100644 include/kaitai/structs/vlq.h create mode 100644 src/kaitai/structs/vlq.cpp diff --git a/include/kaitai/exceptions.h b/include/kaitai/exceptions.h index e86d6fd..b1a2f31 100644 --- a/include/kaitai/exceptions.h +++ b/include/kaitai/exceptions.h @@ -19,6 +19,41 @@ namespace kaitai { +/** + * Common ancestor for all errors related to `bytes_to_str` operation. Also used + * to signal misc non-specific `bytes_to_str` failures. + */ +class bytes_to_str_error: public std::runtime_error { +public: + bytes_to_str_error(const std::string what): + std::runtime_error(std::string("bytes_to_str error: ") + what) {} + + virtual ~bytes_to_str_error() KS_NOEXCEPT {}; +}; + +/** + * Exception to signal that `bytes_to_str` operation was requested to use some encoding + * that is not available in given runtime environment. + */ +class unknown_encoding: public bytes_to_str_error { +public: + unknown_encoding(const std::string enc_name): + bytes_to_str_error(std::string("unknown encoding: `") + enc_name + std::string("`")) {} + + virtual ~unknown_encoding() KS_NOEXCEPT {}; +}; + +/** + * Exception to signal that `bytes_to_str` operation failed to decode given byte sequence. + */ +class illegal_seq_in_encoding: public bytes_to_str_error { +public: + illegal_seq_in_encoding(const std::string what): + bytes_to_str_error("illegal sequence: " + what) {} + + virtual ~illegal_seq_in_encoding() KS_NOEXCEPT {}; +}; + /** * Common ancestor for all error originating from Kaitai Struct usage. * Stores KSY source path, pointing to an element supposedly guilty of @@ -80,7 +115,7 @@ class validation_failed_error: public kstruct_error { template class validation_not_equal_error: public validation_failed_error { public: - validation_not_equal_error(const T& expected, const T& actual, kstream* io, const std::string src_path): + validation_not_equal_error(const T& expected, const T& actual, kstream* io, const std::string src_path): validation_failed_error("not equal", io, src_path), m_expected(expected), m_actual(actual) @@ -89,7 +124,7 @@ class validation_not_equal_error: public validation_failed_error { // "not equal, expected #{expected.inspect}, but got #{actual.inspect}" - virtual ~validation_not_equal_error() KS_NOEXCEPT {}; + virtual ~validation_not_equal_error() KS_NOEXCEPT {}; protected: const T& m_expected; @@ -103,7 +138,7 @@ class validation_not_equal_error: public validation_failed_error { template class validation_less_than_error: public validation_failed_error { public: - validation_less_than_error(const T& min, const T& actual, kstream* io, const std::string src_path): + validation_less_than_error(const T& min, const T& actual, kstream* io, const std::string src_path): validation_failed_error("not in range", io, src_path), m_min(min), m_actual(actual) @@ -112,7 +147,7 @@ class validation_less_than_error: public validation_failed_error { // "not in range, min #{min.inspect}, but got #{actual.inspect}" - virtual ~validation_less_than_error() KS_NOEXCEPT {}; + virtual ~validation_less_than_error() KS_NOEXCEPT {}; protected: const T& m_min; @@ -126,7 +161,7 @@ class validation_less_than_error: public validation_failed_error { template class validation_greater_than_error: public validation_failed_error { public: - validation_greater_than_error(const T& max, const T& actual, kstream* io, const std::string src_path): + validation_greater_than_error(const T& max, const T& actual, kstream* io, const std::string src_path): validation_failed_error("not in range", io, src_path), m_max(max), m_actual(actual) @@ -135,7 +170,7 @@ class validation_greater_than_error: public validation_failed_error { // "not in range, max #{max.inspect}, but got #{actual.inspect}" - virtual ~validation_greater_than_error() KS_NOEXCEPT {}; + virtual ~validation_greater_than_error() KS_NOEXCEPT {}; protected: const T& m_max; @@ -149,7 +184,7 @@ class validation_greater_than_error: public validation_failed_error { template class validation_not_any_of_error: public validation_failed_error { public: - validation_not_any_of_error(const T& actual, kstream* io, const std::string src_path): + validation_not_any_of_error(const T& actual, kstream* io, const std::string src_path): validation_failed_error("not any of the list", io, src_path), m_actual(actual) { @@ -157,7 +192,7 @@ class validation_not_any_of_error: public validation_failed_error { // "not any of the list, got #{actual.inspect}" - virtual ~validation_not_any_of_error() KS_NOEXCEPT {}; + virtual ~validation_not_any_of_error() KS_NOEXCEPT {}; protected: const T& m_actual; @@ -170,7 +205,7 @@ class validation_not_any_of_error: public validation_failed_error { template class validation_expr_error: public validation_failed_error { public: - validation_expr_error(const T& actual, kstream* io, const std::string src_path): + validation_expr_error(const T& actual, kstream* io, const std::string src_path): validation_failed_error("not matching the expression", io, src_path), m_actual(actual) { @@ -178,7 +213,7 @@ class validation_expr_error: public validation_failed_error { // "not matching the expression, got #{actual.inspect}" - virtual ~validation_expr_error() KS_NOEXCEPT {}; + virtual ~validation_expr_error() KS_NOEXCEPT {}; protected: const T& m_actual; diff --git a/include/kaitai/kaitaistream.h b/include/kaitai/kaitaistream.h index 7a56361..c8b5a3a 100644 --- a/include/kaitai/kaitaistream.h +++ b/include/kaitai/kaitaistream.h @@ -2,13 +2,15 @@ #define KAITAI_STREAM_H // Kaitai Struct runtime API version: x.y.z = 'xxxyyyzzz' decimal -#define KAITAI_STRUCT_VERSION 9000L +#define KAITAI_STRUCT_VERSION 11000L #include #include #include #include #include +#include +#include namespace kaitai { @@ -165,7 +167,7 @@ class kstream { static std::string bytes_strip_right(std::string src, char pad_byte); static std::string bytes_terminate(std::string src, char term, bool include); - static std::string bytes_to_str(std::string src, std::string src_enc); + static std::string bytes_to_str(const std::string src, const char *src_enc); //@} @@ -226,31 +228,84 @@ class kstream { * since C++11) in older C++ implementations. */ template - static std::string to_string(I val) { - // check for c++11 support - // https://stackoverflow.com/a/27913885 - #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900) - // check if we don't have something too large like GCC's `__int128_t` - static_assert( - std::is_integral::value && - std::numeric_limits::max() >= 0 && +// check for C++11 support - https://stackoverflow.com/a/40512515 +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900) + // https://stackoverflow.com/a/27913885 + typename std::enable_if< + std::is_integral::value && + // check if we don't have something too large like GCC's `__int128_t` + std::numeric_limits::max() >= 0 && std::numeric_limits::max() <= std::numeric_limits::max(), - "to_string() only works at best with unsigned 64-bit integers" - ); - #endif + std::string + >::type +#else + std::string +#endif + static to_string(I val) { // in theory, `digits10 + 3` would be enough (minus sign + leading digit // + null terminator), but let's add a little more to be safe char buf[std::numeric_limits::digits10 + 5]; if (val < 0) { buf[0] = '-'; - // get absolute value without undefined behavior (https://stackoverflow.com/a/12231604) - unsigned_to_decimal(-static_cast(val), &buf[1]); + + // NB: `val` is negative and we need to get its absolute value (i.e. minus `val`). However, since + // `int64_t` uses two's complement representation, its range is `[-2**63, 2**63 - 1] = + // [-0x8000_0000_0000_0000, 0x7fff_ffff_ffff_ffff]` (both ends inclusive) and thus the naive + // `-val` operation will overflow for `val = std::numeric_limits::min() = + // -0x8000_0000_0000_0000` (because the result of `-val` is mathematically + // `-(-0x8000_0000_0000_0000) = 0x8000_0000_0000_0000`, but the `int64_t` type can represent at + // most `0x7fff_ffff_ffff_ffff`). And signed integer overflow is undefined behavior in C++. + // + // To avoid undefined behavior for `val = -0x8000_0000_0000_0000 = -2**63`, we do the following + // steps for all negative `val`s: + // + // 1. Convert the signed (and negative) `val` to an unsigned `uint64_t` type. This is a + // well-defined operation in C++: the resulting `uint64_t` value will be `val mod 2**64` (`mod` + // is modulo). The maximum `val` we can have here is `-1` (because `val < 0`), a theoretical + // minimum we are able to support would be `-2**64 + 1 = -0xffff_ffff_ffff_ffff` (even though + // in practice the widest standard type is `int64_t` with the minimum of `-2**63`): + // + // * `static_cast(-1) = -1 mod 2**64 = 2**64 + (-1) = 0xffff_ffff_ffff_ffff = 2**64 - 1` + // * `static_cast(-2**64 + 1) = (-2**64 + 1) mod 2**64 = 2**64 + (-2**64 + 1) = 1` + // + // 2. Subtract `static_cast(val)` from `2**64 - 1 = 0xffff_ffff_ffff_ffff`. Since + // `static_cast(val)` is in range `[1, 2**64 - 1]` (see step 1), the result of this + // subtraction will be mathematically in range `[0, (2**64 - 1) - 1] = [0, 2**64 - 2]`. So the + // mathematical result cannot be negative, hence this unsigned integer subtraction can never + // wrap around (which wouldn't be a good thing to rely upon because it confuses programmers and + // code analysis tools). + // + // 3. Since we did mathematically `(2**64 - 1) - (2**64 + val) = -val - 1` so far (and we wanted + // to do `-val`), we add `1` to correct that. From step 2 we know that the result of `-val - 1` + // is in range `[0, 2**64 - 2]`, so adding `1` will not wrap (at most we could get `2**64 - 1 = + // 0xffff_ffff_ffff_ffff`, which is still in the valid range of `uint64_t`). + + unsigned_to_decimal((std::numeric_limits::max() - static_cast(val)) + 1, &buf[1]); } else { unsigned_to_decimal(val, buf); } return std::string(buf); } + /** + * Converts string `str` to an integer value. Throws an exception if the + * string is not a valid integer. + * + * This one is supposed to mirror `std::stoll()` (which is available only + * since C++11) in older C++ implementations. + * + * Major difference between standard `std::stoll()` and `string_to_int()` + * is that this one does not perform any partial conversions and always + * throws `std::invalid_argument` if the string is not a valid integer. + * + * @param str String to convert + * @param base Base of the integer (default: 10) + * @throws std::invalid_argument if the string is not a valid integer + * @throws std::out_of_range if the integer is out of range + * @return Integer value of the string + */ + static int64_t string_to_int(const std::string& str, int base = 10); + /** * Reverses given string `val`, so that the first character becomes the * last and the last one becomes the first. This should be used to avoid @@ -285,6 +340,32 @@ class kstream { static void unsigned_to_decimal(uint64_t number, char *buffer); +#ifdef KS_STR_ENCODING_WIN32API + enum { + KAITAI_CP_UNSUPPORTED = -1, + KAITAI_CP_UTF16LE = -2, + KAITAI_CP_UTF16BE = -3, + }; + + /** + * Converts string name of the encoding into a Windows codepage number. We extend standard Windows codepage list + * with a few special meanings (see KAITAI_CP_* enum), reserving negative values of integer for that. + * @param src_enc string name of the encoding; this should match canonical name of the encoding as per discussion + * in https://github.com/kaitai-io/kaitai_struct/issues/116 + * @return Windows codepage number or member of KAITAI_CP_* enum. + * @ref https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers + */ + static int encoding_to_win_codepage(const char *src_enc); + + /** + * Converts bytes packed in std::string into a UTF-8 string, based on given source encoding indicated by `codepage`. + * @param src bytes to be converted + * @param codepage Windows codepage number or member of KAITAI_CP_* enum. + * @return UTF-8 string + */ + static std::string bytes_to_str(const std::string src, int codepage); +#endif + static const int ZLIB_BUF_SIZE = 128 * 1024; }; diff --git a/include/kaitai/structs/bnk.h b/include/kaitai/structs/bnk.h index 2dd69f4..35002c2 100644 --- a/include/kaitai/structs/bnk.h +++ b/include/kaitai/structs/bnk.h @@ -5,11 +5,13 @@ #include "kaitai/kaitaistruct.h" #include +#include "vlq.h" #include -#if KAITAI_STRUCT_VERSION < 9000L -#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#if KAITAI_STRUCT_VERSION < 11000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.11 or later is required" #endif +class vlq_t; class bnk_t : public kaitai::kstruct { @@ -294,18 +296,14 @@ class bnk_t : public kaitai::kstruct { private: uint32_t m_version; uint32_t m_id; - std::string m_blank1; - std::string m_blank2; std::string m_random; uint32_t m_length; bnk_t* m__root; bnk_t::section_t* m__parent; public: - [[nodiscard]] uint32_t version() const; + uint32_t version() const { return m_version; } uint32_t id() const { return m_id; } - std::string blank1() const { return m_blank1; } - std::string blank2() const { return m_blank2; } std::string random() const { return m_random; } uint32_t length() const { return m_length; } bnk_t* _root() const { return m__root; } @@ -326,13 +324,41 @@ class bnk_t : public kaitai::kstruct { ~event_t(); private: - uint32_t m_event_action_count; + bool f_version; + uint32_t m_version; + + public: + uint32_t version(); + + private: + bool f_event_action_count_value; + int32_t m_event_action_count_value; + + public: + int32_t event_action_count_value(); + + private: + vlq_t* m_event_action_count_new; + bool n_event_action_count_new; + + public: + bool _is_null_event_action_count_new() { event_action_count_new(); return n_event_action_count_new; }; + + private: + uint32_t m_event_action_count_old; + bool n_event_action_count_old; + + public: + bool _is_null_event_action_count_old() { event_action_count_old(); return n_event_action_count_old; }; + + private: std::vector* m_event_actions; bnk_t* m__root; bnk_t::hirc_obj_t* m__parent; public: - uint32_t event_action_count() const { return m_event_action_count; } + vlq_t* event_action_count_new() const { return m_event_action_count_new; } + uint32_t event_action_count_old() const { return m_event_action_count_old; } std::vector* event_actions() const { return m_event_actions; } bnk_t* _root() const { return m__root; } bnk_t::hirc_obj_t* _parent() const { return m__parent; } @@ -1165,4 +1191,4 @@ class bnk_t : public kaitai::kstruct { kaitai::kstruct* _parent() const { return m__parent; } }; -#endif // BNK_H_ +#endif // BNK_H_ \ No newline at end of file diff --git a/include/kaitai/structs/vlq.h b/include/kaitai/structs/vlq.h new file mode 100644 index 0000000..03285f4 --- /dev/null +++ b/include/kaitai/structs/vlq.h @@ -0,0 +1,147 @@ +#ifndef VLQ_H_ +#define VLQ_H_ + +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "kaitai/kaitaistruct.h" +#include +#include + +#if KAITAI_STRUCT_VERSION < 11000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.11 or later is required" +#endif + +/** + * A variable-length unsigned/signed integer using base128 encoding. 1-byte groups + * consist of 1-bit flag of continuation and 7-bit value chunk, and are ordered + * "least significant group first", i.e. in "little-endian" manner. + * + * This particular encoding is specified and used in: + * + * * DWARF debug file format, where it's dubbed "unsigned LEB128" or "ULEB128". + * http://dwarfstd.org/doc/dwarf-2.0.0.pdf - page 139 + * * Google Protocol Buffers, where it's called "Base 128 Varints". + * https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints + * * Apache Lucene, where it's called "VInt" + * https://lucene.apache.org/core/3_5_0/fileformats.html#VInt + * * Apache Avro uses this as a basis for integer encoding, adding ZigZag on + * top of it for signed ints + * https://avro.apache.org/docs/current/spec.html#binary_encode_primitive + * + * More information on this encoding is available at https://en.wikipedia.org/wiki/LEB128 + * + * This particular implementation supports serialized values to up 8 bytes long. + */ + +class vlq_t : public kaitai::kstruct { + +public: + class group_t; + + vlq_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent = 0, vlq_t* p__root = 0); + +private: + void _read(); + void _clean_up(); + +public: + ~vlq_t(); + + /** + * One byte group, clearly divided into 7-bit "value" chunk and 1-bit "continuation" flag. + */ + + class group_t : public kaitai::kstruct { + + public: + + group_t(kaitai::kstream* p__io, vlq_t* p__parent = 0, vlq_t* p__root = 0); + + private: + void _read(); + void _clean_up(); + + public: + ~group_t(); + + private: + bool f_has_next; + bool m_has_next; + + public: + + /** + * If true, then we have more bytes to read + */ + bool has_next(); + + private: + bool f_value; + int32_t m_value; + + public: + + /** + * The 7-bit (base128) numeric value chunk of this group + */ + int32_t value(); + + private: + uint8_t m_b; + vlq_t* m__root; + vlq_t* m__parent; + + public: + uint8_t b() const { return m_b; } + vlq_t* _root() const { return m__root; } + vlq_t* _parent() const { return m__parent; } + }; + +private: + bool f_len; + int32_t m_len; + +public: + int32_t len(); + +private: + bool f_value; + int32_t m_value; + +public: + + /** + * Resulting unsigned value as normal integer + */ + int32_t value(); + +private: + bool f_sign_bit; + int32_t m_sign_bit; + +public: + int32_t sign_bit(); + +private: + bool f_value_signed; + int32_t m_value_signed; + +public: + + /** + * \sa https://graphics.stanford.edu/~seander/bithacks.html#VariableSignExtend Source + */ + int32_t value_signed(); + +private: + std::vector* m_groups; + vlq_t* m__root; + kaitai::kstruct* m__parent; + +public: + std::vector* groups() const { return m_groups; } + vlq_t* _root() const { return m__root; } + kaitai::kstruct* _parent() const { return m__parent; } +}; + +#endif // VLQ_H_ diff --git a/ksy/bnk.ksy b/ksy/bnk.ksy index 626e586..8f05864 100644 --- a/ksy/bnk.ksy +++ b/ksy/bnk.ksy @@ -3,7 +3,7 @@ meta: file-extension: bnk endian: le imports: - - vlq # https://formats.kaitai.io/vlq_base128_le/ + - /home/abheekd/dev/wwise-audio-tools/ksy/vlq # https://formats.kaitai.io/vlq_base128_le/ seq: - id: data @@ -203,7 +203,7 @@ types: - id: sound_object_type type: s1 - id: sound_structure - size: _parent.as.length - (4 * 5) - (1 * 1) - (8 * (included_or_streamed == 0 ? 1 : 0)) + size: '_parent.as.length - (4 * 5) - (1 * 1) - (8 * (included_or_streamed == 0 ? 1 : 0))' #type: sound_structure instances: wem_data: @@ -252,7 +252,7 @@ types: type: u4 if: type == action_type::set_switch - id: extra - type: random_bytes(_parent.length - 4 - 1 - 1 - 4 - 1 - 1 - (1 * parameter_count) - (4 * parameter_count) - 1 - (8 * (type == action_type::set_state ? 1 : 0)) - (8 * (type == action_type::set_switch ? 1 : 0))) + type: 'random_bytes(_parent.length - 4 - 1 - 1 - 4 - 1 - 1 - (1 * parameter_count) - (4 * parameter_count) - 1 - (8 * (type == action_type::set_state ? 1 : 0)) - (8 * (type == action_type::set_switch ? 1 : 0)))' event: seq: - id: event_action_count_new @@ -506,7 +506,7 @@ enums: 3: event_action 4: event 5: random_or_sequence_container - 5: switch_container + 6: switch_container 7: actor_mixer 8: audio_bus 9: blend_container diff --git a/src/kaitai/kaitaistream.cpp b/src/kaitai/kaitaistream.cpp index dc965b8..625b890 100644 --- a/src/kaitai/kaitaistream.cpp +++ b/src/kaitai/kaitaistream.cpp @@ -1,4 +1,5 @@ #include +#include #if defined(__APPLE__) #include @@ -9,7 +10,7 @@ #define __BYTE_ORDER BYTE_ORDER #define __BIG_ENDIAN BIG_ENDIAN #define __LITTLE_ENDIAN LITTLE_ENDIAN -#elif defined(_MSC_VER) || defined(__MINGW32__) // !__APPLE__ +#elif defined(_MSC_VER) // !__APPLE__ #include #define __LITTLE_ENDIAN 1234 #define __BIG_ENDIAN 4321 @@ -17,7 +18,16 @@ #define bswap_16(x) _byteswap_ushort(x) #define bswap_32(x) _byteswap_ulong(x) #define bswap_64(x) _byteswap_uint64(x) -#else // !__APPLE__ or !_MSC_VER +#elif defined(__QNX__) // __QNX__ +#include +#include +#define bswap_16(x) ENDIAN_RET16(x) +#define bswap_32(x) ENDIAN_RET32(x) +#define bswap_64(x) ENDIAN_RET64(x) +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#else // !__APPLE__ or !_MSC_VER or !__QNX__ #include #include #endif @@ -322,7 +332,7 @@ uint64_t kaitai::kstream::read_bits_int_be(int n) { res = m_bits >> -bits_needed; // shift unneeded bits out } - uint64_t mask = (UINT64_C(1) << m_bits_left) - 1; // `m_bits_left` is in range 0..7, so `(1 << 64)` does not have to be considered + uint64_t mask = (static_cast(1) << m_bits_left) - 1; // `m_bits_left` is in range 0..7, so `(1 << 64)` does not have to be considered m_bits &= mask; return res; @@ -366,7 +376,7 @@ uint64_t kaitai::kstream::read_bits_int_le(int n) { m_bits_left = -bits_needed & 7; // `-bits_needed mod 8` if (n < 64) { - uint64_t mask = (UINT64_C(1) << n) - 1; + uint64_t mask = (static_cast(1) << n) - 1; res &= mask; } // if `n == 64`, do nothing @@ -583,6 +593,24 @@ void kaitai::kstream::unsigned_to_decimal(uint64_t number, char *buffer) { *buffer = '\0'; } +int64_t kaitai::kstream::string_to_int(const std::string& str, int base) { + char *str_end; + + errno = 0; + int64_t res = strtoll(str.c_str(), &str_end, base); + + // Check for successful conversion and throw an exception if the entire string was not converted + if (str_end != str.c_str() + str.size()) { + throw std::invalid_argument("string_to_int"); + } + + if (errno == ERANGE) { + throw std::out_of_range("string_to_int"); + } + + return res; +} + std::string kaitai::kstream::reverse(std::string val) { std::reverse(val.begin(), val.end()); @@ -627,14 +655,14 @@ uint8_t kaitai::kstream::byte_array_max(const std::string val) { #include #include -std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) { - iconv_t cd = iconv_open(KS_STR_DEFAULT_ENCODING, src_enc.c_str()); +std::string kaitai::kstream::bytes_to_str(const std::string src, const char *src_enc) { + iconv_t cd = iconv_open(KS_STR_DEFAULT_ENCODING, src_enc); if (cd == (iconv_t)-1) { if (errno == EINVAL) { - throw std::runtime_error("bytes_to_str: invalid encoding pair conversion requested"); + throw unknown_encoding(src_enc); } else { - throw std::runtime_error("bytes_to_str: error opening iconv"); + throw bytes_to_str_error("error opening iconv"); } } @@ -646,7 +674,9 @@ std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) std::string dst(dst_len, ' '); size_t dst_left = dst_len; - char *src_ptr = &src[0]; + // NB: this should be const char *, but for some reason iconv() requires non-const in its 2nd argument, + // so we force it with a cast. + char *src_ptr = const_cast(src.data()); char *dst_ptr = &dst[0]; while (true) { @@ -665,8 +695,12 @@ std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) // of memory, thus our previous pointer "dst" will be invalid; re-point // it using "dst_used". dst_ptr = &dst[dst_used]; + } else if (errno == EILSEQ) { + throw illegal_seq_in_encoding("EILSEQ"); + } else if (errno == EINVAL) { + throw illegal_seq_in_encoding("EINVAL"); } else { - throw std::runtime_error("bytes_to_str: iconv error"); + throw bytes_to_str_error(to_string(errno)); } } else { // conversion successful @@ -676,15 +710,168 @@ std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) } if (iconv_close(cd) != 0) { - throw std::runtime_error("bytes_to_str: iconv close error"); + throw bytes_to_str_error("iconv close error"); } return dst; } #elif defined(KS_STR_ENCODING_NONE) -std::string kaitai::kstream::bytes_to_str(std::string src, std::string src_enc) { +std::string kaitai::kstream::bytes_to_str(const std::string src, const char *src_enc) { return src; } +#elif defined(KS_STR_ENCODING_WIN32API) +#include +#include + +// Unbreak std::numeric_limits::max, as otherwise MSVC substitutes "useful" max() macro. +#undef max + +int kaitai::kstream::encoding_to_win_codepage(const char *src_enc) { + std::string enc(src_enc); + if (enc == "UTF-8") { + return CP_UTF8; + } else if (enc == "UTF-16LE") { + return KAITAI_CP_UTF16LE; + } else if (enc == "UTF-16BE") { + return KAITAI_CP_UTF16BE; + } else if (enc == "IBM437") { + return 437; + } else if (enc == "IBM850") { + return 850; + } else if (enc == "SHIFT_JIS") { + return 932; + } else if (enc == "GB2312") { + return 936; + } else if (enc == "ASCII") { + return 20127; + } else if (enc == "EUC-JP") { + return 20932; + } else if (enc == "ISO-8859-1") { + return 28591; + } else if (enc == "ISO-8859-2") { + return 28592; + } else if (enc == "ISO-8859-3") { + return 28593; + } else if (enc == "ISO-8859-4") { + return 28594; + } else if (enc == "ISO-8859-5") { + return 28595; + } else if (enc == "ISO-8859-6") { + return 28596; + } else if (enc == "ISO-8859-7") { + return 28597; + } else if (enc == "ISO-8859-8") { + return 28598; + } else if (enc == "ISO-8859-9") { + return 28599; + } else if (enc == "ISO-8859-10") { + return 28600; + } else if (enc == "ISO-8859-11") { + return 28601; + } else if (enc == "ISO-8859-13") { + return 28603; + } else if (enc == "ISO-8859-14") { + return 28604; + } else if (enc == "ISO-8859-15") { + return 28605; + } else if (enc == "ISO-8859-16") { + return 28606; + } + + return KAITAI_CP_UNSUPPORTED; +} + +std::string kaitai::kstream::bytes_to_str(const std::string src, const char *src_enc) { + // Step 1: convert encoding name to codepage number + int codepage = encoding_to_win_codepage(src_enc); + if (codepage == KAITAI_CP_UNSUPPORTED) { + throw unknown_encoding(src_enc); + } + return bytes_to_str(src, codepage); +} + +std::string kaitai::kstream::bytes_to_str(const std::string src, int codepage) { + // Shortcut: if we're already in UTF-8, no need to convert anything + if (codepage == CP_UTF8) { + return src; + } + + // Step 2: convert bytes to UTF-16 ("wide char") string + std::wstring utf16; + int32_t utf16_len; + int32_t src_len; + if (src.length() > std::numeric_limits::max()) { + throw bytes_to_str_error("buffers longer than int32_t are unsupported"); + } else { + src_len = static_cast(src.length()); + } + + switch (codepage) { + case KAITAI_CP_UTF16LE: + // If our source is already UTF-16LE, just copy it + + if (src_len % 2 != 0) { + throw illegal_seq_in_encoding("incomplete"); + } + + utf16_len = src_len / 2; + utf16 = std::wstring((wchar_t*)src.c_str(), utf16_len); + break; + case KAITAI_CP_UTF16BE: + // If our source is in UTF-16BE, convert it to UTF-16LE by swapping bytes + + if (src_len % 2 != 0) { + throw illegal_seq_in_encoding("incomplete"); + } + + utf16_len = src_len / 2; + + utf16 = std::wstring(utf16_len, L'\0'); + for (int32_t i = 0; i < utf16_len; i++) { + utf16[i] = (static_cast(src[i * 2]) << 8) | static_cast(src[i * 2 + 1]); + } + break; + default: + // Calculate the length of the UTF-16 string + utf16_len = MultiByteToWideChar(codepage, 0, src.c_str(), src_len, 0, 0); + if (utf16_len == 0) { + throw bytes_to_str_error("MultiByteToWideChar length calculation error"); + } + + // Convert to UTF-16 string + utf16 = std::wstring(utf16_len, L'\0'); + if (MultiByteToWideChar(codepage, MB_ERR_INVALID_CHARS, src.c_str(), src_len, &utf16[0], utf16_len) == 0) { + auto err = GetLastError(); + if (err == ERROR_NO_UNICODE_TRANSLATION) { + throw illegal_seq_in_encoding("MultiByteToWideChar"); + } else { + throw bytes_to_str_error("MultiByteToWideChar conversion error"); + } + } + } + + // Step 3: convert UTF-16 string to UTF-8 string + + // Calculate the length of the UTF-8 string + int utf8_len = WideCharToMultiByte(CP_UTF8, 0, &utf16[0], utf16_len, 0, 0, 0, 0); + if (utf8_len == 0) { + throw bytes_to_str_error("WideCharToMultiByte length calculation error"); + } + + // Convert to UTF-8 string + std::string utf8(utf8_len, '\0'); + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, &utf16[0], utf16_len, &utf8[0], utf8_len, 0, 0) == 0) { + auto err = GetLastError(); + if (err == ERROR_NO_UNICODE_TRANSLATION) { + throw illegal_seq_in_encoding("WideCharToMultiByte"); + } else { + throw bytes_to_str_error("WideCharToMultiByte conversion error"); + } + } + + return utf8; +} + #else -#error Need to decide how to handle strings: please define one of: KS_STR_ENCODING_ICONV, KS_STR_ENCODING_NONE -#endif +#error Need to decide how to handle strings: please define one of: KS_STR_ENCODING_ICONV, KS_STR_ENCODING_WIN32API, KS_STR_ENCODING_NONE +#endif \ No newline at end of file diff --git a/src/kaitai/structs/bnk.cpp b/src/kaitai/structs/bnk.cpp index af74b5e..cc58397 100644 --- a/src/kaitai/structs/bnk.cpp +++ b/src/kaitai/structs/bnk.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "kaitai/structs/bnk.h" +#include "bnk.h" #include "kaitai/exceptions.h" bnk_t::bnk_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, bnk_t* p__root) : kaitai::kstruct(p__io) { @@ -136,9 +136,8 @@ bnk_t::data_obj_section_t::data_obj_section_t(uint32_t p_length, kaitai::kstream } void bnk_t::data_obj_section_t::_read() { - int l_data = _parent()->didx_data()->num_files(); m_data = new std::vector(); - m_data->reserve(l_data); + const int l_data = _parent()->didx_data()->num_files(); for (int i = 0; i < l_data; i++) { m_data->push_back(new data_obj_t(_parent()->didx_data()->objs()->at(i)->offset(), _parent()->didx_data()->objs()->at(i)->length(), m__io, this, m__root)); } @@ -198,9 +197,8 @@ bnk_t::hirc_data_t::hirc_data_t(uint32_t p_length, kaitai::kstream* p__io, bnk_t void bnk_t::hirc_data_t::_read() { m_num_objects = m__io->read_u4le(); - int l_objs = num_objects(); m_objs = new std::vector(); - m_objs->reserve(l_objs); + const int l_objs = num_objects(); for (int i = 0; i < l_objs; i++) { m_objs->push_back(new hirc_obj_t(m__io, this, m__root)); } @@ -235,15 +233,7 @@ bnk_t::bkhd_data_t::bkhd_data_t(uint32_t p_length, kaitai::kstream* p__io, bnk_t void bnk_t::bkhd_data_t::_read() { m_version = m__io->read_u4le(); m_id = m__io->read_u4le(); - m_blank1 = m__io->read_bytes(4); - if (!(blank1() == std::string("\x00\x00\x00\x00", 4))) { - throw kaitai::validation_not_equal_error(std::string("\x00\x00\x00\x00", 4), blank1(), _io(), std::string("/types/bkhd_data/seq/2")); - } - m_blank2 = m__io->read_bytes(4); - if (!(blank2() == std::string("\x00\x00\x00\x00", 4))) { - throw kaitai::validation_not_equal_error(std::string("\x00\x00\x00\x00", 4), blank2(), _io(), std::string("/types/bkhd_data/seq/3")); - } - m_random = m__io->read_bytes((length() - 16)); + m_random = m__io->read_bytes((length() - 8)); } bnk_t::bkhd_data_t::~bkhd_data_t() { @@ -253,12 +243,13 @@ bnk_t::bkhd_data_t::~bkhd_data_t() { void bnk_t::bkhd_data_t::_clean_up() { } -uint32_t bnk_t::bkhd_data_t::version() const { return m_version; } - bnk_t::event_t::event_t(kaitai::kstream* p__io, bnk_t::hirc_obj_t* p__parent, bnk_t* p__root) : kaitai::kstruct(p__io) { m__parent = p__parent; m__root = p__root; + m_event_action_count_new = 0; m_event_actions = 0; + f_version = false; + f_event_action_count_value = false; try { _read(); @@ -269,10 +260,18 @@ bnk_t::event_t::event_t(kaitai::kstream* p__io, bnk_t::hirc_obj_t* p__parent, bn } void bnk_t::event_t::_read() { - m_event_action_count = m__io->read_u4le(); - int l_event_actions = event_action_count(); + n_event_action_count_new = true; + if (version() >= 123) { + n_event_action_count_new = false; + m_event_action_count_new = new vlq_t(m__io); + } + n_event_action_count_old = true; + if (version() < 123) { + n_event_action_count_old = false; + m_event_action_count_old = m__io->read_u4le(); + } m_event_actions = new std::vector(); - m_event_actions->reserve(l_event_actions); + const int l_event_actions = event_action_count_value(); for (int i = 0; i < l_event_actions; i++) { m_event_actions->push_back(m__io->read_u4le()); } @@ -283,11 +282,34 @@ bnk_t::event_t::~event_t() { } void bnk_t::event_t::_clean_up() { + if (!n_event_action_count_new) { + if (m_event_action_count_new) { + delete m_event_action_count_new; m_event_action_count_new = 0; + } + } + if (!n_event_action_count_old) { + } if (m_event_actions) { delete m_event_actions; m_event_actions = 0; } } +uint32_t bnk_t::event_t::version() { + if (f_version) + return m_version; + m_version = static_cast(_root()->data()->at(0)->section_data())->version(); + f_version = true; + return m_version; +} + +int32_t bnk_t::event_t::event_action_count_value() { + if (f_event_action_count_value) + return m_event_action_count_value; + m_event_action_count_value = ((version() >= 123) ? (static_cast(event_action_count_new())->value()) : (static_cast(event_action_count_old()))); + f_event_action_count_value = true; + return m_event_action_count_value; +} + bnk_t::ss_state_group_t::ss_state_group_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, bnk_t* p__root) : kaitai::kstruct(p__io) { m__parent = p__parent; m__root = p__root; @@ -305,9 +327,8 @@ void bnk_t::ss_state_group_t::_read() { m_id = m__io->read_u4le(); m_change_occurs_at = m__io->read_s1(); m_settings_vary_from_default = m__io->read_u2le(); - int l_state_obj = settings_vary_from_default(); m_state_obj = new std::vector(); - m_state_obj->reserve(l_state_obj); + const int l_state_obj = settings_vary_from_default(); for (int i = 0; i < l_state_obj; i++) { m_state_obj->push_back(new ss_state_obj_t(m__io, this, m__root)); } @@ -357,9 +378,8 @@ void bnk_t::sound_structure_t::_read() { n_effect = true; if (effects_count() > 0) { n_effect = false; - int l_effect = effects_count(); m_effect = new std::vector(); - m_effect->reserve(l_effect); + const int l_effect = effects_count(); for (int i = 0; i < l_effect; i++) { m_effect->push_back(new ss_effect_t(m__io, this, m__root)); } @@ -370,25 +390,23 @@ void bnk_t::sound_structure_t::_read() { m_offset_priority_enabled = m__io->read_bits_int_be(1); m__io->align_to_byte(); m_additional_parameter_count = m__io->read_s1(); - int l_parameter_type = additional_parameter_count(); m_parameter_type = new std::vector(); - m_parameter_type->reserve(l_parameter_type); + const int l_parameter_type = additional_parameter_count(); for (int i = 0; i < l_parameter_type; i++) { m_parameter_type->push_back(m__io->read_s1()); } - int l_parameter_value = additional_parameter_count(); m_parameter_value = new std::vector(); - m_parameter_value->reserve(l_parameter_value); + const int l_parameter_value = additional_parameter_count(); for (int i = 0; i < l_parameter_value; i++) { switch (parameter_type()->at(i)) { - case 7: { - m_parameter_value->push_back(m__io->read_u4le()); - break; - } - default: { - m_parameter_value->push_back(m__io->read_f4le()); - break; - } + case 7: { + m_parameter_value->push_back(m__io->read_u4le()); + break; + } + default: { + m_parameter_value->push_back(m__io->read_f4le()); + break; + } } } m_blank1 = m__io->read_bytes(1); @@ -416,9 +434,8 @@ void bnk_t::sound_structure_t::_read() { n_aux_busses = true; if (user_aux_sends_exist()) { n_aux_busses = false; - int l_aux_busses = 4; m_aux_busses = new std::vector(); - m_aux_busses->reserve(l_aux_busses); + const int l_aux_busses = 4; for (int i = 0; i < l_aux_busses; i++) { m_aux_busses->push_back(m__io->read_u4le()); } @@ -446,9 +463,8 @@ void bnk_t::sound_structure_t::_read() { m_parent_virt_voice_settings_override = m__io->read_bits_int_be(1); m__io->align_to_byte(); m_state_group_count = m__io->read_u4le(); - int l_state_group = state_group_count(); m_state_group = new std::vector(); - m_state_group->reserve(l_state_group); + const int l_state_group = state_group_count(); for (int i = 0; i < l_state_group; i++) { m_state_group->push_back(new ss_state_group_t(m__io, this, m__root)); } @@ -522,7 +538,7 @@ bnk_t::stid_obj_t::stid_obj_t(kaitai::kstream* p__io, bnk_t::stid_data_t* p__par void bnk_t::stid_obj_t::_read() { m_id = m__io->read_u4le(); m_name_len = m__io->read_u1(); - m_name = kaitai::kstream::bytes_to_str(m__io->read_bytes(name_len()), std::string("utf-8")); + m_name = kaitai::kstream::bytes_to_str(m__io->read_bytes(name_len()), "UTF-8"); } bnk_t::stid_obj_t::~stid_obj_t() { @@ -548,9 +564,8 @@ bnk_t::didx_data_t::didx_data_t(uint32_t p_length, kaitai::kstream* p__io, bnk_t } void bnk_t::didx_data_t::_read() { - int l_objs = num_files(); m_objs = new std::vector(); - m_objs->reserve(l_objs); + const int l_objs = num_files(); for (int i = 0; i < l_objs; i++) { m_objs->push_back(new didx_obj_t(m__io, this, m__root)); } @@ -597,9 +612,8 @@ void bnk_t::ss_rtpc_t::_read() { m_unk_1 = m__io->read_bytes(1); m_num_points = m__io->read_s1(); m_unk_2 = m__io->read_bytes(1); - int l_point = num_points(); m_point = new std::vector(); - m_point->reserve(l_point); + const int l_point = num_points(); for (int i = 0; i < l_point; i++) { m_point->push_back(new ss_rtpc_point_t(m__io, this, m__root)); } @@ -695,26 +709,26 @@ void bnk_t::hirc_obj_t::_read() { m_length = m__io->read_u4le(); m_id = m__io->read_u4le(); switch (type()) { - case bnk_t::OBJECT_TYPE_SOUND_EFFECT_OR_VOICE: { - m_object_data = new sound_effect_or_voice_t(m__io, this, m__root); - break; - } - case bnk_t::OBJECT_TYPE_EVENT_ACTION: { - m_object_data = new event_action_t(m__io, this, m__root); - break; - } - case bnk_t::OBJECT_TYPE_EVENT: { - m_object_data = new event_t(m__io, this, m__root); - break; - } - case bnk_t::OBJECT_TYPE_SETTINGS: { - m_object_data = new settings_t(m__io, this, m__root); - break; - } - default: { - m_object_data = new random_bytes_t((length() - 4), m__io, this, m__root); - break; - } + case bnk_t::OBJECT_TYPE_SOUND_EFFECT_OR_VOICE: { + m_object_data = new sound_effect_or_voice_t(m__io, this, m__root); + break; + } + case bnk_t::OBJECT_TYPE_EVENT_ACTION: { + m_object_data = new event_action_t(m__io, this, m__root); + break; + } + case bnk_t::OBJECT_TYPE_EVENT: { + m_object_data = new event_t(m__io, this, m__root); + break; + } + case bnk_t::OBJECT_TYPE_SETTINGS: { + m_object_data = new settings_t(m__io, this, m__root); + break; + } + default: { + m_object_data = new random_bytes_t((length() - 4), m__io, this, m__root); + break; + } } } @@ -741,7 +755,7 @@ bnk_t::section_t::section_t(kaitai::kstream* p__io, bnk_t* p__parent, bnk_t* p__ } void bnk_t::section_t::_read() { - m_type = kaitai::kstream::bytes_to_str(m__io->read_bytes(4), std::string("utf-8")); + m_type = kaitai::kstream::bytes_to_str(m__io->read_bytes(4), "UTF-8"); m_length = m__io->read_u4le(); { std::string on = type(); @@ -793,15 +807,13 @@ bnk_t::audio_bus_t::audio_bus_t(kaitai::kstream* p__io, kaitai::kstruct* p__pare void bnk_t::audio_bus_t::_read() { m_parent_audio_bus_id = m__io->read_u4le(); m_additional_parameter_count = m__io->read_s1(); - int l_parameter_type = additional_parameter_count(); m_parameter_type = new std::vector(); - m_parameter_type->reserve(l_parameter_type); + const int l_parameter_type = additional_parameter_count(); for (int i = 0; i < l_parameter_type; i++) { m_parameter_type->push_back(static_cast(m__io->read_s1())); } - int l_parameter_value = additional_parameter_count(); m_parameter_value = new std::vector(); - m_parameter_value->reserve(l_parameter_value); + const int l_parameter_value = additional_parameter_count(); for (int i = 0; i < l_parameter_value; i++) { m_parameter_value->push_back(m__io->read_f4le()); } @@ -848,9 +860,8 @@ void bnk_t::stid_data_t::_read() { throw kaitai::validation_not_equal_error(std::string("\x01\x00\x00\x00", 4), unk1(), _io(), std::string("/types/stid_data/seq/0")); } m_num_soundbanks = m__io->read_u4le(); - int l_objs = num_soundbanks(); m_objs = new std::vector(); - m_objs->reserve(l_objs); + const int l_objs = num_soundbanks(); for (int i = 0; i < l_objs; i++) { m_objs->push_back(new stid_obj_t(m__io, this, m__root)); } @@ -935,9 +946,8 @@ bnk_t::music_segment_t::music_segment_t(kaitai::kstream* p__io, kaitai::kstruct* void bnk_t::music_segment_t::_read() { m_sound_structure = new sound_structure_t(m__io, this, m__root); m_child_obj_count = m__io->read_u4le(); - int l_child_obj = child_obj_count(); m_child_obj = new std::vector(); - m_child_obj->reserve(l_child_obj); + const int l_child_obj = child_obj_count(); for (int i = 0; i < l_child_obj; i++) { m_child_obj->push_back(m__io->read_u4le()); } @@ -1043,15 +1053,13 @@ void bnk_t::event_action_t::_read() { throw kaitai::validation_not_equal_error(std::string("\x00", 1), blank1(), _io(), std::string("/types/event_action/seq/3")); } m_parameter_count = m__io->read_s1(); - int l_parameter_type = parameter_count(); m_parameter_type = new std::vector(); - m_parameter_type->reserve(l_parameter_type); + const int l_parameter_type = parameter_count(); for (int i = 0; i < l_parameter_type; i++) { m_parameter_type->push_back(m__io->read_bytes(1)); } - int l_parameter_value = parameter_count(); m_parameter_value = new std::vector(); - m_parameter_value->reserve(l_parameter_value); + const int l_parameter_value = parameter_count(); for (int i = 0; i < l_parameter_value; i++) { { std::string on = parameter_type()->at(i); @@ -1205,15 +1213,13 @@ bnk_t::settings_t::settings_t(kaitai::kstream* p__io, bnk_t::hirc_obj_t* p__pare void bnk_t::settings_t::_read() { m_settings_count = m__io->read_s1(); - int l_setting_type = settings_count(); m_setting_type = new std::vector(); - m_setting_type->reserve(l_setting_type); + const int l_setting_type = settings_count(); for (int i = 0; i < l_setting_type; i++) { m_setting_type->push_back(m__io->read_bytes(1)); } - int l_setting_val = settings_count(); m_setting_val = new std::vector(); - m_setting_val->reserve(l_setting_val); + const int l_setting_val = settings_count(); for (int i = 0; i < l_setting_val; i++) { m_setting_val->push_back(m__io->read_f4le()); } diff --git a/src/kaitai/structs/vlq.cpp b/src/kaitai/structs/vlq.cpp new file mode 100644 index 0000000..542dfb2 --- /dev/null +++ b/src/kaitai/structs/vlq.cpp @@ -0,0 +1,119 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +#include "vlq.h" + +vlq_t::vlq_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, vlq_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = this; + m_groups = 0; + f_len = false; + f_value = false; + f_sign_bit = false; + f_value_signed = false; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void vlq_t::_read() { + m_groups = new std::vector(); + { + int i = 0; + group_t* _; + do { + _ = new group_t(m__io, this, m__root); + m_groups->push_back(_); + i++; + } while (!(!(_->has_next()))); + } +} + +vlq_t::~vlq_t() { + _clean_up(); +} + +void vlq_t::_clean_up() { + if (m_groups) { + for (std::vector::iterator it = m_groups->begin(); it != m_groups->end(); ++it) { + delete *it; + } + delete m_groups; m_groups = 0; + } +} + +vlq_t::group_t::group_t(kaitai::kstream* p__io, vlq_t* p__parent, vlq_t* p__root) : kaitai::kstruct(p__io) { + m__parent = p__parent; + m__root = p__root; + f_has_next = false; + f_value = false; + + try { + _read(); + } catch(...) { + _clean_up(); + throw; + } +} + +void vlq_t::group_t::_read() { + m_b = m__io->read_u1(); +} + +vlq_t::group_t::~group_t() { + _clean_up(); +} + +void vlq_t::group_t::_clean_up() { +} + +bool vlq_t::group_t::has_next() { + if (f_has_next) + return m_has_next; + m_has_next = (b() & 128) != 0; + f_has_next = true; + return m_has_next; +} + +int32_t vlq_t::group_t::value() { + if (f_value) + return m_value; + m_value = (b() & 127); + f_value = true; + return m_value; +} + +int32_t vlq_t::len() { + if (f_len) + return m_len; + m_len = groups()->size(); + f_len = true; + return m_len; +} + +int32_t vlq_t::value() { + if (f_value) + return m_value; + m_value = (((((((groups()->at(0)->value() + ((len() >= 2) ? ((groups()->at(1)->value() << 7)) : (0))) + ((len() >= 3) ? ((groups()->at(2)->value() << 14)) : (0))) + ((len() >= 4) ? ((groups()->at(3)->value() << 21)) : (0))) + ((len() >= 5) ? ((groups()->at(4)->value() << 28)) : (0))) + ((len() >= 6) ? ((groups()->at(5)->value() << 35)) : (0))) + ((len() >= 7) ? ((groups()->at(6)->value() << 42)) : (0))) + ((len() >= 8) ? ((groups()->at(7)->value() << 49)) : (0))); + f_value = true; + return m_value; +} + +int32_t vlq_t::sign_bit() { + if (f_sign_bit) + return m_sign_bit; + m_sign_bit = (1 << ((7 * len()) - 1)); + f_sign_bit = true; + return m_sign_bit; +} + +int32_t vlq_t::value_signed() { + if (f_value_signed) + return m_value_signed; + m_value_signed = ((value() ^ sign_bit()) - sign_bit()); + f_value_signed = true; + return m_value_signed; +} From 1364b4ecc8014577408a07b36bfd8a22171238c8 Mon Sep 17 00:00:00 2001 From: Abheek Dhawan Date: Wed, 17 Jan 2024 13:41:10 -0600 Subject: [PATCH 5/8] fix: update includes for new source files --- src/kaitai/structs/bnk.cpp | 2 +- src/kaitai/structs/vlq.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kaitai/structs/bnk.cpp b/src/kaitai/structs/bnk.cpp index cc58397..6987410 100644 --- a/src/kaitai/structs/bnk.cpp +++ b/src/kaitai/structs/bnk.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "bnk.h" +#include "kaitai/structs/bnk.h" #include "kaitai/exceptions.h" bnk_t::bnk_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, bnk_t* p__root) : kaitai::kstruct(p__io) { diff --git a/src/kaitai/structs/vlq.cpp b/src/kaitai/structs/vlq.cpp index 542dfb2..3ca2af8 100644 --- a/src/kaitai/structs/vlq.cpp +++ b/src/kaitai/structs/vlq.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "vlq.h" +#include "kaitai/structs/vlq.h" vlq_t::vlq_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, vlq_t* p__root) : kaitai::kstruct(p__io) { m__parent = p__parent; From 71d68b8a2337131dc4185603e0bef2a071de1997 Mon Sep 17 00:00:00 2001 From: Abheek Dhawan Date: Wed, 17 Jan 2024 14:17:08 -0600 Subject: [PATCH 6/8] fix: regenerate cpp parsers for other kaitai structs --- include/kaitai/structs/w3sc.h | 4 ++-- include/kaitai/structs/wem.h | 4 ++-- src/kaitai/structs/w3sc.cpp | 7 +++---- src/kaitai/structs/wem.cpp | 13 ++++++------- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/include/kaitai/structs/w3sc.h b/include/kaitai/structs/w3sc.h index a2d9101..86c13a6 100644 --- a/include/kaitai/structs/w3sc.h +++ b/include/kaitai/structs/w3sc.h @@ -7,8 +7,8 @@ #include #include -#if KAITAI_STRUCT_VERSION < 9000L -#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#if KAITAI_STRUCT_VERSION < 11000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.11 or later is required" #endif class w3sc_t : public kaitai::kstruct { diff --git a/include/kaitai/structs/wem.h b/include/kaitai/structs/wem.h index 36b1dac..9334e3d 100644 --- a/include/kaitai/structs/wem.h +++ b/include/kaitai/structs/wem.h @@ -7,8 +7,8 @@ #include #include -#if KAITAI_STRUCT_VERSION < 9000L -#error "Incompatible Kaitai Struct C++/STL API: version 0.9 or later is required" +#if KAITAI_STRUCT_VERSION < 11000L +#error "Incompatible Kaitai Struct C++/STL API: version 0.11 or later is required" #endif class wem_t : public kaitai::kstruct { diff --git a/src/kaitai/structs/w3sc.cpp b/src/kaitai/structs/w3sc.cpp index db5aace..c356b0a 100644 --- a/src/kaitai/structs/w3sc.cpp +++ b/src/kaitai/structs/w3sc.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "kaitai/structs/w3sc.h" +#include "w3sc.h" #include "kaitai/exceptions.h" w3sc_t::w3sc_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, w3sc_t* p__root) : kaitai::kstruct(p__io) { @@ -155,7 +155,7 @@ std::string w3sc_t::file_info_t::name() { return m_name; std::streampos _pos = m__io->pos(); m__io->seek((_parent()->names_offset() + name_offset())); - m_name = kaitai::kstream::bytes_to_str(m__io->read_bytes_term(0, false, true, true), std::string("UTF-8")); + m_name = kaitai::kstream::bytes_to_str(m__io->read_bytes_term(0, false, true, true), "UTF-8"); m__io->seek(_pos); f_name = true; return m_name; @@ -166,9 +166,8 @@ std::vector* w3sc_t::file_infos() { return m_file_infos; std::streampos _pos = m__io->pos(); m__io->seek(info_offset()); - int l_file_infos = files(); m_file_infos = new std::vector(); - m_file_infos->reserve(l_file_infos); + const int l_file_infos = files(); for (int i = 0; i < l_file_infos; i++) { m_file_infos->push_back(new file_info_t(m__io, this, m__root)); } diff --git a/src/kaitai/structs/wem.cpp b/src/kaitai/structs/wem.cpp index 79a368a..6b2b845 100644 --- a/src/kaitai/structs/wem.cpp +++ b/src/kaitai/structs/wem.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "kaitai/structs/wem.h" +#include "wem.h" #include "kaitai/exceptions.h" wem_t::wem_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, wem_t* p__root) : kaitai::kstruct(p__io) { @@ -127,7 +127,7 @@ void wem_t::fmt_chunk_t::_read() { m_extra_byte_count = m__io->read_u2le(); m_valid_bits_per_sample = m__io->read_u2le(); m_channel_mask = m__io->read_u4le(); - m_guid = kaitai::kstream::bytes_to_str(m__io->read_bytes(42), std::string("utf-8")); + m_guid = kaitai::kstream::bytes_to_str(m__io->read_bytes(42), "UTF-8"); } wem_t::fmt_chunk_t::~fmt_chunk_t() { @@ -150,7 +150,7 @@ wem_t::junk_chunk_t::junk_chunk_t(kaitai::kstream* p__io, wem_t::chunk_t* p__par } void wem_t::junk_chunk_t::_read() { - m_junk = kaitai::kstream::bytes_to_str(m__io->read_bytes(26), std::string("utf-8")); + m_junk = kaitai::kstream::bytes_to_str(m__io->read_bytes(26), "UTF-8"); } wem_t::junk_chunk_t::~junk_chunk_t() { @@ -179,7 +179,7 @@ void wem_t::list_labl_subchunk_t::_read() { } m_size = m__io->read_u4le(); m_cue_point_id = m__io->read_u4le(); - m_data = kaitai::kstream::bytes_to_str(m__io->read_bytes((size() - 4)), std::string("utf-8")); + m_data = kaitai::kstream::bytes_to_str(m__io->read_bytes((size() - 4)), "UTF-8"); } wem_t::list_labl_subchunk_t::~list_labl_subchunk_t() { @@ -263,7 +263,7 @@ wem_t::data_chunk_t::data_chunk_t(uint32_t p_size, kaitai::kstream* p__io, wem_t } void wem_t::data_chunk_t::_read() { - m_data = kaitai::kstream::bytes_to_str(m__io->read_bytes(size()), std::string("utf-8")); + m_data = kaitai::kstream::bytes_to_str(m__io->read_bytes(size()), "UTF-8"); } wem_t::data_chunk_t::~data_chunk_t() { @@ -288,9 +288,8 @@ wem_t::cue_chunk_t::cue_chunk_t(kaitai::kstream* p__io, wem_t::chunk_t* p__paren void wem_t::cue_chunk_t::_read() { m_cue_point_count = m__io->read_u4le(); - int l_cue_points = cue_point_count(); m_cue_points = new std::vector(); - m_cue_points->reserve(l_cue_points); + const int l_cue_points = cue_point_count(); for (int i = 0; i < l_cue_points; i++) { m_cue_points->push_back(new cue_point_subchunk_t(m__io, this, m__root)); } From a6eaaf4c548649c22a1a89646c6678c2fb20fe44 Mon Sep 17 00:00:00 2001 From: Abheek Dhawan Date: Wed, 17 Jan 2024 14:24:35 -0600 Subject: [PATCH 7/8] fix: update include subdirectories for kaitai struct cpp sources --- src/kaitai/structs/w3sc.cpp | 2 +- src/kaitai/structs/wem.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kaitai/structs/w3sc.cpp b/src/kaitai/structs/w3sc.cpp index c356b0a..edb9c08 100644 --- a/src/kaitai/structs/w3sc.cpp +++ b/src/kaitai/structs/w3sc.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "w3sc.h" +#include "kaitai/structs/w3sc.h" #include "kaitai/exceptions.h" w3sc_t::w3sc_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, w3sc_t* p__root) : kaitai::kstruct(p__io) { diff --git a/src/kaitai/structs/wem.cpp b/src/kaitai/structs/wem.cpp index 6b2b845..288c35a 100644 --- a/src/kaitai/structs/wem.cpp +++ b/src/kaitai/structs/wem.cpp @@ -1,6 +1,6 @@ // This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -#include "wem.h" +#include "kaitai/structs/wem.h" #include "kaitai/exceptions.h" wem_t::wem_t(kaitai::kstream* p__io, kaitai::kstruct* p__parent, wem_t* p__root) : kaitai::kstruct(p__io) { From e181c0d64fb378879f959a3c316283a98bb5486f Mon Sep 17 00:00:00 2001 From: Abheek Dhawan <67982792+abheekda1@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:50:21 -0600 Subject: [PATCH 8/8] fix: add vlq to compile source --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6851cd..0c66ab7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ set(SOURCE src/wwtools/bnk.cpp src/kaitai/kaitaistream.cpp src/kaitai/structs/bnk.cpp + src/kaitai/structs/vlq.cpp src/kaitai/structs/w3sc.cpp src/kaitai/structs/wem.cpp )