From 9031a151c58826da705bf2c0ad7ec0f3673fe439 Mon Sep 17 00:00:00 2001 From: Andrii Mamchur Date: Sun, 10 Nov 2013 19:09:19 +0200 Subject: [PATCH] Improvement hex parsing performance --- .../iPhoneJsonLiteViewController.m | 10 +- jsonlite/include/jsonlite_token.h | 19 +- jsonlite/src/jsonlite_parser.c | 231 +++++++++--------- jsonlite/src/jsonlite_token.c | 69 ++++-- objc/JsonLiteParser.m | 44 +--- tests/JsonLiteObjCInputs.m | 45 +--- tests/JsonLiteObjCTokenTests.m | 19 -- tests/JsonLiteObjCUnicodeTests.m | 6 +- 8 files changed, 194 insertions(+), 249 deletions(-) diff --git a/Examples/iPhoneJsonLite/iPhoneJsonLite/iPhoneJsonLiteViewController.m b/Examples/iPhoneJsonLite/iPhoneJsonLite/iPhoneJsonLiteViewController.m index c38e982..918d4d8 100644 --- a/Examples/iPhoneJsonLite/iPhoneJsonLite/iPhoneJsonLiteViewController.m +++ b/Examples/iPhoneJsonLite/iPhoneJsonLite/iPhoneJsonLiteViewController.m @@ -63,7 +63,7 @@ - (void)didReceiveMemoryWarning // Implement viewDidLoad to do additional setup after loading the view, typically from a nib. - (void)viewDidLoad { - NSString *filePath = [[NSBundle mainBundle] pathForResource:@"mesh" ofType:@"json"]; + NSString *filePath = [[NSBundle mainBundle] pathForResource:@"twitter_public_timeline" ofType:@"json"]; self.data = [NSData dataWithContentsOfFile:filePath]; [super viewDidLoad]; } @@ -75,7 +75,7 @@ - (IBAction)parse:(id)sender { clock_get_time(cclock, &start); size_t l = [data length]; - const int COUNT = 100; + const int COUNT = 1000; for (int i = 0; i < COUNT; i++) { @autoreleasepool { JsonLiteParser *parser = [JsonLiteParser parserWithDepth:512]; @@ -91,7 +91,7 @@ - (IBAction)parse:(id)sender { time += (double)(end.tv_nsec - start.tv_nsec) / 1000000000; double speed = (double)l * COUNT / 1024 / 1024 / time; - NSString *jsonlite = [NSString stringWithFormat:@"Time - %f µs\nSpeed - %.2f MBps", time, speed]; + NSString *jsonlite = [NSString stringWithFormat:@"Time - %f s\nSpeed - %.2f MBps", time, speed]; UIAlertView *av = [[UIAlertView alloc] initWithTitle:@"" message:jsonlite delegate:nil @@ -112,7 +112,7 @@ - (IBAction)parse1:(id)sender { host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &cclock); clock_get_time(cclock, &start); - const int COUNT = 100; + const int COUNT = 1000; char parser[1024]; for (int i = 0; i < COUNT; i++) { jsonlite_parser p = jsonlite_parser_init_memory(parser, sizeof(parser)); @@ -126,7 +126,7 @@ - (IBAction)parse1:(id)sender { time += (double)(end.tv_nsec - start.tv_nsec) / 1000000000; double speed = (double)l * COUNT / 1024 / 1024 / time; - NSString *jsonlite = [NSString stringWithFormat:@"Time - %f µs\nSpeed - %.2f MBps", time, speed]; + NSString *jsonlite = [NSString stringWithFormat:@"Time - %f s\nSpeed - %.2f MBps", time, speed]; UIAlertView *av = [[UIAlertView alloc] initWithTitle:@"" message:jsonlite delegate:nil diff --git a/jsonlite/include/jsonlite_token.h b/jsonlite/include/jsonlite_token.h index 76d9efa..c0a6fa8 100644 --- a/jsonlite/include/jsonlite_token.h +++ b/jsonlite/include/jsonlite_token.h @@ -161,7 +161,7 @@ extern "C" { * @param ts jsonlite token * @return 0 if ts is NULL; otherwise required size of for token conversion. */ - size_t jsonlite_token_decode_size_for_uft8(jsonlite_token *ts); + size_t jsonlite_token_size_of_uft8(jsonlite_token *ts); /** @brief Converts specified token to UTF-8 string. * @@ -172,13 +172,13 @@ extern "C" { * @param ts jsonlite token * @return length in bytes of converted string. */ - size_t jsonlite_token_decode_to_uft8(jsonlite_token *ts, uint8_t **buffer); + size_t jsonlite_token_to_uft8(jsonlite_token *ts, uint8_t **buffer); /** @brief Returns a size of memory that is required for token conversion to UTF-16 string. * @param ts jsonlite token * @return 0 if ts is NULL; otherwise required size of for token conversion. */ - size_t jsonlite_token_decode_size_for_uft16(jsonlite_token *ts); + size_t jsonlite_token_size_of_uft16(jsonlite_token *ts); /** @brief Converts specified token to UTF-16 string. * @@ -189,15 +189,10 @@ extern "C" { * @param ts jsonlite token * @return length in bytes of converted string. */ - size_t jsonlite_token_decode_to_uft16(jsonlite_token *ts, uint16_t **buffer); - - /** @brief Converts hex digit to integer value. - * - * @param c a ASCII character. - * @return integer value of hex character, - * if character belongs to set [0,1,2,3,4,5,6,7,8,9,A,B,C,D,E,F,a,b,c,d,e,f]; otherwise 0xFF. - */ - uint8_t jsonlite_hex_char_to_uint8(uint8_t c); + size_t jsonlite_token_to_uft16(jsonlite_token *ts, uint16_t **buffer); + + long jsonlite_token_to_long(jsonlite_token *token); + long long jsonlite_token_to_long_long(jsonlite_token *token); #ifdef __cplusplus } diff --git a/jsonlite/src/jsonlite_parser.c b/jsonlite/src/jsonlite_parser.c index f5c7382..011acaa 100644 --- a/jsonlite/src/jsonlite_parser.c +++ b/jsonlite/src/jsonlite_parser.c @@ -40,20 +40,24 @@ static uint32_t __inline jsonlite_clz(uint32_t x) { #define CALL_VALUE_CALLBACK(cbs, type, token) (cbs.type(&cbs.context, token)) #define CALL_STATE_CALLBACK(cbs, type) (cbs.type(&cbs.context)) -#define CHECK_HEX(c) \ -if ((c) < 48) goto error_escape; \ -if ((c) > 102) goto error_escape; \ -if (57 < (c) && (c) < 65) goto error_escape; \ -if (70 < (c) && (c) < 97) goto error_escape - -#define CASE_NUMBER_TOKEN_END \ -case 9: goto number_parsed; \ -case 10: goto number_parsed; \ -case 13: goto number_parsed; \ -case 32: goto number_parsed; \ -case 44: goto number_parsed; \ -case 93: goto number_parsed; \ -case 125: goto number_parsed +#define HEX_CHAR_TO_INT(c, step) \ +if (0x30 <= *c && *c <= 0x39) { \ + hex_value = *c - 0x30; \ +} else if (0x41 <= *c && *c <= 0x46) { \ + hex_value = *c - 0x37; \ +} else if (0x61 <= *c && *c <= 0x66) { \ + hex_value = *c - 0x57; \ +} else goto error_escape; \ +c += step + +#define CASE_NUMBER_TOKEN_END \ +case 0x09: goto number_parsed; \ +case 0x0A: goto number_parsed; \ +case 0x0D: goto number_parsed; \ +case 0x20: goto number_parsed; \ +case 0x2C: goto number_parsed; \ +case 0x5D: goto number_parsed; \ +case 0x7D: goto number_parsed enum { state_start, @@ -249,6 +253,7 @@ static void jsonlite_do_parse(jsonlite_parser parser) { parse_state *state = parser->current; jsonlite_token token; uint32_t value, utf32; + uint8_t hex_value; *state &= ~state_stop; parser->control = &state; @@ -256,14 +261,14 @@ static void jsonlite_do_parse(jsonlite_parser parser) { structure_finished: if (*state == state_end) goto success; -skip_char_and_spaces: +skip_char_and_whitespaces: c++; select_state: - if (c == l) goto end_of_stream_space; - if (*c == ' ') goto skip_char_and_spaces; - if (*c == '\n') goto skip_char_and_spaces; - if (*c == '\r') goto skip_char_and_spaces; - if (*c == '\t') goto skip_char_and_spaces; + if (c == l) goto end_of_stream_whitespaces; + if (*c == 0x20) goto skip_char_and_whitespaces; + if (*c == 0x0A) goto skip_char_and_whitespaces; + if (*c == 0x0D) goto skip_char_and_whitespaces; + if (*c == 0x09) goto skip_char_and_whitespaces; token_start = c; switch (*state) { @@ -276,8 +281,8 @@ static void jsonlite_do_parse(jsonlite_parser parser) { case state_array_value_end: goto parse_array_value_end; case state_key: goto parse_string_token; case state_start: - if (*c == '{') goto parse_object; - if (*c == '[') goto parse_array_state; + if (*c == 0x7B) goto parse_object; + if (*c == 0x5B) goto parse_array_state; goto error_exp_ooa; case state_end: goto success; default: goto end; @@ -285,29 +290,29 @@ static void jsonlite_do_parse(jsonlite_parser parser) { parse_object: *state = state_object_key_end; CALL_STATE_CALLBACK(parser->callbacks, object_start); - goto skip_char_and_spaces; + goto skip_char_and_whitespaces; parse_array_state: *state = state_array_value_end; CALL_STATE_CALLBACK(parser->callbacks, array_start); - goto skip_char_and_spaces; + goto skip_char_and_whitespaces; parse_key: - if (*c != '"') goto error_exp_key; + if (*c != 0x22) goto error_exp_key; *state = state_colon; *++state = state_key; goto parse_string_token; parse_colon: - if (*c != ':') goto error_exp_colon; + if (*c != 0x3A) goto error_exp_colon; *state = state_object_comma_end; *++state = state_value; - goto skip_char_and_spaces; + goto skip_char_and_whitespaces; parse_key_end: switch (*c) { - case '"': + case 0x22: *state = state_colon; if (++state == last) goto error_depth; *state = state_key; goto parse_string_token; - case '}': + case 0x7D: state--; CALL_STATE_CALLBACK(parser->callbacks, object_end); goto structure_finished; @@ -315,10 +320,10 @@ static void jsonlite_do_parse(jsonlite_parser parser) { } parse_object_comma_end: switch (*c) { - case ',': + case 0x2C: *state = state_object_key; - goto skip_char_and_spaces; - case '}': + goto skip_char_and_whitespaces; + case 0x7D: state--; CALL_STATE_CALLBACK(parser->callbacks, object_end); goto structure_finished; @@ -326,7 +331,7 @@ static void jsonlite_do_parse(jsonlite_parser parser) { } parse_array_value_end: switch (*c) { - case ']': + case 0x5D: state--; CALL_STATE_CALLBACK(parser->callbacks, array_end); goto structure_finished; @@ -338,93 +343,93 @@ static void jsonlite_do_parse(jsonlite_parser parser) { } parse_array_comma_end: switch (*c) { - case ',': + case 0x2C: *++state = state_value; - goto skip_char_and_spaces; - case ']': + goto skip_char_and_whitespaces; + case 0x5D: state--; CALL_STATE_CALLBACK(parser->callbacks, array_end); goto structure_finished; default: goto error_exp_coe; } parse_value: - if ('1' <= *c && *c <= '9') goto parse_digit_leading_number; - if (*c == '0') goto parse_zero_leading_number; - if (*c == '-') goto parse_negative_number; - if (*c == '"') goto parse_string_token; - if (*c == 't') goto parse_true_token; - if (*c == 'f') goto parse_false_token; - if (*c == 'n') goto parse_null_token; - if (*c == '{') goto parse_object; - if (*c == '[') goto parse_array_state; + if (0x31 <= *c && *c <= 0x39) goto parse_digit_leading_number; + if (*c == 0x30) goto parse_zero_leading_number; + if (*c == 0x2D) goto parse_negative_number; + if (*c == 0x22) goto parse_string_token; + if (*c == 0x74) goto parse_true_token; + if (*c == 0x66) goto parse_false_token; + if (*c == 0x6E) goto parse_null_token; + if (*c == 0x7B) goto parse_object; + if (*c == 0x5B) goto parse_array_state; goto error_exp_value; // Number parsing parse_negative_number: token.start = c; token.type.number = jsonlite_number_int | jsonlite_number_negative; - if (++c == l) goto end_of_stream; - if (49 <= *c && *c <= 57) goto parse_digits; - if (*c == 48) goto parse_exponent_fraction; + if (++c == l) goto end_of_stream; + if (0x31 <= *c && *c <= 0x39) goto parse_digits; + if (*c == 0x30) goto parse_exponent_fraction; goto error_number; parse_zero_leading_number: token.start = c; token.type.number = jsonlite_number_int | jsonlite_number_zero_leading; parse_exponent_fraction: - if (++c == l) goto end_of_stream; + if (++c == l) goto end_of_stream; switch (*c) { CASE_NUMBER_TOKEN_END; - case 46: goto parse_fraction; - case 69: goto parse_exponent; - case 101: goto parse_exponent; - default: goto error_number; + case 0x2E: goto parse_fraction; + case 0x45: goto parse_exponent; + case 0x65: goto parse_exponent; + default: goto error_number; } parse_fraction: token.type.number |= jsonlite_number_frac; - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_frac_number; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_frac_number; goto error_number; parse_frac_number: - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_frac_number; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_frac_number; switch (*c) { CASE_NUMBER_TOKEN_END; - case 69: goto parse_exponent; - case 101: goto parse_exponent; - default: goto error_number; + case 0x45: goto parse_exponent; + case 0x65: goto parse_exponent; + default: goto error_number; } parse_exponent: token.type.number |= jsonlite_number_exp; - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_exponent_number; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_exponent_number; switch(*c) { - case 43: goto parse_exponent_sign; - case 45: goto parse_exponent_sign; - default: goto error_number; + case 0x2B: goto parse_exponent_sign; + case 0x2D: goto parse_exponent_sign; + default: goto error_number; } parse_exponent_sign: - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_exponent_number; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_exponent_number; goto error_number; parse_exponent_number: - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_exponent_number; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_exponent_number; switch(*c) { CASE_NUMBER_TOKEN_END; - default: goto error_number; + default: goto error_number; } parse_digit_leading_number: token.start = c; token.type.number = jsonlite_number_int | jsonlite_number_digit_leading; parse_digits: - if (++c == l) goto end_of_stream; - if (48 <= *c && *c <= 57) goto parse_digits; + if (++c == l) goto end_of_stream; + if (0x30 <= *c && *c <= 0x39) goto parse_digits; switch(*c) { CASE_NUMBER_TOKEN_END; - case 46: goto parse_fraction; - case 69: goto parse_exponent; - case 101: goto parse_exponent; - default: goto error_number; + case 0x2E: goto parse_fraction; + case 0x45: goto parse_exponent; + case 0x65: goto parse_exponent; + default: goto error_number; } number_parsed: token.end = c; @@ -438,33 +443,33 @@ static void jsonlite_do_parse(jsonlite_parser parser) { token.start = c + 1; next_char: if (++c == l) goto end_of_stream; - if (*c == '"') goto string_parsed; - if (*c == '\\') goto escaped; - if (*c >= 0x80) goto utf8; + if (*c == 0x22) goto string_parsed; + if (*c == 0x5C) goto escaped; + if (*c > 0x7F) goto utf8; if (*c < 0x20) goto error_token; goto next_char; escaped: token.type.string |= jsonlite_string_escape; if (++c == l) goto end_of_stream; switch (*c) { - case 34: goto next_char; - case 47: goto next_char; - case 92: goto next_char; - case 98: goto next_char; - case 102: goto next_char; - case 110: goto next_char; - case 114: goto next_char; - case 116: goto next_char; - case 117: goto hex; + case 0x22: goto next_char; // " quotation mark U+0022 + case 0x2F: goto next_char; // / solidus U+002F + case 0x5C: goto next_char; // \ reverse solidus U+005C + case 0x62: goto next_char; // b backspace U+0008 + case 0x66: goto next_char; // f form feed U+000C + case 0x6E: goto next_char; // n line feed U+000A + case 0x72: goto next_char; // r carriage return U+000D + case 0x74: goto next_char; // t tab U+0009 + case 0x75: goto hex; // uXXXX U+XXXX default: goto error_escape; } hex: token.type.string |= jsonlite_string_unicode_escape; - if (c++ + 4 >= l) goto end_of_stream; - CHECK_HEX(*c); value = jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c); + if (c++ + 4 >= l) goto end_of_stream; + HEX_CHAR_TO_INT(c, 1); value = hex_value; + HEX_CHAR_TO_INT(c, 1); value = (uint32_t)(value << 4) | hex_value; + HEX_CHAR_TO_INT(c, 1); value = (uint32_t)(value << 4) | hex_value; + HEX_CHAR_TO_INT(c, 0); value = (uint32_t)(value << 4) | hex_value; if ((value & 0xFFFFu) >= 0xFFFEu) token.type.string |= jsonlite_string_unicode_noncharacter; if (value >= 0xFDD0u && value <= 0xFDEFu) token.type.string |= jsonlite_string_unicode_noncharacter; @@ -472,13 +477,13 @@ static void jsonlite_do_parse(jsonlite_parser parser) { // UTF-16 Surrogate utf32 = (value - 0xD800) << 10; - if (c++ + 6 >= l) goto end_of_stream; - if (*c++ != '\\') goto error_escape; - if (*c++ != 'u') goto error_escape; - CHECK_HEX(*c); value = jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c++); - CHECK_HEX(*c); value = (uint32_t)(value << 4) | jsonlite_hex_char_to_uint8(*c); + if (c++ + 6 >= l) goto end_of_stream; + if (*c++ != 0x5C) goto error_escape; + if (*c++ != 0x75) goto error_escape; + HEX_CHAR_TO_INT(c, 1); value = hex_value; + HEX_CHAR_TO_INT(c, 1); value = (uint32_t)(value << 4) | hex_value; + HEX_CHAR_TO_INT(c, 1); value = (uint32_t)(value << 4) | hex_value; + HEX_CHAR_TO_INT(c, 0); value = (uint32_t)(value << 4) | hex_value; if (value < 0xDC00 || value > 0xDFFF) goto error_escape; utf32 += value - 0xDC00 + 0x10000; @@ -507,31 +512,31 @@ static void jsonlite_do_parse(jsonlite_parser parser) { } else { CALL_VALUE_CALLBACK(parser->callbacks, key_found, &token); } - goto skip_char_and_spaces; + goto skip_char_and_whitespaces; // Primitive tokens parse_true_token: if (c++ + 3 >= l) goto end_of_stream; - if (*c++ != 'r') goto error_token; - if (*c++ != 'u') goto error_token; - if (*c++ != 'e') goto error_token; + if (*c++ != 0x72) goto error_token; + if (*c++ != 0x75) goto error_token; + if (*c++ != 0x65) goto error_token; state--; CALL_STATE_CALLBACK(parser->callbacks, true_found); goto select_state; parse_false_token: if (c++ + 4 >= l) goto end_of_stream; - if (*c++ != 'a') goto error_token; - if (*c++ != 'l') goto error_token; - if (*c++ != 's') goto error_token; - if (*c++ != 'e') goto error_token; + if (*c++ != 0x61) goto error_token; + if (*c++ != 0x6C) goto error_token; + if (*c++ != 0x73) goto error_token; + if (*c++ != 0x65) goto error_token; state--; CALL_STATE_CALLBACK(parser->callbacks, false_found); goto select_state; parse_null_token: if (c++ + 3 >= l) goto end_of_stream; - if (*c++ != 'u') goto error_token; - if (*c++ != 'l') goto error_token; - if (*c++ != 'l') goto error_token; + if (*c++ != 0x75) goto error_token; + if (*c++ != 0x6C) goto error_token; + if (*c++ != 0x6C) goto error_token; state--; CALL_STATE_CALLBACK(parser->callbacks, null_found); goto select_state; @@ -550,7 +555,7 @@ error_token: parser->result = jsonlite_result_invalid_token; error_utf8: parser->result = jsonlite_result_invalid_utf8; goto end; // End of stream states. -end_of_stream_space: +end_of_stream_whitespaces: token_start = l; end_of_stream: parser->result = jsonlite_result_end_of_stream; @@ -559,7 +564,7 @@ error_utf8: parser->result = jsonlite_result_invalid_utf8; parser->rest = malloc(parser->rest_size); memcpy(parser->rest, token_start, parser->rest_size); // LCOV_EXCL_LINE } - goto end; + goto end; success: parser->result = jsonlite_result_ok; end: diff --git a/jsonlite/src/jsonlite_token.c b/jsonlite/src/jsonlite_token.c index f37eacb..daddb97 100644 --- a/jsonlite/src/jsonlite_token.c +++ b/jsonlite/src/jsonlite_token.c @@ -17,6 +17,7 @@ #include "../include/jsonlite_token.h" #endif +#include #include #ifdef _MSC_VER @@ -34,16 +35,16 @@ static uint32_t __inline jsonlite_clz( uint32_t x ) { #endif -uint8_t jsonlite_hex_char_to_uint8(uint8_t c) { - uint8_t res = 0xFF; - if (c >= '0' && c <= '9') { - res = c - '0'; - } else if (c >= 'a' && c <= 'f') { - res = c - 'a' + 10; - } else if (c >= 'A' && c <= 'F') { - res = c - 'A' + 10; +static uint8_t jsonlite_hex_char_to_uint8(uint8_t c) { + if (c >= 'a') { + return c - 'a' + 10; } - return res; + + if (c >= 'A') { + return c - 'A' + 10; + } + + return c - '0'; } static int unicode_char_to_utf16(uint32_t ch, uint16_t *utf16) { @@ -60,12 +61,12 @@ static int unicode_char_to_utf16(uint32_t ch, uint16_t *utf16) { return 2; } -size_t jsonlite_token_decode_size_for_uft8(jsonlite_token *ts) { +size_t jsonlite_token_size_of_uft8(jsonlite_token *ts) { return (size_t)(ts->end - ts->start + 1); } -size_t jsonlite_token_decode_to_uft8(jsonlite_token *ts, uint8_t **buffer) { - size_t size = jsonlite_token_decode_size_for_uft8(ts); +size_t jsonlite_token_to_uft8(jsonlite_token *ts, uint8_t **buffer) { + size_t size = jsonlite_token_size_of_uft8(ts); const uint8_t *p = ts->start; const uint8_t *l = ts->end; uint32_t value, utf32; @@ -145,12 +146,12 @@ size_t jsonlite_token_decode_to_uft8(jsonlite_token *ts, uint8_t **buffer) { return c - *buffer; } -size_t jsonlite_token_decode_size_for_uft16(jsonlite_token *ts) { +size_t jsonlite_token_size_of_uft16(jsonlite_token *ts) { return (ts->end - ts->start + 1) * sizeof(uint16_t); } -size_t jsonlite_token_decode_to_uft16(jsonlite_token *ts, uint16_t **buffer) { - size_t size = jsonlite_token_decode_size_for_uft16(ts); +size_t jsonlite_token_to_uft16(jsonlite_token *ts, uint16_t **buffer) { + size_t size = jsonlite_token_size_of_uft16(ts); const uint8_t *p = ts->start; const uint8_t *l = ts->end; uint16_t utf16; @@ -207,3 +208,41 @@ size_t jsonlite_token_decode_to_uft16(jsonlite_token *ts, uint16_t **buffer) { *c = 0; return (c - *buffer) * sizeof(uint16_t); } + +long jsonlite_token_to_long(jsonlite_token *token) { + long res = 0; + int negative = (token->type.number & jsonlite_number_negative) == jsonlite_number_negative; + int length = token->end - token->start - negative; + const uint8_t *c = token->start + negative; + switch (length & 3) { + for (; length > 0; length -= 4) { + case 0: res = res * 10 + *c++ - '0'; + case 3: res = res * 10 + *c++ - '0'; + case 2: res = res * 10 + *c++ - '0'; + case 1: res = res * 10 + *c++ - '0'; + } + } + + return negative ? -res : res; +} + +long long jsonlite_token_to_long_long(jsonlite_token *token) { + long long res = 0; + int negative = (token->type.number & jsonlite_number_negative) == jsonlite_number_negative; + int length = token->end - token->start - negative; + const uint8_t *c = token->start + negative; + switch (length & 7) { + for (; length > 0; length -= 8) { + case 0: res = res * 10 + *c++ - '0'; + case 7: res = res * 10 + *c++ - '0'; + case 6: res = res * 10 + *c++ - '0'; + case 5: res = res * 10 + *c++ - '0'; + case 4: res = res * 10 + *c++ - '0'; + case 3: res = res * 10 + *c++ - '0'; + case 2: res = res * 10 + *c++ - '0'; + case 1: res = res * 10 + *c++ - '0'; + } + } + + return negative ? -res : res; +} diff --git a/objc/JsonLiteParser.m b/objc/JsonLiteParser.m index 8536b61..cf77e29 100644 --- a/objc/JsonLiteParser.m +++ b/objc/JsonLiteParser.m @@ -65,44 +65,6 @@ static Class class_JsonLiteStringToken; static Class class_JsonLiteNumberToken; -static long token_to_long(jsonlite_token *token) { - long res = 0; - int negative = (token->type.number & jsonlite_number_negative) == jsonlite_number_negative; - ptrdiff_t len = token->end - token->start - negative; - const uint8_t *c = token->start + negative; - switch (len & 3) { - for (; len > 0; len -= 4) { - case 0: res = res * 10 + *c++ - '0'; - case 3: res = res * 10 + *c++ - '0'; - case 2: res = res * 10 + *c++ - '0'; - case 1: res = res * 10 + *c++ - '0'; - } - } - - return negative ? -res : res; -} - -static long long token_to_long_long(jsonlite_token *token) { - long long res = 0; - int negative = (token->type.number & jsonlite_number_negative) == jsonlite_number_negative; - ptrdiff_t len = token->end - token->start - negative; - const uint8_t *c = token->start + negative; - switch (len & 7) { - for (; len > 0; len -= 8) { - case 0: res = res * 10 + *c++ - '0'; - case 7: res = res * 10 + *c++ - '0'; - case 6: res = res * 10 + *c++ - '0'; - case 5: res = res * 10 + *c++ - '0'; - case 4: res = res * 10 + *c++ - '0'; - case 3: res = res * 10 + *c++ - '0'; - case 2: res = res * 10 + *c++ - '0'; - case 1: res = res * 10 + *c++ - '0'; - } - } - - return negative ? -res : res; -} - @implementation JsonLiteToken - (id)copyValue { @@ -139,7 +101,7 @@ @implementation JsonLiteStringToken - (NSString *)allocEscapedString:(jsonlite_token *)ts { void *buffer = NULL; - size_t size = jsonlite_token_decode_to_uft16(ts, (uint16_t **)&buffer); + size_t size = jsonlite_token_to_uft16(ts, (uint16_t **)&buffer); NSString *str = (NSString *)CFStringCreateWithBytesNoCopy(NULL, (const UInt8 *)buffer, size, @@ -205,10 +167,10 @@ - (id)copyValue { } else { ptrdiff_t length = token->end - token->start; if (length < (ptrdiff_t)log10(LONG_MAX)) { - long i = token_to_long(token); + long i = jsonlite_token_to_long(token); number = CFNumberCreate(NULL, kCFNumberLongType, &i); } else { - long long i = token_to_long_long(token); + long long i = jsonlite_token_to_long_long(token); number = CFNumberCreate(NULL, kCFNumberLongLongType, &i); } } diff --git a/tests/JsonLiteObjCInputs.m b/tests/JsonLiteObjCInputs.m index 2adcce7..a99961c 100644 --- a/tests/JsonLiteObjCInputs.m +++ b/tests/JsonLiteObjCInputs.m @@ -146,10 +146,10 @@ - (void)testJsonLiteParserTokenC { token.end = str + sizeof(str) - 1; token.type.string = jsonlite_string_ascii; - size_t size = jsonlite_token_decode_size_for_uft8(&token); + size_t size = jsonlite_token_size_of_uft8(&token); STAssertTrue(size == sizeof(str), @"Incorrect size - %d", size); - size = jsonlite_token_decode_to_uft8(&token, &buffer); + size = jsonlite_token_to_uft8(&token, &buffer); STAssertTrue(buffer != NULL, @"Buffer is null"); STAssertTrue(size == strlen((char *)str), @"Size is not zero"); STAssertTrue(memcmp(str, buffer, sizeof(str)) == 0, @"String are not equal"); @@ -157,53 +157,16 @@ - (void)testJsonLiteParserTokenC { free(buffer); buffer = NULL; - size = jsonlite_token_decode_size_for_uft16(&token); + size = jsonlite_token_size_of_uft16(&token); STAssertTrue(size == sizeof(str) * 2, @"Incorrect size - %d", size); - size = jsonlite_token_decode_to_uft16(&token, &unicode); + size = jsonlite_token_to_uft16(&token, &unicode); STAssertTrue(unicode != NULL, @"Buffer is null"); STAssertTrue(size == sizeof(str16) - sizeof(uint16_t), @"Size is not zero"); STAssertTrue(memcmp(str16, unicode, sizeof(str16)) == 0, @"String are not equal"); free(unicode); unicode = NULL; - - uint8_t n = jsonlite_hex_char_to_uint8('0'); - STAssertTrue(n == 0, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('9'); - STAssertTrue(n == 9, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('5'); - STAssertTrue(n == 5, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('a'); - STAssertTrue(n == 10, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('f'); - STAssertTrue(n == 15, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('c'); - STAssertTrue(n == 12, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('A'); - STAssertTrue(n == 10, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('F'); - STAssertTrue(n == 15, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('C'); - STAssertTrue(n == 12, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('\0'); - STAssertTrue(n == 0xFF, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('z'); - STAssertTrue(n == 0xFF, @"Incorrect value"); - - n = jsonlite_hex_char_to_uint8('Z'); - STAssertTrue(n == 0xFF, @"Incorrect value"); - } - (void)testJsonLiteParserObjC { diff --git a/tests/JsonLiteObjCTokenTests.m b/tests/JsonLiteObjCTokenTests.m index 28bf162..0c4ea4a 100644 --- a/tests/JsonLiteObjCTokenTests.m +++ b/tests/JsonLiteObjCTokenTests.m @@ -28,25 +28,6 @@ - (void)testToken { STAssertNotNil(obj, @"Object is nil"); } -- (void)testTokenFunctions { - char *token_string = "abc"; - jsonlite_token token; - token.start = (uint8_t *)token_string; - token.end = (uint8_t *)token_string + sizeof(token_string) - 1; - - size_t size = jsonlite_hex_char_to_uint8('Q'); - STAssertTrue(size == 0xFF, @"Is not 0XFF"); - - size = jsonlite_hex_char_to_uint8('7'); - STAssertTrue(size == 7, @"Is not 7"); - - size = jsonlite_hex_char_to_uint8('a'); - STAssertTrue(size == 10, @"Is not 10"); - - size = jsonlite_hex_char_to_uint8('A'); - STAssertTrue(size == 10, @"Is not 10"); -} - - (void)testZeroTerminatedByTab { NSString *json = @"[0\t]"; NSData *data = [json dataUsingEncoding:NSUTF8StringEncoding]; diff --git a/tests/JsonLiteObjCUnicodeTests.m b/tests/JsonLiteObjCUnicodeTests.m index 251e7ae..96ae032 100644 --- a/tests/JsonLiteObjCUnicodeTests.m +++ b/tests/JsonLiteObjCUnicodeTests.m @@ -559,7 +559,7 @@ static void string_found(jsonlite_callback_context *ctx, struct jsonlite_token * UnicodeTestCtx *context = (UnicodeTestCtx *)ctx->client_state; NSDictionary *dict = context->dict; uint8_t *buffer = NULL; - size_t size = jsonlite_token_decode_to_uft8(token, &buffer); + size_t size = jsonlite_token_to_uft8(token, &buffer); NSString *str = [[NSString alloc] initWithBytes:buffer length:size encoding:NSUTF8StringEncoding]; [str autorelease]; @@ -580,7 +580,7 @@ static void string_found(jsonlite_callback_context *ctx, struct jsonlite_token * static void string_token_found(jsonlite_callback_context *ctx, struct jsonlite_token *token) { uint8_t *buffer_utf8 = NULL; uint16_t *buffer_utf16 = NULL; - size_t size = jsonlite_token_decode_to_uft8(token, &buffer_utf8); + size_t size = jsonlite_token_to_uft8(token, &buffer_utf8); NSString *str1 = [[NSString alloc] initWithBytes:buffer_utf8 length:size @@ -588,7 +588,7 @@ static void string_token_found(jsonlite_callback_context *ctx, struct jsonlite_t [str1 autorelease]; free(buffer_utf8); - size = jsonlite_token_decode_to_uft16(token, &buffer_utf16); + size = jsonlite_token_to_uft16(token, &buffer_utf16); NSString *str2 = (NSString *)CFStringCreateWithBytesNoCopy(NULL, (const UInt8 *)buffer_utf16, size,