From 1257c39b331187566f628482818695457af2ba4f Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 21 Apr 2020 18:02:36 +0200 Subject: [PATCH 1/2] Parse HTTP header with PicoHTTPParser For now replace the simple piece of code that detects the end of the header. This actually adds code instead of simplifying. Simplification will occur later on, when the parsed header is returned to callers. --- Makefile.am | 3 +- src/http.c | 105 +++++-- src/picohttpparser.c | 645 +++++++++++++++++++++++++++++++++++++++++++ src/picohttpparser.h | 87 ++++++ tests/lint/run.sh | 8 +- 5 files changed, 823 insertions(+), 25 deletions(-) create mode 100644 src/picohttpparser.c create mode 100644 src/picohttpparser.h diff --git a/Makefile.am b/Makefile.am index 78319800..7a11c811 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,7 +7,8 @@ openfortivpn_SOURCES = src/config.c src/config.h src/hdlc.c src/hdlc.h \ src/tunnel.h src/main.c src/ssl.h src/xml.c \ src/xml.h src/userinput.c src/userinput.h \ src/openssl_hostname_validation.c \ - src/openssl_hostname_validation.h + src/openssl_hostname_validation.h \ + src/picohttpparser.c src/picohttpparser.h openfortivpn_CFLAGS = -Wall -pedantic openfortivpn_CPPFLAGS = -DSYSCONFDIR=\"$(sysconfdir)\" \ -DPPP_PATH=\"@PPP_PATH@\" \ diff --git a/src/http.c b/src/http.c index 1be3d538..ee90a41c 100644 --- a/src/http.c +++ b/src/http.c @@ -21,6 +21,7 @@ #include "ipv4.h" #include "userinput.h" #include "log.h" +#include "picohttpparser.h" #include #include @@ -145,6 +146,57 @@ static const char *find_header( } +static inline int header_name_cmp(const struct phr_header header, const char *s) +{ + return strncmp(header.name, s, header.name_len); +} + + +static inline int header_value_cmp(const struct phr_header header, const char *s) +{ + return strncmp(header.value, s, header.value_len); +} + + +/* + * Extract "Content-Size" and "Transfer-Encoding: chunked" from headers. + * + * @param[out] content_size + * @param[out] chunked + * @return 1 in case of success + * < 0 in case of error + */ +static int parse_response_header(const struct phr_header *headers, size_t num_headers, + uint32_t *content_size, int *chunked) +{ + uint32_t content_length = 0; + int transfer_encoding_chunked = 0; + + for (int i = 0; i != num_headers; ++i) { + if (header_name_cmp(headers[i], "Content-Length") == 0) { + long l = strtol(headers[i].name, NULL, 10); + + if (errno || l < 0) + return ERR_HTTP_INVALID; + else if (l > UINT32_MAX) + return ERR_HTTP_TOO_LONG; + content_length = l; + } + if (header_name_cmp(headers[i], "Transfer-Encoding") == 0) { + if (header_value_cmp(headers[i], "chunked") == 0) + transfer_encoding_chunked = 1; + else + return ERR_HTTP_INVALID; + } + } + if (content_length && transfer_encoding_chunked) + return ERR_HTTP_INVALID; + *content_size = content_length; + *chunked = transfer_encoding_chunked; + return 1; +} + + /* * Receives data from the HTTP server. * @@ -162,7 +214,7 @@ int http_receive( { uint32_t capacity = HTTP_BUFFER_SIZE; char *buffer; - uint32_t bytes_read = 0; + uint32_t bytes_read = 0, last_bytes_read = 0; uint32_t header_size = 0; uint32_t content_size = 0; int chunked = 0; @@ -184,31 +236,44 @@ int http_receive( free(buffer); return ERR_HTTP_SSL; } + last_bytes_read = bytes_read; bytes_read += n; log_debug_details("%s:\n%s\n", __func__, buffer); if (!header_size) { /* Have we reached the end of the HTTP header? */ - static const char EOH[4] = "\r\n\r\n"; - const char *eoh = memmem(buffer, bytes_read, - EOH, sizeof(EOH)); - - if (eoh) { - header_size = eoh - buffer + sizeof(EOH); - - /* Get the body size. */ - const char *header = find_header(buffer, - "Content-Length: ", - header_size); - - if (header) - content_size = atoi(header); - - if (find_header(buffer, - "Transfer-Encoding: chunked", - header_size)) - chunked = 1; + int minor_version, status; + const char *msg; + size_t msg_len; + struct phr_header headers[100]; // FIXME + size_t num_headers = ARRAY_SIZE(headers); + + n = phr_parse_response(buffer, bytes_read, + &minor_version, &status, + &msg, &msg_len, + headers, &num_headers, + last_bytes_read); + if (n > 0) { + header_size = n; + n = parse_response_header(headers, num_headers, + &content_size, &chunked); + if (n < 0) { + free(buffer); + return n; + } + if (content_size > UINT32_MAX - header_size) { + free(buffer); + return ERR_HTTP_TOO_LONG; + } else if (content_size == 0 && !chunked) { + free(buffer); + return ERR_HTTP_INVALID; + } + } else if (n == -2) { + /* response is partial, continue the loop */ + } else { + /* failed to parse the response */ + assert(n == -1); } } diff --git a/src/picohttpparser.c b/src/picohttpparser.c new file mode 100644 index 00000000..74ccc3ef --- /dev/null +++ b/src/picohttpparser.c @@ -0,0 +1,645 @@ +/* + * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase, + * Shigeo Mitsunari + * + * The software is licensed under either the MIT License (below) or the Perl + * license. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +#include +#include +#ifdef __SSE4_2__ +#ifdef _MSC_VER +#include +#else +#include +#endif +#endif +#include "picohttpparser.h" + +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#ifdef _MSC_VER +#define ALIGNED(n) _declspec(align(n)) +#else +#define ALIGNED(n) __attribute__((aligned(n))) +#endif + +#define IS_PRINTABLE_ASCII(c) ((unsigned char)(c)-040u < 0137u) + +#define CHECK_EOF() \ + if (buf == buf_end) { \ + *ret = -2; \ + return NULL; \ + } + +#define EXPECT_CHAR_NO_CHECK(ch) \ + if (*buf++ != ch) { \ + *ret = -1; \ + return NULL; \ + } + +#define EXPECT_CHAR(ch) \ + CHECK_EOF(); \ + EXPECT_CHAR_NO_CHECK(ch); + +#define ADVANCE_TOKEN(tok, toklen) \ + do { \ + const char *tok_start = buf; \ + static const char ALIGNED(16) ranges2[16] = "\000\040\177\177"; \ + int found2; \ + buf = findchar_fast(buf, buf_end, ranges2, 4, &found2); \ + if (!found2) { \ + CHECK_EOF(); \ + } \ + while (1) { \ + if (*buf == ' ') { \ + break; \ + } else if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { \ + if ((unsigned char)*buf < '\040' || *buf == '\177') { \ + *ret = -1; \ + return NULL; \ + } \ + } \ + ++buf; \ + CHECK_EOF(); \ + } \ + tok = tok_start; \ + toklen = buf - tok_start; \ + } while (0) + +static const char *token_char_map = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\1\0\1\1\1\1\1\0\0\1\1\0\1\1\0\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0" + "\0\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\1\1" + "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\1\0\1\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; + +static const char *findchar_fast(const char *buf, const char *buf_end, const char *ranges, size_t ranges_size, int *found) +{ + *found = 0; +#if __SSE4_2__ + if (likely(buf_end - buf >= 16)) { + __m128i ranges16 = _mm_loadu_si128((const __m128i *)ranges); + + size_t left = (buf_end - buf) & ~15; + do { + __m128i b16 = _mm_loadu_si128((const __m128i *)buf); + int r = _mm_cmpestri(ranges16, ranges_size, b16, 16, _SIDD_LEAST_SIGNIFICANT | _SIDD_CMP_RANGES | _SIDD_UBYTE_OPS); + if (unlikely(r != 16)) { + buf += r; + *found = 1; + break; + } + buf += 16; + left -= 16; + } while (likely(left != 0)); + } +#else + /* suppress unused parameter warning */ + (void)buf_end; + (void)ranges; + (void)ranges_size; +#endif + return buf; +} + +static const char *get_token_to_eol(const char *buf, const char *buf_end, const char **token, size_t *token_len, int *ret) +{ + const char *token_start = buf; + +#ifdef __SSE4_2__ + static const char ALIGNED(16) ranges1[16] = "\0\010" /* allow HT */ + "\012\037" /* allow SP and up to but not including DEL */ + "\177\177"; /* allow chars w. MSB set */ + int found; + buf = findchar_fast(buf, buf_end, ranges1, 6, &found); + if (found) + goto FOUND_CTL; +#else + /* find non-printable char within the next 8 bytes, this is the hottest code; manually inlined */ + while (likely(buf_end - buf >= 8)) { +#define DOIT() \ + do { \ + if (unlikely(!IS_PRINTABLE_ASCII(*buf))) \ + goto NonPrintable; \ + ++buf; \ + } while (0) + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); + DOIT(); +#undef DOIT + continue; + NonPrintable: + if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { + goto FOUND_CTL; + } + ++buf; + } +#endif + for (;; ++buf) { + CHECK_EOF(); + if (unlikely(!IS_PRINTABLE_ASCII(*buf))) { + if ((likely((unsigned char)*buf < '\040') && likely(*buf != '\011')) || unlikely(*buf == '\177')) { + goto FOUND_CTL; + } + } + } +FOUND_CTL: + if (likely(*buf == '\015')) { + ++buf; + EXPECT_CHAR('\012'); + *token_len = buf - 2 - token_start; + } else if (*buf == '\012') { + *token_len = buf - token_start; + ++buf; + } else { + *ret = -1; + return NULL; + } + *token = token_start; + + return buf; +} + +static const char *is_complete(const char *buf, const char *buf_end, size_t last_len, int *ret) +{ + int ret_cnt = 0; + buf = last_len < 3 ? buf : buf + last_len - 3; + + while (1) { + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + CHECK_EOF(); + EXPECT_CHAR('\012'); + ++ret_cnt; + } else if (*buf == '\012') { + ++buf; + ++ret_cnt; + } else { + ++buf; + ret_cnt = 0; + } + if (ret_cnt == 2) { + return buf; + } + } + + *ret = -2; + return NULL; +} + +#define PARSE_INT(valp_, mul_) \ + if (*buf < '0' || '9' < *buf) { \ + buf++; \ + *ret = -1; \ + return NULL; \ + } \ + *(valp_) = (mul_) * (*buf++ - '0'); + +#define PARSE_INT_3(valp_) \ + do { \ + int res_ = 0; \ + PARSE_INT(&res_, 100) \ + *valp_ = res_; \ + PARSE_INT(&res_, 10) \ + *valp_ += res_; \ + PARSE_INT(&res_, 1) \ + *valp_ += res_; \ + } while (0) + +/* returned pointer is always within [buf, buf_end), or null */ +static const char *parse_http_version(const char *buf, const char *buf_end, int *minor_version, int *ret) +{ + /* we want at least [HTTP/1.] to try to parse */ + if (buf_end - buf < 9) { + *ret = -2; + return NULL; + } + EXPECT_CHAR_NO_CHECK('H'); + EXPECT_CHAR_NO_CHECK('T'); + EXPECT_CHAR_NO_CHECK('T'); + EXPECT_CHAR_NO_CHECK('P'); + EXPECT_CHAR_NO_CHECK('/'); + EXPECT_CHAR_NO_CHECK('1'); + EXPECT_CHAR_NO_CHECK('.'); + PARSE_INT(minor_version, 1); + return buf; +} + +static const char *parse_headers(const char *buf, const char *buf_end, struct phr_header *headers, size_t *num_headers, + size_t max_headers, int *ret) +{ + for (;; ++*num_headers) { + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + break; + } else if (*buf == '\012') { + ++buf; + break; + } + if (*num_headers == max_headers) { + *ret = -1; + return NULL; + } + if (!(*num_headers != 0 && (*buf == ' ' || *buf == '\t'))) { + /* parsing name, but do not discard SP before colon, see + * http://www.mozilla.org/security/announce/2006/mfsa2006-33.html */ + headers[*num_headers].name = buf; + static const char ALIGNED(16) ranges1[] = "\x00 " /* control chars and up to SP */ + "\"\"" /* 0x22 */ + "()" /* 0x28,0x29 */ + ",," /* 0x2c */ + "//" /* 0x2f */ + ":@" /* 0x3a-0x40 */ + "[]" /* 0x5b-0x5d */ + "{\377"; /* 0x7b-0xff */ + int found; + buf = findchar_fast(buf, buf_end, ranges1, sizeof(ranges1) - 1, &found); + if (!found) { + CHECK_EOF(); + } + while (1) { + if (*buf == ':') { + break; + } else if (!token_char_map[(unsigned char)*buf]) { + *ret = -1; + return NULL; + } + ++buf; + CHECK_EOF(); + } + if ((headers[*num_headers].name_len = buf - headers[*num_headers].name) == 0) { + *ret = -1; + return NULL; + } + ++buf; + for (;; ++buf) { + CHECK_EOF(); + if (!(*buf == ' ' || *buf == '\t')) { + break; + } + } + } else { + headers[*num_headers].name = NULL; + headers[*num_headers].name_len = 0; + } + const char *value; + size_t value_len; + if ((buf = get_token_to_eol(buf, buf_end, &value, &value_len, ret)) == NULL) { + return NULL; + } + /* remove trailing SPs and HTABs */ + const char *value_end = value + value_len; + for (; value_end != value; --value_end) { + const char c = *(value_end - 1); + if (!(c == ' ' || c == '\t')) { + break; + } + } + headers[*num_headers].value = value; + headers[*num_headers].value_len = value_end - value; + } + return buf; +} + +static const char *parse_request(const char *buf, const char *buf_end, const char **method, size_t *method_len, const char **path, + size_t *path_len, int *minor_version, struct phr_header *headers, size_t *num_headers, + size_t max_headers, int *ret) +{ + /* skip first empty line (some clients add CRLF after POST content) */ + CHECK_EOF(); + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + } else if (*buf == '\012') { + ++buf; + } + + /* parse request line */ + ADVANCE_TOKEN(*method, *method_len); + do { + ++buf; + } while (*buf == ' '); + ADVANCE_TOKEN(*path, *path_len); + do { + ++buf; + } while (*buf == ' '); + if (*method_len == 0 || *path_len == 0) { + *ret = -1; + return NULL; + } + if ((buf = parse_http_version(buf, buf_end, minor_version, ret)) == NULL) { + return NULL; + } + if (*buf == '\015') { + ++buf; + EXPECT_CHAR('\012'); + } else if (*buf == '\012') { + ++buf; + } else { + *ret = -1; + return NULL; + } + + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); +} + +int phr_parse_request(const char *buf_start, size_t len, const char **method, size_t *method_len, const char **path, + size_t *path_len, int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf_start + len; + size_t max_headers = *num_headers; + int r; + + *method = NULL; + *method_len = 0; + *path = NULL; + *path_len = 0; + *minor_version = -1; + *num_headers = 0; + + /* if last_len != 0, check if the request is complete (a fast countermeasure + againt slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_request(buf, buf_end, method, method_len, path, path_len, minor_version, headers, num_headers, max_headers, + &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +static const char *parse_response(const char *buf, const char *buf_end, int *minor_version, int *status, const char **msg, + size_t *msg_len, struct phr_header *headers, size_t *num_headers, size_t max_headers, int *ret) +{ + /* parse "HTTP/1.x" */ + if ((buf = parse_http_version(buf, buf_end, minor_version, ret)) == NULL) { + return NULL; + } + /* skip space */ + if (*buf != ' ') { + *ret = -1; + return NULL; + } + do { + ++buf; + } while (*buf == ' '); + /* parse status code, we want at least [:digit:][:digit:][:digit:] to try to parse */ + if (buf_end - buf < 4) { + *ret = -2; + return NULL; + } + PARSE_INT_3(status); + + /* get message includig preceding space */ + if ((buf = get_token_to_eol(buf, buf_end, msg, msg_len, ret)) == NULL) { + return NULL; + } + if (*msg_len == 0) { + /* ok */ + } else if (**msg == ' ') { + /* remove preceding space */ + do { + ++*msg; + --*msg_len; + } while (**msg == ' '); + } else { + /* garbage found after status code */ + *ret = -1; + return NULL; + } + + return parse_headers(buf, buf_end, headers, num_headers, max_headers, ret); +} + +int phr_parse_response(const char *buf_start, size_t len, int *minor_version, int *status, const char **msg, size_t *msg_len, + struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf + len; + size_t max_headers = *num_headers; + int r; + + *minor_version = -1; + *status = 0; + *msg = NULL; + *msg_len = 0; + *num_headers = 0; + + /* if last_len != 0, check if the response is complete (a fast countermeasure + against slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_response(buf, buf_end, minor_version, status, msg, msg_len, headers, num_headers, max_headers, &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +int phr_parse_headers(const char *buf_start, size_t len, struct phr_header *headers, size_t *num_headers, size_t last_len) +{ + const char *buf = buf_start, *buf_end = buf + len; + size_t max_headers = *num_headers; + int r; + + *num_headers = 0; + + /* if last_len != 0, check if the response is complete (a fast countermeasure + against slowloris */ + if (last_len != 0 && is_complete(buf, buf_end, last_len, &r) == NULL) { + return r; + } + + if ((buf = parse_headers(buf, buf_end, headers, num_headers, max_headers, &r)) == NULL) { + return r; + } + + return (int)(buf - buf_start); +} + +enum { + CHUNKED_IN_CHUNK_SIZE, + CHUNKED_IN_CHUNK_EXT, + CHUNKED_IN_CHUNK_DATA, + CHUNKED_IN_CHUNK_CRLF, + CHUNKED_IN_TRAILERS_LINE_HEAD, + CHUNKED_IN_TRAILERS_LINE_MIDDLE +}; + +static int decode_hex(int ch) +{ + if ('0' <= ch && ch <= '9') { + return ch - '0'; + } else if ('A' <= ch && ch <= 'F') { + return ch - 'A' + 0xa; + } else if ('a' <= ch && ch <= 'f') { + return ch - 'a' + 0xa; + } else { + return -1; + } +} + +ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, size_t *_bufsz) +{ + size_t dst = 0, src = 0, bufsz = *_bufsz; + ssize_t ret = -2; /* incomplete */ + + while (1) { + switch (decoder->_state) { + case CHUNKED_IN_CHUNK_SIZE: + for (;; ++src) { + int v; + if (src == bufsz) + goto Exit; + if ((v = decode_hex(buf[src])) == -1) { + if (decoder->_hex_count == 0) { + ret = -1; + goto Exit; + } + break; + } + if (decoder->_hex_count == sizeof(size_t) * 2) { + ret = -1; + goto Exit; + } + decoder->bytes_left_in_chunk = decoder->bytes_left_in_chunk * 16 + v; + ++decoder->_hex_count; + } + decoder->_hex_count = 0; + decoder->_state = CHUNKED_IN_CHUNK_EXT; + /* fallthru */ + case CHUNKED_IN_CHUNK_EXT: + /* RFC 7230 A.2 "Line folding in chunk extensions is disallowed" */ + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] == '\012') + break; + } + ++src; + if (decoder->bytes_left_in_chunk == 0) { + if (decoder->consume_trailer) { + decoder->_state = CHUNKED_IN_TRAILERS_LINE_HEAD; + break; + } else { + goto Complete; + } + } + decoder->_state = CHUNKED_IN_CHUNK_DATA; + /* fallthru */ + case CHUNKED_IN_CHUNK_DATA: { + size_t avail = bufsz - src; + if (avail < decoder->bytes_left_in_chunk) { + if (dst != src) + memmove(buf + dst, buf + src, avail); + src += avail; + dst += avail; + decoder->bytes_left_in_chunk -= avail; + goto Exit; + } + if (dst != src) + memmove(buf + dst, buf + src, decoder->bytes_left_in_chunk); + src += decoder->bytes_left_in_chunk; + dst += decoder->bytes_left_in_chunk; + decoder->bytes_left_in_chunk = 0; + decoder->_state = CHUNKED_IN_CHUNK_CRLF; + } + /* fallthru */ + case CHUNKED_IN_CHUNK_CRLF: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] != '\015') + break; + } + if (buf[src] != '\012') { + ret = -1; + goto Exit; + } + ++src; + decoder->_state = CHUNKED_IN_CHUNK_SIZE; + break; + case CHUNKED_IN_TRAILERS_LINE_HEAD: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] != '\015') + break; + } + if (buf[src++] == '\012') + goto Complete; + decoder->_state = CHUNKED_IN_TRAILERS_LINE_MIDDLE; + /* fallthru */ + case CHUNKED_IN_TRAILERS_LINE_MIDDLE: + for (;; ++src) { + if (src == bufsz) + goto Exit; + if (buf[src] == '\012') + break; + } + ++src; + decoder->_state = CHUNKED_IN_TRAILERS_LINE_HEAD; + break; + default: + assert(!"decoder is corrupt"); + } + } + +Complete: + ret = bufsz - src; +Exit: + if (dst != src) + memmove(buf + dst, buf + src, bufsz - src); + *_bufsz = dst; + return ret; +} + +int phr_decode_chunked_is_in_data(struct phr_chunked_decoder *decoder) +{ + return decoder->_state == CHUNKED_IN_CHUNK_DATA; +} + +#undef CHECK_EOF +#undef EXPECT_CHAR +#undef ADVANCE_TOKEN diff --git a/src/picohttpparser.h b/src/picohttpparser.h new file mode 100644 index 00000000..0849f844 --- /dev/null +++ b/src/picohttpparser.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2009-2014 Kazuho Oku, Tokuhiro Matsuno, Daisuke Murase, + * Shigeo Mitsunari + * + * The software is licensed under either the MIT License (below) or the Perl + * license. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef picohttpparser_h +#define picohttpparser_h + +#include + +#ifdef _MSC_VER +#define ssize_t intptr_t +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* contains name and value of a header (name == NULL if is a continuing line + * of a multiline header */ +struct phr_header { + const char *name; + size_t name_len; + const char *value; + size_t value_len; +}; + +/* returns number of bytes consumed if successful, -2 if request is partial, + * -1 if failed */ +int phr_parse_request(const char *buf, size_t len, const char **method, size_t *method_len, const char **path, size_t *path_len, + int *minor_version, struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* ditto */ +int phr_parse_response(const char *_buf, size_t len, int *minor_version, int *status, const char **msg, size_t *msg_len, + struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* ditto */ +int phr_parse_headers(const char *buf, size_t len, struct phr_header *headers, size_t *num_headers, size_t last_len); + +/* should be zero-filled before start */ +struct phr_chunked_decoder { + size_t bytes_left_in_chunk; /* number of bytes left in current chunk */ + char consume_trailer; /* if trailing headers should be consumed */ + char _hex_count; + char _state; +}; + +/* the function rewrites the buffer given as (buf, bufsz) removing the chunked- + * encoding headers. When the function returns without an error, bufsz is + * updated to the length of the decoded data available. Applications should + * repeatedly call the function while it returns -2 (incomplete) every time + * supplying newly arrived data. If the end of the chunked-encoded data is + * found, the function returns a non-negative number indicating the number of + * octets left undecoded at the tail of the supplied buffer. Returns -1 on + * error. + */ +ssize_t phr_decode_chunked(struct phr_chunked_decoder *decoder, char *buf, size_t *bufsz); + +/* returns if the chunked decoder is in middle of chunked data */ +int phr_decode_chunked_is_in_data(struct phr_chunked_decoder *decoder); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tests/lint/run.sh b/tests/lint/run.sh index 2e2e574a..b370e881 100755 --- a/tests/lint/run.sh +++ b/tests/lint/run.sh @@ -3,12 +3,12 @@ rc=0 -./tests/lint/eol-at-eof.sh $(git ls-files | grep -v openssl_hostname_validation) || rc=1 +./tests/lint/eol-at-eof.sh $(git ls-files | grep -v 'openssl_hostname_validation\|picohttpparser') || rc=1 -./tests/lint/line_length.py $(git ls-files '*.[ch]' | grep -v openssl_hostname_validation) || rc=1 +./tests/lint/line_length.py $(git ls-files '*.[ch]' | grep -v 'openssl_hostname_validation\|picohttpparser') || rc=1 -./tests/lint/astyle.sh $(git ls-files '*.[ch]' | grep -v openssl_hostname_validation) || rc=1 +./tests/lint/astyle.sh $(git ls-files '*.[ch]' | grep -v 'openssl_hostname_validation\|picohttpparser') || rc=1 -./tests/lint/checkpatch.sh $(git ls-files '*.[ch]' | grep -v openssl_hostname_validation) || rc=1 +./tests/lint/checkpatch.sh $(git ls-files '*.[ch]' | grep -v 'openssl_hostname_validation\|picohttpparser') || rc=1 exit $rc From 77e25224ecea8f249f1e85fe75adc82b12a3d939 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 24 Apr 2020 18:21:15 +0200 Subject: [PATCH 2/2] WIP --- src/http.c | 85 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/src/http.c b/src/http.c index ee90a41c..527d9be8 100644 --- a/src/http.c +++ b/src/http.c @@ -43,6 +43,9 @@ /* * URL-encodes a string for HTTP requests. * + * RFC 3986 describes the percent-encoding mechanism: + * https://www.rfc-editor.org/info/rfc3986 + * * The dest buffer size MUST be at least strlen(str) * 3 + 1. * * @param[out] dest the buffer to write the URL-encoded string @@ -172,7 +175,7 @@ static int parse_response_header(const struct phr_header *headers, size_t num_he uint32_t content_length = 0; int transfer_encoding_chunked = 0; - for (int i = 0; i != num_headers; ++i) { + for (int i = 0; i != num_headers; i++) { if (header_name_cmp(headers[i], "Content-Length") == 0) { long l = strtol(headers[i].name, NULL, 10); @@ -218,6 +221,7 @@ int http_receive( uint32_t header_size = 0; uint32_t content_size = 0; int chunked = 0; + struct phr_chunked_decoder decoder = {0}; // zero-clear buffer = malloc(capacity + 1); // room for terminal '\0' if (buffer == NULL) @@ -225,7 +229,13 @@ int http_receive( while (1) { int n; + int minor_version, status; + const char *msg; + size_t msg_len; + struct phr_header headers[100]; // FIXME + size_t num_headers = ARRAY_SIZE(headers); + /* read SSL stream into buffer as long as ERR_SSL_AGAIN */ while ((n = safe_ssl_read(tunnel->ssl_handle, (uint8_t *) buffer + bytes_read, capacity - bytes_read)) == ERR_SSL_AGAIN) @@ -241,23 +251,22 @@ int http_receive( log_debug_details("%s:\n%s\n", __func__, buffer); - if (!header_size) { - /* Have we reached the end of the HTTP header? */ - int minor_version, status; - const char *msg; - size_t msg_len; - struct phr_header headers[100]; // FIXME - size_t num_headers = ARRAY_SIZE(headers); - + if (content_size == 0 && !chunked) { + /* parse the HTTP response header */ n = phr_parse_response(buffer, bytes_read, - &minor_version, &status, - &msg, &msg_len, - headers, &num_headers, - last_bytes_read); + &minor_version, &status, + &msg, &msg_len, + headers, &num_headers, + last_bytes_read); if (n > 0) { header_size = n; + log_error("*** header_size: %u\n", header_size); n = parse_response_header(headers, num_headers, &content_size, &chunked); + if (content_size) + log_error("*** content_size: %u\n", content_size); + if (chunked) + log_error("*** chunked: %d\n", chunked); if (n < 0) { free(buffer); return n; @@ -272,24 +281,46 @@ int http_receive( } else if (n == -2) { /* response is partial, continue the loop */ } else { - /* failed to parse the response */ + /* failed to parse the response header */ assert(n == -1); + free(buffer); + return ERR_HTTP_INVALID; } } - - if (header_size) { - /* Have we reached the end of the HTTP body? */ - if (chunked) { - static const char EOB[7] = "\r\n0\r\n\r\n"; - - /* Last chunk terminator. Done naively. */ - if (bytes_read >= sizeof(EOB) && - !memcmp(&buffer[bytes_read - sizeof(EOB)], - EOB, sizeof(EOB))) - break; + if (chunked) { + size_t bufsz = bytes_read - header_size; + + n = phr_decode_chunked(&decoder, + &buffer[header_size], &bufsz); + + if (n >= 0) { + /* body is complete */ + if (n > 0) { + /* garbage after body */ + log_warn("Garbage after body (%d bytes).\n", + n); + } + } else if (n == -2) { + /* body is incomplete, continue the loop */ } else { - if (bytes_read >= header_size + content_size) - break; + /* failed to parse the chunked body */ + assert(n == -1); + free(buffer); + return ERR_HTTP_INVALID; + } + //~ /* parse the HTTP response chunked body */ + //~ static const char EOB[7] = "\r\n0\r\n\r\n"; + + //~ /* Last chunk terminator. Done naively. */ + //~ if (bytes_read >= sizeof(EOB) && + //~ !memcmp(&buffer[bytes_read - sizeof(EOB)], + //~ EOB, sizeof(EOB))) + //~ break; + } else if (content_size > 0) { + /* read until the end of the HTTP response body */ + if (bytes_read >= header_size + content_size) { + buffer[bytes_read] = '\0'; + break; } }