Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add test for parsing one charactar at a time, fix #125 and #126 #124

Closed
wants to merge 4 commits into from
Closed

Add test for parsing one charactar at a time, fix #125 and #126 #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4450873
add test for parsing one charactar at a time
samoconnor Nov 22, 2017
9bef22e
fix #125
samoconnor Nov 22, 2017
2665984
1 char at a time test for requests
samoconnor Nov 22, 2017
9431200
Fix #126
samoconnor Nov 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/HTTP.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,4 @@ try
HTTP.parse(HTTP.Response, "HTTP/1.1 200 OK\r\n\r\n")
HTTP.parse(HTTP.Request, "GET / HTTP/1.1\r\n\r\n")
HTTP.get(HTTP.Client(nothing), "www.google.com")
end
end
91 changes: 52 additions & 39 deletions src/parser.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ mutable struct Parser
content_length::UInt64
fieldbuffer::Vector{UInt8}
valuebuffer::Vector{UInt8}
previous_field::Ref{String}
end

Parser() = Parser(s_start_req_or_res, 0x00, 0, 0, 0, 0, UInt8[], UInt8[])
Parser() = Parser(s_start_req_or_res, 0x00, 0, 0, 0, 0, UInt8[], UInt8[], Ref{String}())

function reset!(p::Parser)
p.state = start_state
Expand All @@ -44,6 +45,7 @@ function reset!(p::Parser)
p.content_length = 0x0000000000000000
empty!(p.fieldbuffer)
empty!(p.valuebuffer)
p.previous_field = Ref{String}()
return
end

Expand All @@ -55,15 +57,23 @@ macro nread(n)
end

onmessagebegin(r) = @debug(PARSING_DEBUG, "onmessagebegin")

function onurlbytes(p::Parser, bytes, i, j)
@debug(PARSING_DEBUG, "onurlbytes")
append!(p.valuebuffer, view(bytes, i:j))
return
end

# should we just make a copy of the byte vector for URI here?
function onurl(r, bytes, i, j)
function onurl(p::Parser, r)
@debug(PARSING_DEBUG, "onurl")
@debug(PARSING_DEBUG, i - j + 1)
@debug(PARSING_DEBUG, "'$(String(bytes[i:j]))'")
@debug(PARSING_DEBUG, "'$(String(p.valuebuffer))'")
@debug(PARSING_DEBUG, r.method)
uri = URIs.http_parser_parse_url(bytes, i, j - i + 1, r.method == CONNECT)
url = copy(p.valuebuffer)
uri = URIs.http_parser_parse_url(url, 1, length(url), r.method == CONNECT)
@debug(PARSING_DEBUG, uri)
setfield!(r, :uri, uri)
empty!(p.valuebuffer)
return
end
onstatus(r) = @debug(PARSING_DEBUG, "onstatus")
Expand All @@ -77,7 +87,7 @@ function onheadervalue(p::Parser, bytes, i, j)
append!(p.valuebuffer, view(bytes, i:j))
return
end
function onheadervalue(p, r, bytes, i, j, issetcookie, host, KEY, canonicalizeheaders)
function onheadervalue(p, r, bytes, i, j, issetcookie, host, canonicalizeheaders)
@debug(PARSING_DEBUG, "onheadervalue2")
append!(p.valuebuffer, view(bytes, i:j))
if canonicalizeheaders
Expand All @@ -88,13 +98,13 @@ function onheadervalue(p, r, bytes, i, j, issetcookie, host, KEY, canonicalizehe
val = unsafe_string(pointer(p.valuebuffer), length(p.valuebuffer))
if key == ""
# the header value was parsed in two parts,
# KEY[] holds the most recently parsed header field,
# p.previous_field[] holds the most recently parsed header field,
# and we already stored the first part of the header value in r.headers
# get the first part and concatenate it with the second part we have now
key = KEY[]
key = p.previous_field[]
setindex!(r.headers, string(r.headers[key], val), key)
else
KEY[] = key
p.previous_field[] = key
val2 = get!(r.headers, key, val)
val2 !== val && setindex!(r.headers, string(val2, ", ", val), key)
end
Expand Down Expand Up @@ -151,7 +161,9 @@ function parse(T::Type{<:Union{Request, Response}}, str;
err, headerscomplete, messagecomplete, upgrade = parse!(r, DEFAULT_PARSER, Vector{UInt8}(str);
lenient=lenient, maxuri=maxuri, maxheader=maxheader, maxbody=maxbody, maintask=maintask, canonicalizeheaders=canonicalizeheaders)
err != HPE_OK && throw(ParsingError("error parsing $T: $(ParsingErrorCodeMap[err])"))
extra[] = upgrade
if isassigned(upgrade)
extra[] = upgrade[]
end
return r
end

Expand All @@ -165,7 +177,7 @@ function parse!(r::Union{Request, Response}, parser, bytes, len=length(bytes);
lenient::Bool=true, host::String="", method::Method=GET,
maxuri::Int64=DEFAULT_MAX_URI, maxheader::Int64=DEFAULT_MAX_HEADER,
maxbody::Int64=DEFAULT_MAX_BODY, maintask::Task=current_task(),
canonicalizeheaders::Bool=true)::Tuple{ParsingErrorCode, Bool, Bool, String}
canonicalizeheaders::Bool=true)::Tuple{ParsingErrorCode, Bool, Bool, Ref{String}}
return parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader, maxbody, maintask, canonicalizeheaders)
end
function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader, maxbody, maintask, canonicalizeheaders)
Expand All @@ -174,19 +186,18 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
status_mark = url_mark = header_field_mark = header_field_end_mark = header_value_mark = body_mark = 0
errno = HPE_OK
upgrade = issetcookie = headersdone = false
KEY = Ref{String}()
@debug(PARSING_DEBUG, len)
@debug(PARSING_DEBUG, ParsingStateCode(p_state))
if len == 0
if p_state == s_body_identity_eof
parser.state = p_state
onmessagecomplete(r)
@debug(PARSING_DEBUG, "this 6")
return HPE_OK, true, true, ""
return HPE_OK, true, true, Ref{String}()
elseif @anyeq(p_state, s_dead, s_start_req_or_res, s_start_res, s_start_req)
return HPE_OK, false, false, ""
return HPE_OK, false, false, Ref{String}()
else
return HPE_INVALID_EOF_STATE, false, false, ""
return HPE_INVALID_EOF_STATE, false, false, Ref{String}()
end
end

Expand All @@ -200,13 +211,17 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
end
if @anyeq(p_state, s_req_path, s_req_schema, s_req_schema_slash, s_req_schema_slash_slash,
s_req_server_start, s_req_server, s_req_server_with_at,
s_req_query_string_start, s_req_query_string, s_req_fragment)
s_req_query_string_start, s_req_query_string, s_req_fragment,
s_req_fragment_start)
url_mark = 1
elseif p_state == s_res_status
status_mark = 1
end
p = 1
old_p = 0
while p <= len
@assert p > old_p;
old_p = p
@inbounds ch = Char(bytes[p])
@debug(PARSING_DEBUG, "top of main for-loop")
@debug(PARSING_DEBUG, Base.escape_string(string(ch)))
Expand Down Expand Up @@ -508,15 +523,17 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
p_state = s_req_http_start
parser.state = p_state
p - url_mark > maxuri && @err(HPE_URI_OVERFLOW)
onurl(r, bytes, url_mark, p-1)
onurlbytes(parser, bytes, url_mark, p-1)
onurl(parser, r)
url_mark = 0
elseif ch in (CR, LF)
r.major = Int16(0)
r.minor = Int16(9)
p_state = ifelse(ch == CR, s_req_line_almost_done, s_header_field_start)
parser.state = p_state
p - url_mark > maxuri && @err(HPE_URI_OVERFLOW)
onurl(r, bytes, url_mark, p-1)
onurlbytes(parser, bytes, url_mark, p-1)
onurl(parser, r)
url_mark = 0
else
p_state = URIs.parseurlchar(p_state, ch, strict)
Expand Down Expand Up @@ -640,7 +657,10 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
ch = Char(bytes[p])
@debug(PARSING_DEBUG, Base.escape_string(string(ch)))
c = (!strict && ch == ' ') ? ' ' : tokens[Int(ch)+1]
c == Char(0) && break
if c == Char(0)
@errorif(ch != ':', HPE_INVALID_HEADER_TOKEN)
break
end
h = parser.header_state
if h == h_general
@debug(PARSING_DEBUG, parser.header_state)
Expand Down Expand Up @@ -730,18 +750,14 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,

@nread(p - start)

if p >= len
p -= 1
@goto breakout
end
if ch == ':'
p_state = s_header_value_discard_ws
parser.state = p_state
header_field_end_mark = p
onheaderfield(parser, bytes, header_field_mark, p - 1)
header_field_mark = 0
else
@err(HPE_INVALID_HEADER_TOKEN)
@assert tokens[Int(ch)+1] != Char(0) || !strict && ch == ' '
end

elseif p_state == s_header_value_discard_ws
Expand Down Expand Up @@ -810,7 +826,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.header_state = h
parser.state = p_state
@debug(PARSING_DEBUG, "onheadervalue 1")
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, KEY, canonicalizeheaders)
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, canonicalizeheaders)
header_value_mark = 0
break
elseif ch == LF
Expand All @@ -819,7 +835,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.header_state = h
parser.state = p_state
@debug(PARSING_DEBUG, "onheadervalue 2")
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, KEY, canonicalizeheaders)
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, canonicalizeheaders)
header_value_mark = 0
@goto reexecute
elseif !lenient && !isheaderchar(ch)
Expand Down Expand Up @@ -970,10 +986,6 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,

@nread(p - start)

if p == len
p -= 1
end

elseif p_state == s_header_almost_done
@debug(PARSING_DEBUG, ParsingStateCode(p_state))
@errorif(ch != LF, HPE_LF_EXPECTED)
Expand Down Expand Up @@ -1024,7 +1036,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
p_state = s_header_field_start
parser.state = p_state
@debug(PARSING_DEBUG, "onheadervalue 3")
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, KEY, canonicalizeheaders)
onheadervalue(parser, r, bytes, header_value_mark, p - 1, issetcookie, host, canonicalizeheaders)
header_value_mark = 0
@goto reexecute
end
Expand Down Expand Up @@ -1085,15 +1097,15 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.state = p_state
onmessagecomplete(r)
@debug(PARSING_DEBUG, "this 1")
return errno, true, true, String(bytes[p+1:end])
return errno, true, true, Ref{String}(String(bytes[p+1:end]))
end

if parser.flags & F_SKIPBODY > 0
p_state = ifelse(http_should_keep_alive(parser, r), start_state, s_dead)
parser.state = p_state
onmessagecomplete(r)
@debug(PARSING_DEBUG, "this 2")
return errno, true, true, ""
return errno, true, true, Ref{String}()
elseif parser.flags & F_CHUNKED > 0
#= chunked encoding - ignore Content-Length header =#
p_state = s_chunk_size_start
Expand All @@ -1104,7 +1116,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.state = p_state
onmessagecomplete(r)
@debug(PARSING_DEBUG, "this 3")
return errno, true, true, ""
return errno, true, true, Ref{String}()
elseif parser.content_length != ULLONG_MAX
#= Content-Length header given and non-zero =#
p_state = s_body_identity
Expand All @@ -1116,7 +1128,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.state = p_state
onmessagecomplete(r)
@debug(PARSING_DEBUG, "this 4")
return errno, true, true, String(bytes[p+1:end])
return errno, true, true, Ref{String}(String(bytes[p+1:end]))
else
#= Read body until EOF =#
p_state = s_body_identity_eof
Expand Down Expand Up @@ -1173,7 +1185,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
if upgrade
#= Exit, the rest of the message is in a different protocol. =#
parser.state = p_state
return errno, true, true, String(bytes[p+1:end])
return errno, true, true, Ref{String}(String(bytes[p+1:end]))
end

elseif p_state == s_chunk_size_start
Expand Down Expand Up @@ -1301,7 +1313,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
@debug(PARSING_DEBUG, p)
header_value_mark > 0 && onheadervalue(parser, bytes, header_value_mark, min(len, p))
url_mark > 0 && (min(len, p) - url_mark > maxuri) && @err(HPE_URI_OVERFLOW)
url_mark > 0 && onurl(r, bytes, url_mark, min(len, p))
url_mark > 0 && onurlbytes(parser, bytes, url_mark, min(len, p))
@debug(PARSING_DEBUG, "this onbody 3")
body_mark > 0 && onbody(r, maintask, bytes, body_mark, min(len, p - 1))
status_mark > 0 && onstatus(r)
Expand All @@ -1312,7 +1324,8 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
b = p_state == start_state || p_state == s_dead
he = b | (headersdone || p_state >= s_headers_done)
m = b | (p_state >= s_message_done)
return errno, he, m, String(bytes[p:end])
ug = p >= len ? Ref{String}() : Ref{String}(String(bytes[p:end]))
return errno, he, m, ug

@label error
if errno == HPE_OK
Expand All @@ -1323,7 +1336,7 @@ function parse!(r, parser, bytes, len, lenient, host, method, maxuri, maxheader,
parser.header_state = 0x00
@debug(PARSING_DEBUG, "exiting due to error...")
@debug(PARSING_DEBUG, errno)
return errno, false, false, ""
return errno, false, false, Ref{String}()
end

#= Does the parser need to see an EOF to find the end of the message? =#
Expand Down
10 changes: 6 additions & 4 deletions src/precompile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ function _precompile_()
@assert precompile(HTTP.Cookies.pathmatch, (HTTP.Cookies.Cookie, String,))
@assert precompile(HTTP.onheaderfield, (HTTP.Parser, Array{UInt8, 1}, Int64, Int64,))
@assert precompile(HTTP.isjson, (Array{UInt8, 1}, UInt64, Int64,))
@assert precompile(HTTP.onheadervalue, (HTTP.Parser, HTTP.Response, Array{UInt8, 1}, Int64, Int64, Bool, String, Base.RefValue{String}, Bool))
@assert precompile(HTTP.onheadervalue, (HTTP.Parser, HTTP.Response, Array{UInt8, 1}, Int64, Int64, Bool, String, Bool))
@assert precompile(HTTP.isjson, (Array{UInt8, 1}, Int64, Int64,))
@assert precompile(HTTP.onurl, (HTTP.Response, Array{UInt8, 1}, Int64, Int64,))
@assert precompile(HTTP.onurlbytes, (HTTP.Parser, Array{UInt8, 1}, Int64, Int64,))
@assert precompile(HTTP.onurl, (HTTP.Parser, HTTP.Response,))
@assert precompile(HTTP.onurl, (HTTP.Parser, HTTP.Request,))
@assert precompile(HTTP.Response, (Int64, String,))
@assert precompile(HTTP.URIs.getindex, (Array{UInt8, 1}, HTTP.URIs.Offset,))
@assert precompile(HTTP.iscompressed, (Array{UInt8, 1},))
Expand Down Expand Up @@ -83,7 +85,7 @@ function _precompile_()
@assert precompile(HTTP.sniff, (Array{UInt8, 1},))
@assert precompile(HTTP.mark, (HTTP.Multipart{Base.IOStream},))
@assert precompile(HTTP.seek, (HTTP.Form, Int64,))
@assert precompile(HTTP.onheadervalue, (HTTP.Parser, HTTP.Request, Array{UInt8, 1}, Int64, Int64, Bool, String, Base.RefValue{String}, Bool))
@assert precompile(HTTP.onheadervalue, (HTTP.Parser, HTTP.Request, Array{UInt8, 1}, Int64, Int64, Bool, String, Bool))
@assert precompile(HTTP.FIFOBuffers.write, (HTTP.FIFOBuffers.FIFOBuffer, String,))
@assert precompile(HTTP.URIs.escape, (String, String,))
@assert precompile(HTTP.dead!, (HTTP.Connection{MbedTLS.SSLContext},))
Expand Down Expand Up @@ -197,4 +199,4 @@ function _precompile_()
@assert precompile(HTTP.startline, (Base.GenericIOBuffer{Array{UInt8, 1}}, HTTP.Request,))
end
end
_precompile_()
_precompile_()
4 changes: 2 additions & 2 deletions src/server.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ function process!(server::Server{T, H}, parser, request, i, tcp, rl, starttime,
response = HTTP.Response(417)
error = true
end
elseif length(upgrade) > 0
elseif isassigned(upgrade)
HTTP.@log "received upgrade request on connection i=$i"
response = HTTP.Response(501, "upgrade requests are not currently supported")
error = true
Expand Down Expand Up @@ -338,4 +338,4 @@ serve(; host::IPAddr=IPv4(127,0,0,1),
args...) =
serve(host, port, handler, logger; cert=cert, key=key, verbose=verbose, args...)

end # module
end # module
Loading